HTML tags. html_short_title = '%s Documentation' % release +# Deployment preview information, from Netlify +# (See netlify.toml and https://docs.netlify.com/configure-builds/environment-variables/#git-metadata) +html_context = { + "is_deployment_preview": os.getenv("IS_DEPLOYMENT_PREVIEW"), + "repository_url": os.getenv("REPOSITORY_URL"), + "pr_id": os.getenv("REVIEW_ID") +} + # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. html_last_updated_fmt = '%b %d, %Y' diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 98ccf4224804..460161cd3202 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -8,6 +8,19 @@ <a href="/3/{{ pagename }}{{ file_suffix }}">{% trans %} Python documentation for the current stable release{% endtrans %}</a>. </div> {%- endif %} + +{%- if is_deployment_preview %} +<div id="deployment-preview-warning" style="padding: .5em; text-align: center; background-color: #fff2ba; color: #6a580e;"> + <div style="float: right; margin-top: -10px; margin-left: 10px;"> + <a href="https://www.netlify.com"> + <img src="https://www.netlify.com/img/global/badges/netlify-color-accent.svg" alt="Deploys by Netlify" /> + </a> + </div> + {% trans %}This is a deploy preview created from a <a href="{{ repository_url }}/pull/{{ pr_id }}">pull request</a>. + For authoritative documentation, see the {% endtrans %} + <a href="https://docs.python.org/3/{{ pagename }}{{ file_suffix }}">{% trans %} the current stable release{% endtrans %}</a>. +</div> +{%- endif %} {% endblock %} {% block rootrellink %} diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 000000000000..52675b3d701e --- /dev/null +++ b/netlify.toml @@ -0,0 +1,8 @@ +[build] + base = "Doc/" + command = "make html" + publish = "build/html" + +[build.environment] + PYTHON_VERSION = "3.8" + IS_DEPLOYMENT_PREVIEW = "true" \ No newline at end of file From webhook-mailer at python.org Tue Dec 6 09:01:50 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 14:01:50 -0000 Subject: [Python-checkins] GH-99729: Unlink frames before clearing them (GH-100030) Message-ID: <mailman.2823.1670335311.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b72014c783e5698beb18ee1249597e510b8bcb5a commit: b72014c783e5698beb18ee1249597e510b8bcb5a branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T14:01:38Z summary: GH-99729: Unlink frames before clearing them (GH-100030) files: A Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst M Lib/test/test_frame.py M Python/bytecodes.c M Python/ceval.c M Python/frame.c M Python/generated_cases.c.h diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index a7db22007ded..ed413f105e5b 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -2,6 +2,7 @@ import re import sys import textwrap +import threading import types import unittest import weakref @@ -11,6 +12,7 @@ _testcapi = None from test import support +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -329,6 +331,46 @@ def f(): if old_enabled: gc.enable() + @support.cpython_only + @threading_helper.requires_working_threading() + def test_sneaky_frame_object_teardown(self): + + class SneakyDel: + def __del__(self): + """ + Stash a reference to the entire stack for walking later. + + It may look crazy, but you'd be surprised how common this is + when using a test runner (like pytest). The typical recipe is: + ResourceWarning + -Werror + a custom sys.unraisablehook. + """ + nonlocal sneaky_frame_object + sneaky_frame_object = sys._getframe() + + class SneakyThread(threading.Thread): + """ + A separate thread isn't needed to make this code crash, but it does + make crashes more consistent, since it means sneaky_frame_object is + backed by freed memory after the thread completes! + """ + + def run(self): + """Run SneakyDel.__del__ as this frame is popped.""" + ref = SneakyDel() + + sneaky_frame_object = None + t = SneakyThread() + t.start() + t.join() + # sneaky_frame_object can be anything, really, but it's crucial that + # SneakyThread.run's frame isn't anywhere on the stack while it's being + # torn down: + self.assertIsNotNone(sneaky_frame_object) + while sneaky_frame_object is not None: + self.assertIsNot( + sneaky_frame_object.f_code, SneakyThread.run.__code__ + ) + sneaky_frame_object = sneaky_frame_object.f_back @unittest.skipIf(_testcapi is None, 'need _testcapi') class TestCAPI(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst new file mode 100644 index 000000000000..3fe21a8a21bf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst @@ -0,0 +1,3 @@ +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 41dd1acc937d..d0480ac01eb6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -619,7 +619,10 @@ dummy_func( DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; } diff --git a/Python/ceval.c b/Python/ceval.c index 80bfa21ad0b6..9e4179e56071 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1009,14 +1009,6 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject return 0; } -static _PyInterpreterFrame * -pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - _PyInterpreterFrame *prev_frame = frame->previous; - _PyEvalFrameClearAndPop(tstate, frame); - return prev_frame; -} - int _Py_CheckRecursiveCallPy( PyThreadState *tstate) @@ -1432,7 +1424,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); if (frame == &entry_frame) { /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; diff --git a/Python/frame.c b/Python/frame.c index 52f6ef428291..b1525cca5112 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -127,6 +127,9 @@ _PyFrame_Clear(_PyInterpreterFrame *frame) * to have cleared the enclosing generator, if any. */ assert(frame->owner != FRAME_OWNED_BY_GENERATOR || _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); + // GH-99729: Clearing this frame can expose the stack (via finalizers). It's + // crucial that this frame has been unlinked, and is no longer visible: + assert(_PyThreadState_GET()->cframe->current_frame != frame); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3a403824b499..0805386866b3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -628,7 +628,10 @@ DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; } From webhook-mailer at python.org Tue Dec 6 10:16:20 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 15:16:20 -0000 Subject: [Python-checkins] [3.11] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#99949) Message-ID: <mailman.2824.1670339781.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3fae04b10e2655a20a3aadb5e0d63e87206d0c67 commit: 3fae04b10e2655a20a3aadb5e0d63e87206d0c67 branch: 3.11 author: Serhiy Storchaka <storchaka at gmail.com> committer: ambv <lukasz at langa.pl> date: 2022-12-06T16:15:44+01:00 summary: [3.11] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#99949) It no longer emits a deprecation warning if the current event loop was set. Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst M Doc/library/asyncio-eventloop.rst M Doc/library/asyncio-llapi-index.rst M Doc/library/asyncio-policy.rst M Lib/asyncio/events.py M Lib/test/test_asyncio/test_base_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_futures.py M Lib/test/test_asyncio/test_streams.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_asyncio/test_unix_events.py M Lib/test/test_coroutines.py M Modules/_asynciomodule.c diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index a10658df5df8..28b7a900583b 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -43,10 +43,12 @@ an event loop: Get the current event loop. - If there is no current event loop set in the current OS thread, - the OS thread is main, and :func:`set_event_loop` has not yet - been called, asyncio will create a new event loop and set it as the - current one. + When called from a coroutine or a callback (e.g. scheduled with + call_soon or similar API), this function will always return the + running event loop. + + If there is no running event loop set, the function will return + the result of ``get_event_loop_policy().get_event_loop()`` call. Because this function has rather complex behavior (especially when custom event loop policies are in use), using the @@ -58,10 +60,14 @@ an event loop: event loop. .. deprecated:: 3.10 - Emits a deprecation warning if there is no running event loop. - In future Python releases, this function may become an alias of - :func:`get_running_loop` and will accordingly raise a - :exc:`RuntimeError` if there is no running event loop. + Deprecation warning is emitted if there is no current event loop. + In Python 3.12 it will be an error. + + .. note:: + In Python versions 3.10.0--3.10.8 and 3.11.0 this function + (and other functions which used it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if + the current loop was set. .. function:: set_event_loop(loop) diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index b7ad888a7b67..9ce48a24444e 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,7 +19,7 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (current or via the policy). + - Get an event loop instance (running or current via the current policy). * - :func:`asyncio.set_event_loop` - Set the event loop as current via the current policy. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index bfc3e3090fdc..d0af45febd14 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -112,6 +112,11 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. + .. deprecated:: 3.11.1 + :meth:`get_event_loop` now emits a :exc:`DeprecationWarning` if there + is no current event loop set and a new event loop has been implicitly + created. In Python 3.12 it will be an error. + .. class:: WindowsSelectorEventLoopPolicy diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0d26ea545baa..af3f9e970b51 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -671,6 +671,21 @@ def get_event_loop(self): if (self._local._loop is None and not self._local._set_called and threading.current_thread() is threading.main_thread()): + stacklevel = 2 + try: + f = sys._getframe(1) + except AttributeError: + pass + else: + while f: + module = f.f_globals.get('__name__') + if not (module == 'asyncio' or module.startswith('asyncio.')): + break + f = f.f_back + stacklevel += 1 + import warnings + warnings.warn('There is no current event loop', + DeprecationWarning, stacklevel=stacklevel) self.set_event_loop(self.new_event_loop()) if self._local._loop is None: @@ -786,12 +801,13 @@ def get_event_loop(): def _get_event_loop(stacklevel=3): + # This internal method is going away in Python 3.12, left here only for + # backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. + # Similarly, this method's C equivalent in _asyncio is going away as well. + # See GH-99949 for more details. current_loop = _get_running_loop() if current_loop is not None: return current_loop - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 063174adacfa..6ba602dd619f 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -746,7 +746,7 @@ async def coro(): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio', - 'loop = asyncio.get_event_loop()', + 'loop = asyncio.new_event_loop()', 'print(loop.get_debug())')) # Test with -E to not fail if the unit test was run with diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 05d9107b28e2..c431fea40163 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2547,8 +2547,9 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.assertEqual(cm.filename, __file__) self.assertIsInstance(loop, asyncio.AbstractEventLoop) self.assertIs(policy._local._loop, loop) @@ -2562,7 +2563,10 @@ def test_get_event_loop_calls_set_event_loop(self): policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.addCleanup(loop.close) + self.assertEqual(cm.filename, __file__) # policy._local._loop must be set through .set_event_loop() # (the unix DefaultEventLoopPolicy needs this call to attach @@ -2596,7 +2600,8 @@ def test_new_event_loop(self): def test_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() - old_loop = policy.get_event_loop() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) self.assertRaises(TypeError, policy.set_event_loop, object()) @@ -2709,15 +2714,11 @@ def get_event_loop(self): asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2731,16 +2732,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) - + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) @@ -2764,10 +2760,8 @@ def test_get_event_loop_returns_running_loop2(self): self.addCleanup(loop2.close) self.assertEqual(cm.filename, __file__) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2781,15 +2775,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - self.assertIs(asyncio.get_event_loop(), loop) - self.assertEqual(cm.filename, __file__) + self.assertIs(asyncio.get_event_loop(), loop) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index f8fe2e76b622..987772e0b196 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -145,10 +145,8 @@ def test_initial_state(self): self.assertTrue(f.cancelled()) def test_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - self._new_future() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() def test_constructor_use_running_loop(self): async def test(): @@ -158,12 +156,10 @@ async def test(): self.assertIs(f.get_loop(), self.loop) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - f = self._new_future() - self.assertEqual(cm.filename, __file__) + f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -499,10 +495,8 @@ def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) ex.shutdown(wait=True) def test_wrap_future_use_running_loop(self): @@ -517,16 +511,14 @@ async def test(): ex.shutdown(wait=True) def test_wrap_future_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - f2 = asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) ex.shutdown(wait=True) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 0c49099bc499..95fc7a159bae 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -810,10 +810,8 @@ def test_read_all_from_pipe_reader(self): self.assertEqual(data, b'data') def test_streamreader_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() def test_streamreader_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -827,21 +825,17 @@ async def test(): def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) - with self.assertWarns(DeprecationWarning) as cm: - reader = asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) def test_streamreaderprotocol_constructor_without_loop(self): reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) def test_streamreaderprotocol_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -855,13 +849,11 @@ async def test(): def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - protocol = asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) def test_multiple_drain(self): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 6b875cfb4dcb..9900e307a477 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -200,10 +200,8 @@ async def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -213,12 +211,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -1536,10 +1532,8 @@ async def coro(): self.addCleanup(a.close) futs = asyncio.as_completed([a]) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - list(futs) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + list(futs) def test_as_completed_coroutine_use_running_loop(self): loop = self.new_test_loop() @@ -1969,10 +1963,8 @@ async def coro(): inner = coro() self.addCleanup(inner.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.shield(inner) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) def test_shield_coroutine_use_running_loop(self): async def coro(): @@ -1986,15 +1978,13 @@ async def test(): self.assertEqual(res, 42) def test_shield_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 async def coro(): return 42 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - outer = asyncio.shield(coro()) - self.assertEqual(cm.filename, __file__) + outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) @@ -2820,7 +2810,7 @@ def test_current_task_no_running_loop(self): self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): asyncio.current_task() def test_current_task_with_implicit_loop(self): @@ -2984,10 +2974,8 @@ def _gather(self, *args, **kwargs): return asyncio.gather(*args, **kwargs) def test_constructor_empty_sequence_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() def test_constructor_empty_sequence_use_running_loop(self): async def gather(): @@ -3000,12 +2988,10 @@ async def gather(): self.assertEqual(fut.result(), []) def test_constructor_empty_sequence_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather() - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) @@ -3093,10 +3079,8 @@ async def coro(): self.addCleanup(gen1.close) gen2 = coro() self.addCleanup(gen2.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) def test_constructor_use_running_loop(self): async def coro(): @@ -3110,16 +3094,14 @@ async def gather(): self.one_loop.run_until_complete(fut) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 async def coro(): return 'abc' asyncio.set_event_loop(self.other_loop) self.addCleanup(asyncio.set_event_loop, None) gen1 = coro() gen2 = coro() - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather(gen1, gen2) self.assertIs(fut._loop, self.other_loop) self.other_loop.run_until_complete(fut) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 1d922783ce3b..01c1214c7f7c 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1740,7 +1740,8 @@ def f(): def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() - loop = policy.get_event_loop() + loop = policy.new_event_loop() + policy.set_event_loop(loop) # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index a15736e9f945..10f1a9efbcbd 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2411,7 +2411,8 @@ class UnawaitedWarningDuringShutdownTest(unittest.TestCase): def test_unawaited_warning_during_shutdown(self): code = ("import asyncio\n" "async def f(): pass\n" - "asyncio.gather(f())\n") + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") assert_python_ok("-c", code) code = ("import sys\n" diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst new file mode 100644 index 000000000000..dd4f43351ac7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst @@ -0,0 +1,3 @@ +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index d503018ea6c3..40f1f80be447 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -339,13 +339,6 @@ get_event_loop(int stacklevel) return loop; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "There is no current event loop", - stacklevel)) - { - return NULL; - } - policy = PyObject_CallNoArgs(asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; @@ -3129,6 +3122,11 @@ _asyncio_get_event_loop_impl(PyObject *module) return get_event_loop(1); } +// This internal method is going away in Python 3.12, left here only for +// backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. +// Similarly, this method's Python equivalent in asyncio.events is going +// away as well. +// See GH-99949 for more details. /*[clinic input] _asyncio._get_event_loop stacklevel: int = 3 From webhook-mailer at python.org Tue Dec 6 12:02:31 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 17:02:31 -0000 Subject: [Python-checkins] [3.11] GH-99729: Unlink frames before clearing them (#100047) Message-ID: <mailman.2825.1670346152.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2182a71eedbc8e95c3cf2d8c0aa2fd66c7a93db4 commit: 2182a71eedbc8e95c3cf2d8c0aa2fd66c7a93db4 branch: 3.11 author: Brandt Bucher <brandtbucher at microsoft.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T17:02:19Z summary: [3.11] GH-99729: Unlink frames before clearing them (#100047) files: A Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst M Lib/test/test_frame.py M Python/ceval.c M Python/frame.c diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 4b86a60d2f4c..9cb2686f1759 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -2,11 +2,13 @@ import re import sys import textwrap +import threading import types import unittest import weakref from test import support +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -325,6 +327,46 @@ def f(): if old_enabled: gc.enable() + @support.cpython_only + @threading_helper.requires_working_threading() + def test_sneaky_frame_object_teardown(self): + + class SneakyDel: + def __del__(self): + """ + Stash a reference to the entire stack for walking later. + + It may look crazy, but you'd be surprised how common this is + when using a test runner (like pytest). The typical recipe is: + ResourceWarning + -Werror + a custom sys.unraisablehook. + """ + nonlocal sneaky_frame_object + sneaky_frame_object = sys._getframe() + + class SneakyThread(threading.Thread): + """ + A separate thread isn't needed to make this code crash, but it does + make crashes more consistent, since it means sneaky_frame_object is + backed by freed memory after the thread completes! + """ + + def run(self): + """Run SneakyDel.__del__ as this frame is popped.""" + ref = SneakyDel() + + sneaky_frame_object = None + t = SneakyThread() + t.start() + t.join() + # sneaky_frame_object can be anything, really, but it's crucial that + # SneakyThread.run's frame isn't anywhere on the stack while it's being + # torn down: + self.assertIsNotNone(sneaky_frame_object) + while sneaky_frame_object is not None: + self.assertIsNot( + sneaky_frame_object.f_code, SneakyThread.run.__code__ + ) + sneaky_frame_object = sneaky_frame_object.f_back if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst new file mode 100644 index 000000000000..3fe21a8a21bf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst @@ -0,0 +1,3 @@ +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. diff --git a/Python/ceval.c b/Python/ceval.c index 8cbe838ddf2b..a34e4ffb72f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1617,14 +1617,6 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject return 0; } -static _PyInterpreterFrame * -pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - _PyInterpreterFrame *prev_frame = frame->previous; - _PyEvalFrameClearAndPop(tstate, frame); - return prev_frame; -} - /* It is only between the PRECALL instruction and the following CALL, * that this has any meaning. */ @@ -2441,7 +2433,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallTstate(tstate); if (!frame->is_entry) { - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; } @@ -5833,7 +5828,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(tstate->cframe->current_frame == frame->previous); return NULL; } - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); resume_with_error: SET_LOCALS_FROM_FRAME(); diff --git a/Python/frame.c b/Python/frame.c index d8f2f801f38c..3ea3a2ced40d 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -123,6 +123,9 @@ _PyFrame_Clear(_PyInterpreterFrame *frame) * to have cleared the enclosing generator, if any. */ assert(frame->owner != FRAME_OWNED_BY_GENERATOR || _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); + // GH-99729: Clearing this frame can expose the stack (via finalizers). It's + // crucial that this frame has been unlinked, and is no longer visible: + assert(_PyThreadState_GET()->cframe->current_frame != frame); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; From webhook-mailer at python.org Tue Dec 6 12:40:59 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 17:40:59 -0000 Subject: [Python-checkins] [3.10] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#100059) Message-ID: <mailman.2826.1670348461.3313.python-checkins@python.org> https://github.com/python/cpython/commit/300d812fd1c4d9244e71de0d228cc72439d312a7 commit: 300d812fd1c4d9244e71de0d228cc72439d312a7 branch: 3.10 author: ?ukasz Langa <lukasz at langa.pl> committer: ambv <lukasz at langa.pl> date: 2022-12-06T18:40:30+01:00 summary: [3.10] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#100059) It no longer emits a deprecation warning if the current event loop was set. (cherry picked from commit 3fae04b10e2655a20a3aadb5e0d63e87206d0c67) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst M Doc/library/asyncio-eventloop.rst M Doc/library/asyncio-llapi-index.rst M Doc/library/asyncio-policy.rst M Lib/asyncio/events.py M Lib/test/test_asyncio/test_base_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_futures.py M Lib/test/test_asyncio/test_streams.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_asyncio/test_unix_events.py M Lib/test/test_coroutines.py M Modules/_asynciomodule.c diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 86ca5d0f2ab5..a23be64ec39a 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -43,10 +43,12 @@ an event loop: Get the current event loop. - If there is no current event loop set in the current OS thread, - the OS thread is main, and :func:`set_event_loop` has not yet - been called, asyncio will create a new event loop and set it as the - current one. + When called from a coroutine or a callback (e.g. scheduled with + call_soon or similar API), this function will always return the + running event loop. + + If there is no running event loop set, the function will return + the result of ``get_event_loop_policy().get_event_loop()`` call. Because this function has rather complex behavior (especially when custom event loop policies are in use), using the @@ -58,10 +60,14 @@ an event loop: event loop. .. deprecated:: 3.10 - Emits a deprecation warning if there is no running event loop. - In future Python releases, this function may become an alias of - :func:`get_running_loop` and will accordingly raise a - :exc:`RuntimeError` if there is no running event loop. + Deprecation warning is emitted if there is no current event loop. + In Python 3.12 it will be an error. + + .. note:: + In Python versions 3.10.0--3.10.8 this function + (and other functions which used it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if + the current loop was set. .. function:: set_event_loop(loop) diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index 63ab1483e1fb..d903ea0dc247 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,7 +19,7 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (current or via the policy). + - Get an event loop instance (running or current via the current policy). * - :func:`asyncio.set_event_loop` - Set the event loop as current via the current policy. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index bfc3e3090fdc..a4cd5aa992af 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -112,6 +112,11 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. + .. deprecated:: 3.10.9 + :meth:`get_event_loop` now emits a :exc:`DeprecationWarning` if there + is no current event loop set and a new event loop has been implicitly + created. In Python 3.12 it will be an error. + .. class:: WindowsSelectorEventLoopPolicy diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 5ab1acc41bf3..faac53769412 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -650,6 +650,21 @@ def get_event_loop(self): if (self._local._loop is None and not self._local._set_called and threading.current_thread() is threading.main_thread()): + stacklevel = 2 + try: + f = sys._getframe(1) + except AttributeError: + pass + else: + while f: + module = f.f_globals.get('__name__') + if not (module == 'asyncio' or module.startswith('asyncio.')): + break + f = f.f_back + stacklevel += 1 + import warnings + warnings.warn('There is no current event loop', + DeprecationWarning, stacklevel=stacklevel) self.set_event_loop(self.new_event_loop()) if self._local._loop is None: @@ -763,12 +778,13 @@ def get_event_loop(): def _get_event_loop(stacklevel=3): + # This internal method is going away in Python 3.12, left here only for + # backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. + # Similarly, this method's C equivalent in _asyncio is going away as well. + # See GH-99949 for more details. current_loop = _get_running_loop() if current_loop is not None: return current_loop - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index d77bf95a7b37..7f7e371db1b4 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -752,7 +752,7 @@ async def coro(): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio', - 'loop = asyncio.get_event_loop()', + 'loop = asyncio.new_event_loop()', 'print(loop.get_debug())')) # Test with -E to not fail if the unit test was run with diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index bed7b5d30b85..92c69de742ba 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2561,8 +2561,9 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.assertEqual(cm.filename, __file__) self.assertIsInstance(loop, asyncio.AbstractEventLoop) self.assertIs(policy._local._loop, loop) @@ -2576,7 +2577,10 @@ def test_get_event_loop_calls_set_event_loop(self): policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.addCleanup(loop.close) + self.assertEqual(cm.filename, __file__) # policy._local._loop must be set through .set_event_loop() # (the unix DefaultEventLoopPolicy needs this call to attach @@ -2610,7 +2614,8 @@ def test_new_event_loop(self): def test_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() - old_loop = policy.get_event_loop() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) self.assertRaises(AssertionError, policy.set_event_loop, object()) @@ -2723,15 +2728,11 @@ def get_event_loop(self): asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2745,16 +2746,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) - + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) @@ -2778,10 +2774,8 @@ def test_get_event_loop_returns_running_loop2(self): self.addCleanup(loop2.close) self.assertEqual(cm.warnings[0].filename, __file__) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2795,15 +2789,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - self.assertIs(asyncio.get_event_loop(), loop) - self.assertEqual(cm.warnings[0].filename, __file__) + self.assertIs(asyncio.get_event_loop(), loop) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 012674260370..0b47aa28df66 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -145,10 +145,8 @@ def test_initial_state(self): self.assertTrue(f.cancelled()) def test_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - self._new_future() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() def test_constructor_use_running_loop(self): async def test(): @@ -158,12 +156,10 @@ async def test(): self.assertIs(f.get_loop(), self.loop) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - f = self._new_future() - self.assertEqual(cm.warnings[0].filename, __file__) + f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -499,10 +495,8 @@ def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.wrap_future(f1) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) ex.shutdown(wait=True) def test_wrap_future_use_running_loop(self): @@ -517,16 +511,14 @@ async def test(): ex.shutdown(wait=True) def test_wrap_future_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - f2 = asyncio.wrap_future(f1) - self.assertEqual(cm.warnings[0].filename, __file__) + f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) ex.shutdown(wait=True) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 44fcd6589c80..994041c10f0a 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -747,10 +747,8 @@ def test_read_all_from_pipe_reader(self): self.assertEqual(data, b'data') def test_streamreader_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReader() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() def test_streamreader_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -764,21 +762,17 @@ async def test(): def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) - with self.assertWarns(DeprecationWarning) as cm: - reader = asyncio.StreamReader() - self.assertEqual(cm.warnings[0].filename, __file__) + reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) def test_streamreaderprotocol_constructor_without_loop(self): reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) def test_streamreaderprotocol_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -792,13 +786,11 @@ async def test(): def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - protocol = asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.warnings[0].filename, __file__) + protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) def test_multiple_drain(self): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 05a822ba458b..b08068e6173e 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -210,10 +210,8 @@ async def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -223,12 +221,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.warnings[0].filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -247,10 +243,8 @@ def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -260,12 +254,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.warnings[0].filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -1488,10 +1480,8 @@ async def coro(): self.addCleanup(a.close) futs = asyncio.as_completed([a]) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - list(futs) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + list(futs) def test_as_completed_coroutine_use_running_loop(self): loop = self.new_test_loop() @@ -1507,17 +1497,14 @@ async def test(): loop.run_until_complete(test()) def test_as_completed_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 async def coro(): return 42 loop = self.new_test_loop() asyncio.set_event_loop(loop) self.addCleanup(asyncio.set_event_loop, None) - futs = asyncio.as_completed([coro()]) - with self.assertWarns(DeprecationWarning) as cm: - futs = list(futs) - self.assertEqual(cm.warnings[0].filename, __file__) + futs = list(asyncio.as_completed([coro()])) self.assertEqual(len(futs), 1) self.assertEqual(loop.run_until_complete(futs[0]), 42) @@ -1987,10 +1974,8 @@ async def coro(): inner = coro() self.addCleanup(inner.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.shield(inner) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) def test_shield_coroutine_use_running_loop(self): async def coro(): @@ -2004,15 +1989,13 @@ async def test(): self.assertEqual(res, 42) def test_shield_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 async def coro(): return 42 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - outer = asyncio.shield(coro()) - self.assertEqual(cm.warnings[0].filename, __file__) + outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) @@ -2950,7 +2933,7 @@ def test_current_task_no_running_loop(self): self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): asyncio.current_task() def test_current_task_with_implicit_loop(self): @@ -3114,10 +3097,8 @@ def _gather(self, *args, **kwargs): return asyncio.gather(*args, **kwargs) def test_constructor_empty_sequence_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() def test_constructor_empty_sequence_use_running_loop(self): async def gather(): @@ -3130,12 +3111,10 @@ async def gather(): self.assertEqual(fut.result(), []) def test_constructor_empty_sequence_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather() - self.assertEqual(cm.warnings[0].filename, __file__) + fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) @@ -3223,10 +3202,8 @@ async def coro(): self.addCleanup(gen1.close) gen2 = coro() self.addCleanup(gen2.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather(gen1, gen2) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) def test_constructor_use_running_loop(self): async def coro(): @@ -3240,16 +3217,14 @@ async def gather(): self.one_loop.run_until_complete(fut) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 async def coro(): return 'abc' asyncio.set_event_loop(self.other_loop) self.addCleanup(asyncio.set_event_loop, None) gen1 = coro() gen2 = coro() - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather(gen1, gen2) - self.assertEqual(cm.warnings[0].filename, __file__) + fut = asyncio.gather(gen1, gen2) self.assertIs(fut._loop, self.other_loop) self.other_loop.run_until_complete(fut) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 1d922783ce3b..01c1214c7f7c 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1740,7 +1740,8 @@ def f(): def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() - loop = policy.get_event_loop() + loop = policy.new_event_loop() + policy.set_event_loop(loop) # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index acff24537cf8..f4c526890718 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2319,7 +2319,8 @@ class UnawaitedWarningDuringShutdownTest(unittest.TestCase): def test_unawaited_warning_during_shutdown(self): code = ("import asyncio\n" "async def f(): pass\n" - "asyncio.gather(f())\n") + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") assert_python_ok("-c", code) code = ("import sys\n" diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst new file mode 100644 index 000000000000..dd4f43351ac7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst @@ -0,0 +1,3 @@ +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c627382a53f0..8d0ff690b538 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -332,13 +332,6 @@ get_event_loop(int stacklevel) return loop; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "There is no current event loop", - stacklevel)) - { - return NULL; - } - policy = PyObject_CallNoArgs(asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; @@ -3092,6 +3085,11 @@ _asyncio_get_event_loop_impl(PyObject *module) return get_event_loop(1); } +// This internal method is going away in Python 3.12, left here only for +// backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. +// Similarly, this method's Python equivalent in asyncio.events is going +// away as well. +// See GH-99949 for more details. /*[clinic input] _asyncio._get_event_loop stacklevel: int = 3 From webhook-mailer at python.org Tue Dec 6 12:42:20 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 17:42:20 -0000 Subject: [Python-checkins] gh-93453: No longer create an event loop in get_event_loop() (#98440) Message-ID: <mailman.2827.1670348540.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fd38a2f0ec03b4eec5e3cfd41241d198b1ee555a commit: fd38a2f0ec03b4eec5e3cfd41241d198b1ee555a branch: main author: Serhiy Storchaka <storchaka at gmail.com> committer: ambv <lukasz at langa.pl> date: 2022-12-06T18:42:12+01:00 summary: gh-93453: No longer create an event loop in get_event_loop() (#98440) asyncio.get_event_loop() now always return either running event loop or the result of get_event_loop_policy().get_event_loop() call. The latter should now raise an RuntimeError if no current event loop was set instead of creating and setting a new event loop. It affects also a number of asyncio functions and constructors which call get_event_loop() implicitly: ensure_future(), shield(), gather(), etc. DeprecationWarning is no longer emitted if there is no running event loop but the current event loop was set. Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst M Doc/library/asyncio-eventloop.rst M Doc/library/asyncio-llapi-index.rst M Doc/library/asyncio-policy.rst M Doc/whatsnew/3.12.rst M Lib/asyncio/events.py M Lib/asyncio/futures.py M Lib/asyncio/streams.py M Lib/asyncio/tasks.py M Lib/test/test_asyncio/test_base_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_futures.py M Lib/test/test_asyncio/test_streams.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_asyncio/test_unix_events.py M Lib/test/test_coroutines.py M Modules/_asynciomodule.c M Modules/clinic/_asynciomodule.c.h diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 0bcaed5477fa..fd47b0c24d8a 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -43,10 +43,12 @@ an event loop: Get the current event loop. - If there is no current event loop set in the current OS thread, - the OS thread is main, and :func:`set_event_loop` has not yet - been called, asyncio will create a new event loop and set it as the - current one. + When called from a coroutine or a callback (e.g. scheduled with + call_soon or similar API), this function will always return the + running event loop. + + If there is no running event loop set, the function will return + the result of calling ``get_event_loop_policy().get_event_loop()``. Because this function has rather complex behavior (especially when custom event loop policies are in use), using the @@ -57,11 +59,11 @@ an event loop: instead of using these lower level functions to manually create and close an event loop. - .. deprecated:: 3.10 - Emits a deprecation warning if there is no running event loop. - In future Python releases, this function may become an alias of - :func:`get_running_loop` and will accordingly raise a - :exc:`RuntimeError` if there is no running event loop. + .. note:: + In Python versions 3.10.0--3.10.8 and 3.11.0 this function + (and other functions which used it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if + the current loop was set. .. function:: set_event_loop(loop) diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index b7ad888a7b67..9ce48a24444e 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,7 +19,7 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (current or via the policy). + - Get an event loop instance (running or current via the current policy). * - :func:`asyncio.set_event_loop` - Set the event loop as current via the current policy. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 98c850158746..ccd952449475 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -116,6 +116,10 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. + .. versionchanged:: 3.12 + :meth:`get_event_loop` now raises a :exc:`RuntimeError` if there is no + current event loop set. + .. class:: WindowsSelectorEventLoopPolicy diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 6f5ce818d961..e9e25b9dc466 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -686,6 +686,18 @@ Changes in the Python API around process-global resources, which are best managed from the main interpreter. (Contributed by Dong-hee Na in :gh:`99127`.) +* :func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like + :func:`~asyncio.ensure_future`, :func:`~asyncio.shield` or + :func:`~asyncio.gather`, and also the + :meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of + :class:`~asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` + if called when there is no running event loop and the current event loop was + not set. + Previously they implicitly created and set a new current event loop. + :exc:`DeprecationWarning` is no longer emitted if there is no running + event loop but the current event loop is set in the policy. + (Contributed by Serhiy Storchaka in :gh:`93453`.) + Build Changes ============= diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 2836bbcc463f..34a8869dff8d 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -619,7 +619,7 @@ def get_event_loop(self): Returns an event loop object implementing the BaseEventLoop interface, or raises an exception in case no event loop has been set for the - current context and the current policy does not specify to create one. + current context. It should never return None.""" raise NotImplementedError @@ -672,11 +672,6 @@ def get_event_loop(self): Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) @@ -786,16 +781,9 @@ def get_event_loop(): the result of `get_event_loop_policy().get_event_loop()` call. """ # NOTE: this function is implemented in C (see _asynciomodule.c) - return _py__get_event_loop() - - -def _get_event_loop(stacklevel=3): current_loop = _get_running_loop() if current_loop is not None: return current_loop - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() @@ -825,7 +813,6 @@ def set_child_watcher(watcher): _py__set_running_loop = _set_running_loop _py_get_running_loop = get_running_loop _py_get_event_loop = get_event_loop -_py__get_event_loop = _get_event_loop try: @@ -833,7 +820,7 @@ def set_child_watcher(watcher): # functions in asyncio. Pure Python implementation is # about 4 times slower than C-accelerated. from _asyncio import (_get_running_loop, _set_running_loop, - get_running_loop, get_event_loop, _get_event_loop) + get_running_loop, get_event_loop) except ImportError: pass else: @@ -842,7 +829,6 @@ def set_child_watcher(watcher): _c__set_running_loop = _set_running_loop _c_get_running_loop = get_running_loop _c_get_event_loop = get_event_loop - _c__get_event_loop = _get_event_loop if hasattr(os, 'fork'): diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 3a6b44a09108..97fc4e3fcb60 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -77,7 +77,7 @@ def __init__(self, *, loop=None): the default event loop. """ if loop is None: - self._loop = events._get_event_loop() + self._loop = events.get_event_loop() else: self._loop = loop self._callbacks = [] @@ -413,7 +413,7 @@ def wrap_future(future, *, loop=None): assert isinstance(future, concurrent.futures.Future), \ f'concurrent.futures.Future is expected, got {future!r}' if loop is None: - loop = events._get_event_loop() + loop = events.get_event_loop() new_future = loop.create_future() _chain_future(future, new_future) return new_future diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index c4d837a11708..3bd99043d096 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -125,7 +125,7 @@ class FlowControlMixin(protocols.Protocol): def __init__(self, loop=None): if loop is None: - self._loop = events._get_event_loop(stacklevel=4) + self._loop = events.get_event_loop() else: self._loop = loop self._paused = False @@ -404,7 +404,7 @@ def __init__(self, limit=_DEFAULT_LIMIT, loop=None): self._limit = limit if loop is None: - self._loop = events._get_event_loop() + self._loop = events.get_event_loop() else: self._loop = loop self._buffer = bytearray() diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 571013745aa0..fa853283c0c5 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -582,7 +582,7 @@ def as_completed(fs, *, timeout=None): from .queues import Queue # Import here to avoid circular import problem. done = Queue() - loop = events._get_event_loop() + loop = events.get_event_loop() todo = {ensure_future(f, loop=loop) for f in set(fs)} timeout_handle = None @@ -668,7 +668,7 @@ def _ensure_future(coro_or_future, *, loop=None): 'is required') if loop is None: - loop = events._get_event_loop(stacklevel=4) + loop = events.get_event_loop() try: return loop.create_task(coro_or_future) except RuntimeError: @@ -749,7 +749,7 @@ def gather(*coros_or_futures, return_exceptions=False): gather won't cancel any other awaitables. """ if not coros_or_futures: - loop = events._get_event_loop() + loop = events.get_event_loop() outer = loop.create_future() outer.set_result([]) return outer diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 7421d18dc636..65dd4d42708b 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -746,7 +746,7 @@ async def coro(): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio', - 'loop = asyncio.get_event_loop()', + 'loop = asyncio.new_event_loop()', 'print(loop.get_debug())')) # Test with -E to not fail if the unit test was run with diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index cabe75f56d9f..153b2de81722 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2550,29 +2550,8 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - - loop = policy.get_event_loop() - self.assertIsInstance(loop, asyncio.AbstractEventLoop) - - self.assertIs(policy._local._loop, loop) - self.assertIs(loop, policy.get_event_loop()) - loop.close() - - def test_get_event_loop_calls_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() - - with mock.patch.object( - policy, "set_event_loop", - wraps=policy.set_event_loop) as m_set_event_loop: - - loop = policy.get_event_loop() - - # policy._local._loop must be set through .set_event_loop() - # (the unix DefaultEventLoopPolicy needs this call to attach - # the child watcher correctly) - m_set_event_loop.assert_called_with(loop) - - loop.close() + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + policy.get_event_loop() def test_get_event_loop_after_set_none(self): policy = asyncio.DefaultEventLoopPolicy() @@ -2599,7 +2578,8 @@ def test_new_event_loop(self): def test_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() - old_loop = policy.get_event_loop() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) self.assertRaises(TypeError, policy.set_event_loop, object()) @@ -2716,15 +2696,11 @@ def get_event_loop(self): asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2738,16 +2714,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) - + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) @@ -2766,15 +2737,11 @@ def test_get_event_loop_returns_running_loop2(self): loop = asyncio.new_event_loop() self.addCleanup(loop.close) - with self.assertWarns(DeprecationWarning) as cm: - loop2 = asyncio.get_event_loop() - self.addCleanup(loop2.close) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2788,15 +2755,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - self.assertIs(asyncio.get_event_loop(), loop) - self.assertEqual(cm.filename, __file__) + self.assertIs(asyncio.get_event_loop(), loop) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 83ea01c24525..56b0b864de2d 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -146,10 +146,8 @@ def test_initial_state(self): self.assertTrue(f.cancelled()) def test_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - self._new_future() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() def test_constructor_use_running_loop(self): async def test(): @@ -159,12 +157,10 @@ async def test(): self.assertIs(f.get_loop(), self.loop) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - f = self._new_future() - self.assertEqual(cm.filename, __file__) + f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -500,10 +496,8 @@ def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) ex.shutdown(wait=True) def test_wrap_future_use_running_loop(self): @@ -518,16 +512,14 @@ async def test(): ex.shutdown(wait=True) def test_wrap_future_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - f2 = asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) ex.shutdown(wait=True) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 01d5407a497a..7f9dc6218083 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -816,10 +816,8 @@ def test_read_all_from_pipe_reader(self): self.assertEqual(data, b'data') def test_streamreader_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() def test_streamreader_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -833,21 +831,17 @@ async def test(): def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) - with self.assertWarns(DeprecationWarning) as cm: - reader = asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) def test_streamreaderprotocol_constructor_without_loop(self): reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) def test_streamreaderprotocol_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -861,13 +855,11 @@ async def test(): def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - protocol = asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) def test_multiple_drain(self): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index d8ba2f4e2a74..bb1ffdf2df74 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -196,10 +196,8 @@ async def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -209,12 +207,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -1532,10 +1528,8 @@ async def coro(): self.addCleanup(a.close) futs = asyncio.as_completed([a]) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - list(futs) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + list(futs) def test_as_completed_coroutine_use_running_loop(self): loop = self.new_test_loop() @@ -1965,10 +1959,8 @@ async def coro(): inner = coro() self.addCleanup(inner.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.shield(inner) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) def test_shield_coroutine_use_running_loop(self): async def coro(): @@ -1982,15 +1974,13 @@ async def test(): self.assertEqual(res, 42) def test_shield_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 async def coro(): return 42 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - outer = asyncio.shield(coro()) - self.assertEqual(cm.filename, __file__) + outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) @@ -2827,7 +2817,7 @@ def test_current_task_no_running_loop(self): self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): asyncio.current_task() def test_current_task_with_implicit_loop(self): @@ -2991,10 +2981,8 @@ def _gather(self, *args, **kwargs): return asyncio.gather(*args, **kwargs) def test_constructor_empty_sequence_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() def test_constructor_empty_sequence_use_running_loop(self): async def gather(): @@ -3007,12 +2995,10 @@ async def gather(): self.assertEqual(fut.result(), []) def test_constructor_empty_sequence_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather() - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) @@ -3100,10 +3086,8 @@ async def coro(): self.addCleanup(gen1.close) gen2 = coro() self.addCleanup(gen2.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) def test_constructor_use_running_loop(self): async def coro(): @@ -3117,16 +3101,14 @@ async def gather(): self.one_loop.run_until_complete(fut) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 async def coro(): return 'abc' asyncio.set_event_loop(self.other_loop) self.addCleanup(asyncio.set_event_loop, None) gen1 = coro() gen2 = coro() - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather(gen1, gen2) self.assertIs(fut._loop, self.other_loop) self.other_loop.run_until_complete(fut) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 309a1cfdb4aa..600a5900da08 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1775,7 +1775,8 @@ def f(): def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() - loop = policy.get_event_loop() + loop = policy.new_event_loop() + policy.set_event_loop(loop) # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property @@ -1884,13 +1885,15 @@ async def test_fork_not_share_event_loop(self): # child try: loop = asyncio.get_event_loop_policy().get_event_loop() - os.write(w, str(id(loop)).encode()) + except RuntimeError: + os.write(w, b'NO LOOP') + except: + os.write(w, b'ERROR:' + ascii(sys.exc_info()).encode()) finally: os._exit(0) else: # parent - child_loop = int(os.read(r, 100).decode()) - self.assertNotEqual(child_loop, id(loop)) + self.assertEqual(os.read(r, 100), b'NO LOOP') wait_process(pid, exitcode=0) @hashlib_helper.requires_hashdigest('md5') diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index f91c9cc47741..43a3ff0536fe 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2418,7 +2418,8 @@ class UnawaitedWarningDuringShutdownTest(unittest.TestCase): def test_unawaited_warning_during_shutdown(self): code = ("import asyncio\n" "async def f(): pass\n" - "asyncio.gather(f())\n") + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") assert_python_ok("-c", code) code = ("import sys\n" diff --git a/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst b/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst new file mode 100644 index 000000000000..5aee4b4f0aa7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst @@ -0,0 +1,9 @@ +:func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like +:func:`asyncio.ensure_future`, :func:`asyncio.shield` or +:func:`asyncio.gather`, and also the +:meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of +:class:`asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` +if called when there is no running event loop and the current event loop was +not set. Previously they implicitly created and set a new current event +loop. :exc:`DeprecationWarning` is no longer emitted if there is no running +event loop but the current event loop was set. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index cabcaec94bec..60369d89dc39 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -357,7 +357,7 @@ set_running_loop(asyncio_state *state, PyObject *loop) static PyObject * -get_event_loop(asyncio_state *state, int stacklevel) +get_event_loop(asyncio_state *state) { PyObject *loop; PyObject *policy; @@ -369,13 +369,6 @@ get_event_loop(asyncio_state *state, int stacklevel) return loop; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "There is no current event loop", - stacklevel)) - { - return NULL; - } - policy = PyObject_CallNoArgs(state->asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; @@ -538,7 +531,7 @@ future_init(FutureObj *fut, PyObject *loop) if (loop == Py_None) { asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - loop = get_event_loop(state, 1); + loop = get_event_loop(state); if (loop == NULL) { return -1; } @@ -3229,20 +3222,7 @@ _asyncio_get_event_loop_impl(PyObject *module) /*[clinic end generated code: output=2a2d8b2f824c648b input=9364bf2916c8655d]*/ { asyncio_state *state = get_asyncio_state(module); - return get_event_loop(state, 1); -} - -/*[clinic input] -_asyncio._get_event_loop - stacklevel: int = 3 -[clinic start generated code]*/ - -static PyObject * -_asyncio__get_event_loop_impl(PyObject *module, int stacklevel) -/*[clinic end generated code: output=9c1d6d3c802e67c9 input=d17aebbd686f711d]*/ -{ - asyncio_state *state = get_asyncio_state(module); - return get_event_loop(state, stacklevel-1); + return get_event_loop(state); } /*[clinic input] @@ -3620,7 +3600,6 @@ PyDoc_STRVAR(module_doc, "Accelerator module for asyncio"); static PyMethodDef asyncio_methods[] = { _ASYNCIO_GET_EVENT_LOOP_METHODDEF - _ASYNCIO__GET_EVENT_LOOP_METHODDEF _ASYNCIO_GET_RUNNING_LOOP_METHODDEF _ASYNCIO__GET_RUNNING_LOOP_METHODDEF _ASYNCIO__SET_RUNNING_LOOP_METHODDEF diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 11db478a8b48..f2fbb352c2c6 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -987,68 +987,6 @@ _asyncio_get_event_loop(PyObject *module, PyObject *Py_UNUSED(ignored)) return _asyncio_get_event_loop_impl(module); } -PyDoc_STRVAR(_asyncio__get_event_loop__doc__, -"_get_event_loop($module, /, stacklevel=3)\n" -"--\n" -"\n"); - -#define _ASYNCIO__GET_EVENT_LOOP_METHODDEF \ - {"_get_event_loop", _PyCFunction_CAST(_asyncio__get_event_loop), METH_FASTCALL|METH_KEYWORDS, _asyncio__get_event_loop__doc__}, - -static PyObject * -_asyncio__get_event_loop_impl(PyObject *module, int stacklevel); - -static PyObject * -_asyncio__get_event_loop(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(stacklevel), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"stacklevel", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_get_event_loop", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int stacklevel = 3; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - stacklevel = _PyLong_AsInt(args[0]); - if (stacklevel == -1 && PyErr_Occurred()) { - goto exit; - } -skip_optional_pos: - return_value = _asyncio__get_event_loop_impl(module, stacklevel); - -exit: - return return_value; -} - PyDoc_STRVAR(_asyncio_get_running_loop__doc__, "get_running_loop($module, /)\n" "--\n" @@ -1304,4 +1242,4 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=550bc6603df89ed9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=83580c190031241c input=a9049054013a1b77]*/ From webhook-mailer at python.org Tue Dec 6 13:51:27 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 18:51:27 -0000 Subject: [Python-checkins] Python 3.9.16 Message-ID: <mailman.2828.1670352688.3313.python-checkins@python.org> https://github.com/python/cpython/commit/595f9ccb0c059f2fb5bf13643bfc0cdd5b55a422 commit: 595f9ccb0c059f2fb5bf13643bfc0cdd5b55a422 branch: 3.9 author: ?ukasz Langa <lukasz at langa.pl> committer: ambv <lukasz at langa.pl> date: 2022-12-06T18:59:46+01:00 summary: Python 3.9.16 files: A Misc/NEWS.d/3.9.16.rst D Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst D Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Include/patchlevel.h M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 8af690f9eded..e33d9a1d0170 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 9 -#define PY_MICRO_VERSION 15 +#define PY_MICRO_VERSION 16 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.9.15+" +#define PY_VERSION "3.9.16" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS.d/3.9.16.rst b/Misc/NEWS.d/3.9.16.rst new file mode 100644 index 000000000000..408cd9ca300c --- /dev/null +++ b/Misc/NEWS.d/3.9.16.rst @@ -0,0 +1,90 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-09-07-10-42-00 +.. gh-issue: 97514 +.. nonce: Yggdsl +.. section: Security + +On Linux the :mod:`multiprocessing` module returns to using filesystem +backed unix domain sockets for communication with the *forkserver* process +instead of the Linux abstract socket namespace. Only code that chooses to +use the :ref:`"forkserver" start method <multiprocessing-start-methods>` is +affected. + +Abstract sockets have no permissions and could allow any user on the system +in the same `network namespace +<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often +the whole system) to inject code into the multiprocessing *forkserver* +process. This was a potential privilege escalation. Filesystem based socket +permissions restrict this to the *forkserver* process user as was the +default in Python 3.8 and earlier. + +This prevents Linux `CVE-2022-42919 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. + +.. + +.. date: 2022-04-27-18-25-30 +.. gh-issue: 68966 +.. nonce: gjS8zs +.. section: Security + +The deprecated mailcap module now refuses to inject unsafe text (filenames, +MIME types, parameters) into shell commands. Instead of using such text, it +will warn and act as if a match was not found (or for test commands, as if +the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst deleted file mode 100644 index da81a1f6993d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst +++ /dev/null @@ -1,4 +0,0 @@ -The deprecated mailcap module now refuses to inject unsafe text (filenames, -MIME types, parameters) into shell commands. Instead of using such text, it -will warn and act as if a match was not found (or for test commands, as if -the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst deleted file mode 100644 index 02d95b570520..000000000000 --- a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst +++ /dev/null @@ -1,15 +0,0 @@ -On Linux the :mod:`multiprocessing` module returns to using filesystem backed -unix domain sockets for communication with the *forkserver* process instead of -the Linux abstract socket namespace. Only code that chooses to use the -:ref:`"forkserver" start method <multiprocessing-start-methods>` is affected. - -Abstract sockets have no permissions and could allow any user on the system in -the same `network namespace -<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often the -whole system) to inject code into the multiprocessing *forkserver* process. -This was a potential privilege escalation. Filesystem based socket permissions -restrict this to the *forkserver* process user as was the default in Python 3.8 -and earlier. - -This prevents Linux `CVE-2022-42919 -<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/README.rst b/README.rst index d78b6bc9e86f..4c804d677cf9 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.9.15 +This is Python version 3.9.16 ============================= .. image:: https://travis-ci.org/python/cpython.svg?branch=3.9 From webhook-mailer at python.org Tue Dec 6 14:34:15 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 19:34:15 -0000 Subject: [Python-checkins] Python 3.8.16 Message-ID: <mailman.2829.1670355256.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1e3d2d52109c9d82ba307116e912d16bb4b0dbb7 commit: 1e3d2d52109c9d82ba307116e912d16bb4b0dbb7 branch: 3.8 author: ?ukasz Langa <lukasz at langa.pl> committer: ambv <lukasz at langa.pl> date: 2022-12-06T19:59:58+01:00 summary: Python 3.8.16 files: A Misc/NEWS.d/3.8.16.rst D Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Include/patchlevel.h M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 04b2eb8ad3fc..8b5b8d3a85d6 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 8 -#define PY_MICRO_VERSION 15 +#define PY_MICRO_VERSION 16 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.8.15+" +#define PY_VERSION "3.8.16" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS.d/3.8.16.rst b/Misc/NEWS.d/3.8.16.rst new file mode 100644 index 000000000000..c320159b73b5 --- /dev/null +++ b/Misc/NEWS.d/3.8.16.rst @@ -0,0 +1,66 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-04-27-18-25-30 +.. gh-issue: 68966 +.. nonce: gjS8zs +.. section: Security + +The deprecated mailcap module now refuses to inject unsafe text (filenames, +MIME types, parameters) into shell commands. Instead of using such text, it +will warn and act as if a match was not found (or for test commands, as if +the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst deleted file mode 100644 index da81a1f6993d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst +++ /dev/null @@ -1,4 +0,0 @@ -The deprecated mailcap module now refuses to inject unsafe text (filenames, -MIME types, parameters) into shell commands. Instead of using such text, it -will warn and act as if a match was not found (or for test commands, as if -the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/README.rst b/README.rst index f244a926fa79..3aed3a5c47d4 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.8.15 +This is Python version 3.8.16 ============================= .. image:: https://travis-ci.org/python/cpython.svg?branch=3.8 From webhook-mailer at python.org Tue Dec 6 15:36:08 2022 From: webhook-mailer at python.org (ned-deily) Date: Tue, 06 Dec 2022 20:36:08 -0000 Subject: [Python-checkins] Python 3.7.16 Message-ID: <mailman.2830.1670358969.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3f82aa744678620a811927dc4e56ad9c7c3d0c14 commit: 3f82aa744678620a811927dc4e56ad9c7c3d0c14 branch: 3.7 author: Ned Deily <nad at python.org> committer: ned-deily <nad at python.org> date: 2022-12-06T14:00:00-05:00 summary: Python 3.7.16 files: A Misc/NEWS.d/3.7.16.rst D Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index ab3fc9b392f7..f943b207b1ed 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 15 +#define PY_MICRO_VERSION 16 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.7.15+" +#define PY_VERSION "3.7.16" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 95eb9d4b6744..5f09d234bdb2 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Oct 10 05:27:13 2022 +# Autogenerated by Sphinx on Tue Dec 6 13:59:35 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' diff --git a/Misc/NEWS.d/3.7.16.rst b/Misc/NEWS.d/3.7.16.rst new file mode 100644 index 000000000000..8991a1f9c45f --- /dev/null +++ b/Misc/NEWS.d/3.7.16.rst @@ -0,0 +1,56 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-04-27-18-25-30 +.. gh-issue: 68966 +.. nonce: gjS8zs +.. section: Security + +The deprecated mailcap module now refuses to inject unsafe text (filenames, +MIME types, parameters) into shell commands. Instead of using such text, it +will warn and act as if a match was not found (or for test commands, as if +the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst deleted file mode 100644 index da81a1f6993d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst +++ /dev/null @@ -1,4 +0,0 @@ -The deprecated mailcap module now refuses to inject unsafe text (filenames, -MIME types, parameters) into shell commands. Instead of using such text, it -will warn and act as if a match was not found (or for test commands, as if -the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/README.rst b/README.rst index 2c80b7b7383e..28fb2a398f14 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.7.15+ -============================== +This is Python version 3.7.16 +============================= .. image:: https://travis-ci.org/python/cpython.svg?branch=3.7 :alt: CPython build status on Travis CI From webhook-mailer at python.org Tue Dec 6 16:11:36 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 21:11:36 -0000 Subject: [Python-checkins] Python 3.11.1 Message-ID: <mailman.2831.1670361097.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a7a450f84a087421603170c2dad226bb881d4d9a commit: a7a450f84a087421603170c2dad226bb881d4d9a branch: 3.11 author: Pablo Galindo <pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T19:05:27Z summary: Python 3.11.1 files: A Misc/NEWS.d/3.11.1.rst D Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst D Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst D Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst D Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst D Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst D Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst D Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst D Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst D Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst D Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst D Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst D Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst D Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst D Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst D Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst D Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst D Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst D Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst D Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst D Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst D Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst D Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst D Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst D Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst D Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst D Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst D Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst D Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst D Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst D Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst D Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst D Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst D Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst D Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst D Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst D Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst D Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst D Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst D Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst D Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst D Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst D Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst D Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst D Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst D Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst D Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst D Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst D Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst D Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst D Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst D Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst D Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst D Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst D Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst D Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst D Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst D Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst D Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst D Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst D Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst D Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst D Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst D Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst D Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst D Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst D Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst D Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst D Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst D Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst D Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst D Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst D Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst D Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst D Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst D Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst D Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst D Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst D Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst D Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst D Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst D Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst D Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst D Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst D Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst D Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst D Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst D Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst D Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst D Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst D Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst D Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst D Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst D Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst D Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst D Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst D Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst D Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst D Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst D Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst D Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst D Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst D Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst D Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst D Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst D Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst D Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst D Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst D Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst D Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c37154baa22b..7f217c1db4fe 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 11 -#define PY_MICRO_VERSION 0 +#define PY_MICRO_VERSION 1 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.11.0+" +#define PY_VERSION "3.11.1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 7bba5cb88093..11a1c73bf70f 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Oct 24 18:35:07 2022 +# Autogenerated by Sphinx on Tue Dec 6 19:05:00 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -358,7 +358,7 @@ 'yield_expression)]\n' '\n' 'The difference from normal Assignment statements is that only ' - 'single\n' + 'a single\n' 'target is allowed.\n' '\n' 'For simple names as assignment targets, if in class or module ' @@ -408,12 +408,13 @@ 'analysis\n' ' tools and IDEs.\n' '\n' - 'Changed in version 3.8: Now annotated assignments allow same\n' - 'expressions in the right hand side as the regular ' - 'assignments.\n' - 'Previously, some expressions (like un-parenthesized tuple ' - 'expressions)\n' - 'caused a syntax error.\n', + 'Changed in version 3.8: Now annotated assignments allow the ' + 'same\n' + 'expressions in the right hand side as regular assignments. ' + 'Previously,\n' + 'some expressions (like un-parenthesized tuple expressions) ' + 'caused a\n' + 'syntax error.\n', 'async': 'Coroutines\n' '**********\n' '\n' @@ -2551,12 +2552,13 @@ 'that\n' 'multiple "except*" clauses can execute, each handling part of ' 'the\n' - 'exception group. Each clause executes once and handles an ' - 'exception\n' - 'group of all matching exceptions. Each exception in the group ' - 'is\n' - 'handled by at most one "except*" clause, the first that matches ' - 'it.\n' + 'exception group. Each clause executes at most once and handles ' + 'an\n' + 'exception group of all matching exceptions. Each exception in ' + 'the\n' + 'group is handled by at most one "except*" clause, the first ' + 'that\n' + 'matches it.\n' '\n' ' >>> try:\n' ' ... raise ExceptionGroup("eg",\n' @@ -2579,21 +2581,32 @@ ' | ValueError: 1\n' ' +------------------------------------\n' '\n' - ' Any remaining exceptions that were not handled by any ' - ':keyword:`!except*`\n' - ' clause are re-raised at the end, combined into an exception ' - 'group along with\n' - ' all exceptions that were raised from within ' - ':keyword:`!except*` clauses.\n' + 'Any remaining exceptions that were not handled by any "except*" ' + 'clause\n' + 'are re-raised at the end, combined into an exception group along ' + 'with\n' + 'all exceptions that were raised from within "except*" clauses.\n' + '\n' + 'If the raised exception is not an exception group and its type ' + 'matches\n' + 'one of the "except*" clauses, it is caught and wrapped by an ' + 'exception\n' + 'group with an empty message string.\n' + '\n' + ' >>> try:\n' + ' ... raise BlockingIOError\n' + ' ... except* BlockingIOError as e:\n' + ' ... print(repr(e))\n' + ' ...\n' + " ExceptionGroup('', (BlockingIOError()))\n" '\n' - ' An :keyword:`!except*` clause must have a matching type,\n' - ' and this type cannot be a subclass of ' - ':exc:`BaseExceptionGroup`.\n' - ' It is not possible to mix :keyword:`except` and ' - ':keyword:`!except*`\n' - ' in the same :keyword:`try`.\n' - ' :keyword:`break`, :keyword:`continue` and :keyword:`return`\n' - ' cannot appear in an :keyword:`!except*` clause.\n' + 'An "except*" clause must have a matching type, and this type ' + 'cannot be\n' + 'a subclass of "BaseExceptionGroup". It is not possible to mix ' + '"except"\n' + 'and "except*" in the same "try". "break", "continue" and ' + '"return"\n' + 'cannot appear in an "except*" clause.\n' '\n' '\n' '"else" clause\n' @@ -7308,7 +7321,7 @@ 'the clauses had been separated out into individual import ' 'statements.\n' '\n' - 'The details of the first step, finding and loading modules are\n' + 'The details of the first step, finding and loading modules, are\n' 'described in greater detail in the section on the import system, ' 'which\n' 'also describes the various types of packages and modules that can ' @@ -11096,8 +11109,9 @@ 'y)" is\n' 'typically invalid without special support in "MyClass". To ' 'be able to\n' - 'use that kind of patterns, the class needs to define a\n' - '*__match_args__* attribute.\n' + 'use that kind of pattern, the class needs to define a ' + '*__match_args__*\n' + 'attribute.\n' '\n' 'object.__match_args__\n' '\n' @@ -11302,6 +11316,10 @@ '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' + ' If *sub* is empty, returns the number of empty strings ' + 'between\n' + ' characters which is the length of the string plus one.\n' + '\n' "str.encode(encoding='utf-8', errors='strict')\n" '\n' ' Return an encoded version of the string as a bytes ' @@ -11808,7 +11826,7 @@ 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11849,7 +11867,7 @@ " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -12650,9 +12668,10 @@ 'the type matches some of the exceptions in the group. This means ' 'that\n' 'multiple "except*" clauses can execute, each handling part of the\n' - 'exception group. Each clause executes once and handles an exception\n' - 'group of all matching exceptions. Each exception in the group is\n' - 'handled by at most one "except*" clause, the first that matches it.\n' + 'exception group. Each clause executes at most once and handles an\n' + 'exception group of all matching exceptions. Each exception in the\n' + 'group is handled by at most one "except*" clause, the first that\n' + 'matches it.\n' '\n' ' >>> try:\n' ' ... raise ExceptionGroup("eg",\n' @@ -12673,20 +12692,31 @@ ' | ValueError: 1\n' ' +------------------------------------\n' '\n' - ' Any remaining exceptions that were not handled by any ' - ':keyword:`!except*`\n' - ' clause are re-raised at the end, combined into an exception group ' - 'along with\n' - ' all exceptions that were raised from within :keyword:`!except*` ' - 'clauses.\n' + 'Any remaining exceptions that were not handled by any "except*" ' + 'clause\n' + 'are re-raised at the end, combined into an exception group along ' + 'with\n' + 'all exceptions that were raised from within "except*" clauses.\n' '\n' - ' An :keyword:`!except*` clause must have a matching type,\n' - ' and this type cannot be a subclass of :exc:`BaseExceptionGroup`.\n' - ' It is not possible to mix :keyword:`except` and ' - ':keyword:`!except*`\n' - ' in the same :keyword:`try`.\n' - ' :keyword:`break`, :keyword:`continue` and :keyword:`return`\n' - ' cannot appear in an :keyword:`!except*` clause.\n' + 'If the raised exception is not an exception group and its type ' + 'matches\n' + 'one of the "except*" clauses, it is caught and wrapped by an ' + 'exception\n' + 'group with an empty message string.\n' + '\n' + ' >>> try:\n' + ' ... raise BlockingIOError\n' + ' ... except* BlockingIOError as e:\n' + ' ... print(repr(e))\n' + ' ...\n' + " ExceptionGroup('', (BlockingIOError()))\n" + '\n' + 'An "except*" clause must have a matching type, and this type cannot ' + 'be\n' + 'a subclass of "BaseExceptionGroup". It is not possible to mix ' + '"except"\n' + 'and "except*" in the same "try". "break", "continue" and "return"\n' + 'cannot appear in an "except*" clause.\n' '\n' '\n' '"else" clause\n' @@ -13947,17 +13977,11 @@ 'dictionaries or\n' 'other mutable types (that are compared by value rather than ' 'by object\n' - 'identity) may not be used as keys. Numeric types used for ' - 'keys obey\n' - 'the normal rules for numeric comparison: if two numbers ' - 'compare equal\n' - '(such as "1" and "1.0") then they can be used ' - 'interchangeably to index\n' - 'the same dictionary entry. (Note however, that since ' - 'computers store\n' - 'floating-point numbers as approximations it is usually ' - 'unwise to use\n' - 'them as dictionary keys.)\n' + 'identity) may not be used as keys. Values that compare equal ' + '(such as\n' + '"1", "1.0", and "True") can be used interchangeably to index ' + 'the same\n' + 'dictionary entry.\n' '\n' 'class dict(**kwargs)\n' 'class dict(mapping, **kwargs)\n' diff --git a/Misc/NEWS.d/3.11.1.rst b/Misc/NEWS.d/3.11.1.rst new file mode 100644 index 000000000000..62cf53afe507 --- /dev/null +++ b/Misc/NEWS.d/3.11.1.rst @@ -0,0 +1,1545 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-09-28-12-10-57 +.. gh-issue: 97612 +.. nonce: y6NvOQ +.. section: Security + +Fix a shell code injection vulnerability in the +``get-remote-certificate.py`` example script. The script no longer uses a +shell to run ``openssl`` commands. Issue reported and initial fix by Caleb +Shortt. Patch by Victor Stinner. + +.. + +.. date: 2022-11-30-15-29-08 +.. gh-issue: 99886 +.. nonce: feJkSv +.. section: Core and Builtins + +Fix a crash when an object which does not have a dictionary frees its +instance values. + +.. + +.. date: 2022-11-30-11-09-40 +.. gh-issue: 99891 +.. nonce: 9VomwB +.. section: Core and Builtins + +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo + +.. + +.. date: 2022-11-26-05-34-00 +.. gh-issue: 99729 +.. nonce: A3ovwQ +.. section: Core and Builtins + +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. + +.. + +.. date: 2022-11-21-11-27-14 +.. gh-issue: 99578 +.. nonce: DcKoBJ +.. section: Core and Builtins + +Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by +Victor Stinner. + +.. + +.. date: 2022-11-19-22-27-52 +.. gh-issue: 99581 +.. nonce: yKYPbf +.. section: Core and Builtins + +Fixed a bug that was causing a buffer overflow if the tokenizer copies a +line missing the newline caracter from a file that is as long as the +available tokenizer buffer. Patch by Pablo galindo + +.. + +.. date: 2022-11-18-11-24-25 +.. gh-issue: 99553 +.. nonce: F64h-n +.. section: Core and Builtins + +Fix bug where an :exc:`ExceptionGroup` subclass can wrap a +:exc:`BaseException`. + +.. + +.. date: 2022-11-12-01-39-57 +.. gh-issue: 99370 +.. nonce: _cu32j +.. section: Core and Builtins + +Fix zip path for venv created from a non-installed python on POSIX +platforms. + +.. + +.. date: 2022-11-10-02-11-23 +.. gh-issue: 99298 +.. nonce: NeArAJ +.. section: Core and Builtins + +Fix an issue that could potentially cause incorrect error handling for some +bytecode instructions. + +.. + +.. date: 2022-11-08-16-35-25 +.. gh-issue: 99205 +.. nonce: 2YOoFT +.. section: Core and Builtins + +Fix an issue that prevented :c:type:`PyThreadState` and +:c:type:`PyInterpreterState` memory from being freed properly. + +.. + +.. date: 2022-11-07-10-29-41 +.. gh-issue: 99181 +.. nonce: bfG4bI +.. section: Core and Builtins + +Fix failure in :keyword:`except* <except_star>` with unhashable exceptions. + +.. + +.. date: 2022-11-07-08-17-12 +.. gh-issue: 99204 +.. nonce: Mf4hMD +.. section: Core and Builtins + +Fix calculation of :data:`sys._base_executable` when inside a POSIX virtual +environment using copies of the python binary when the base installation +does not provide the executable name used by the venv. Calculation will fall +back to alternative names ("python<MAJOR>", "python<MAJOR>.<MINOR>"). + +.. + +.. date: 2022-11-06-22-59-02 +.. gh-issue: 96055 +.. nonce: TmQuJn +.. section: Core and Builtins + +Update :mod:`faulthandler` to emit an error message with the proper +unexpected signal number. Patch by Dong-hee Na. + +.. + +.. date: 2022-11-06-13-25-01 +.. gh-issue: 99153 +.. nonce: uE3CVL +.. section: Core and Builtins + +Fix location of :exc:`SyntaxError` for a :keyword:`try` block with both +:keyword:`except` and :keyword:`except* <except_star>`. + +.. + +.. date: 2022-11-06-00-17-58 +.. gh-issue: 99103 +.. nonce: bFA9BX +.. section: Core and Builtins + +Fix the error reporting positions of specialized traceback anchors when the +source line contains Unicode characters. + +.. + +.. date: 2022-10-31-21-01-35 +.. gh-issue: 98852 +.. nonce: MYaRN6 +.. section: Core and Builtins + +Fix subscription of type aliases containing bare generic types or types like +:class:`~typing.TypeVar`: for example ``tuple[A, T][int]`` and +``tuple[TypeVar, T][int]``, where ``A`` is a generic type, and ``T`` is a +type variable. + +.. + +.. date: 2022-10-31-18-03-10 +.. gh-issue: 98925 +.. nonce: zpdjVd +.. section: Core and Builtins + +Lower the recursion depth for marshal on WASI to support wasmtime 2.0/main. + +.. + +.. date: 2022-10-28-14-52-55 +.. gh-issue: 98783 +.. nonce: iG0kMs +.. section: Core and Builtins + +Fix multiple crashes in debug mode when ``str`` subclasses are used instead +of ``str`` itself. + +.. + +.. date: 2022-10-21-11-28-53 +.. gh-issue: 99257 +.. nonce: nmcuf- +.. section: Core and Builtins + +Fix an issue where member descriptors (such as those for +:attr:`~object.__slots__`) could behave incorrectly or crash instead of +raising a :exc:`TypeError` when accessed via an instance of an invalid type. + +.. + +.. date: 2022-10-19-23-48-46 +.. gh-issue: 98374 +.. nonce: eOBh8M +.. section: Core and Builtins + +Suppress ImportError for invalid query for help() command. Patch by Dong-hee +Na. + +.. + +.. date: 2022-10-19-01-01-08 +.. gh-issue: 98415 +.. nonce: ZS2eWh +.. section: Core and Builtins + +Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by +Chaim Sanders + +.. + +.. date: 2022-10-15-23-15-14 +.. gh-issue: 92119 +.. nonce: PMSwwG +.. section: Core and Builtins + +Print exception class name instead of its string representation when raising +errors from :mod:`ctypes` calls. + +.. + +.. date: 2022-10-06-15-45-57 +.. gh-issue: 96078 +.. nonce: fS-6mU +.. section: Core and Builtins + +:func:`os.sched_yield` now release the GIL while calling sched_yield(2). +Patch by Dong-hee Na. + +.. + +.. date: 2022-10-06-05-41-01 +.. gh-issue: 93354 +.. nonce: 6BpHl2 +.. section: Core and Builtins + +Fix an issue that could delay the specialization of :opcode:`PRECALL` +instructions. + +.. + +.. date: 2022-10-05-17-02-22 +.. gh-issue: 97943 +.. nonce: LYAWlE +.. section: Core and Builtins + +Bugfix: :func:`PyFunction_GetAnnotations` should return a borrowed +reference. It was returning a new reference. + +.. + +.. date: 2022-10-04-02-00-10 +.. gh-issue: 97779 +.. nonce: f3N1hI +.. section: Core and Builtins + +Ensure that all Python frame objects are backed by "complete" frames. + +.. + +.. date: 2022-10-01-08-55-09 +.. gh-issue: 97591 +.. nonce: pw6kkH +.. section: Core and Builtins + +Fixed a missing incref/decref pair in ``Exception.__setstate__()``. Patch by +Ofey Chan. + +.. + +.. date: 2022-09-29-15-19-29 +.. gh-issue: 94526 +.. nonce: wq5m6T +.. section: Core and Builtins + +Fix the Python path configuration used to initialized :data:`sys.path` at +Python startup. Paths are no longer encoded to UTF-8/strict to avoid +encoding errors if it contains surrogate characters (bytes paths are decoded +with the surrogateescape error handler). Patch by Victor Stinner. + +.. + +.. date: 2022-09-20-11-06-45 +.. gh-issue: 95921 +.. nonce: dkcRQn +.. section: Core and Builtins + +Fix overly-broad source position information for chained comparisons used as +branching conditions. + +.. + +.. date: 2022-09-16-16-54-35 +.. gh-issue: 96387 +.. nonce: GRzewg +.. section: Core and Builtins + +At Python exit, sometimes a thread holding the GIL can wait forever for a +thread (usually a daemon thread) which requested to drop the GIL, whereas +the thread already exited. To fix the race condition, the thread which +requested the GIL drop now resets its request before exiting. Issue +discovered and analyzed by Mingliang ZHAO. Patch by Victor Stinner. + +.. + +.. date: 2022-09-16-12-36-13 +.. gh-issue: 96864 +.. nonce: PLU3i8 +.. section: Core and Builtins + +Fix a possible assertion failure, fatal error, or :exc:`SystemError` if a +line tracing event raises an exception while opcode tracing is enabled. + +.. + +.. date: 2022-09-13-12-06-46 +.. gh-issue: 96678 +.. nonce: NqGFyb +.. section: Core and Builtins + +Fix undefined behaviour in C code of null pointer arithmetic. + +.. + +.. date: 2022-09-12-16-58-22 +.. gh-issue: 96754 +.. nonce: 0GRme5 +.. section: Core and Builtins + +Make sure that all frame objects created are created from valid interpreter +frames. Prevents the possibility of invalid frames in backtraces and signal +handlers. + +.. + +.. date: 2022-08-29-13-06-58 +.. gh-issue: 95196 +.. nonce: eGRR4b +.. section: Core and Builtins + +Disable incorrect pickling of the C implemented classmethod descriptors. + +.. + +.. date: 2022-08-15-21-08-11 +.. gh-issue: 96005 +.. nonce: 6eoc8k +.. section: Core and Builtins + +On WASI :data:`~errno.ENOTCAPABLE` is now mapped to :exc:`PermissionError`. +The :mod:`errno` modules exposes the new error number. ``getpath.py`` now +ignores :exc:`PermissionError` when it cannot open landmark files +``pybuilddir.txt`` and ``pyenv.cfg``. + +.. + +.. date: 2022-06-10-16-37-44 +.. gh-issue: 93696 +.. nonce: 65BI2R +.. section: Core and Builtins + +Allow :mod:`pdb` to locate source for frozen modules in the standard +library. + +.. + +.. bpo: 31718 +.. date: 2020-02-23-23-48-15 +.. nonce: sXko5e +.. section: Core and Builtins + +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. + +.. + +.. bpo: 38031 +.. date: 2019-09-04-19-09-49 +.. nonce: Yq4L72 +.. section: Core and Builtins + +Fix a possible assertion failure in :class:`io.FileIO` when the opener +returns an invalid file descriptor. + +.. + +.. date: 2022-12-05-13-40-15 +.. gh-issue: 100001 +.. nonce: 78ReYp +.. section: Library + +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. + +.. + +.. date: 2022-12-02-13-05-00 +.. gh-issue: 93453 +.. nonce: EFj1NN +.. section: Library + +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. + +.. + +.. date: 2022-11-21-17-56-18 +.. gh-issue: 51524 +.. nonce: nTykx8 +.. section: Library + +Fix bug when calling trace.CoverageResults with valid infile. + +.. + +.. date: 2022-11-21-13-49-03 +.. gh-issue: 99645 +.. nonce: 9w1QKq +.. section: Library + +Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now +``addClassCleanup()`` uses separate lists for different ``TestCase`` +subclasses, and ``doClassCleanups()`` only cleans up the particular class. + +.. + +.. date: 2022-11-15-10-55-24 +.. gh-issue: 97001 +.. nonce: KeQuVF +.. section: Library + +Release the GIL when calling termios APIs to avoid blocking threads. + +.. + +.. date: 2022-11-13-02-06-56 +.. gh-issue: 99341 +.. nonce: 8-OlwB +.. section: Library + +Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when +changing line numbers. + +.. + +.. date: 2022-11-12-15-45-51 +.. gh-issue: 99418 +.. nonce: FxfAXS +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes URL schemes that begin +with a digit, a plus sign, or a minus sign to be parsed incorrectly. + +.. + +.. date: 2022-11-12-12-15-30 +.. gh-issue: 99382 +.. nonce: dKg_rW +.. section: Library + +Check the number of arguments in substitution in user generics containing a +:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. + +.. + +.. date: 2022-11-12-12-10-23 +.. gh-issue: 99379 +.. nonce: bcGhxF +.. section: Library + +Fix substitution of :class:`~typing.ParamSpec` followed by +:class:`~typing.TypeVarTuple` in generic aliases. + +.. + +.. date: 2022-11-12-12-08-34 +.. gh-issue: 99344 +.. nonce: 7M_u8G +.. section: Library + +Fix substitution of :class:`~typing.TypeVarTuple` and +:class:`~typing.ParamSpec` together in user generics. + +.. + +.. date: 2022-11-09-20-48-42 +.. gh-issue: 74044 +.. nonce: zBj26K +.. section: Library + +Fixed bug where :func:`inspect.signature` reported incorrect arguments for +decorated methods. + +.. + +.. date: 2022-11-09-12-16-35 +.. gh-issue: 99275 +.. nonce: klOqoL +.. section: Library + +Fix ``SystemError`` in :mod:`ctypes` when exception was not set during +``__initsubclass__``. + +.. + +.. date: 2022-11-09-08-40-52 +.. gh-issue: 99277 +.. nonce: J1P44O +.. section: Library + +Remove older version of ``_SSLProtocolTransport.get_write_buffer_limits`` in +:mod:`!asyncio.sslproto` + +.. + +.. date: 2022-11-08-11-15-37 +.. gh-issue: 99248 +.. nonce: 1vt8xI +.. section: Library + +fix negative numbers failing in verify() + +.. + +.. date: 2022-11-06-12-44-51 +.. gh-issue: 99155 +.. nonce: vLZOzi +.. section: Library + +Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. + +.. + +.. date: 2022-11-05-23-16-15 +.. gh-issue: 93464 +.. nonce: ucd4vP +.. section: Library + +``enum.auto()`` is now correctly activated when combined with other +assignment values. E.g. ``ONE = auto(), 'some text'`` will now evaluate as +``(1, 'some text')``. + +.. + +.. date: 2022-11-05-17-16-40 +.. gh-issue: 99134 +.. nonce: Msgspf +.. section: Library + +Update the bundled copy of pip to version 22.3.1. + +.. + +.. date: 2022-11-02-05-54-02 +.. gh-issue: 83004 +.. nonce: 0v8iyw +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_zoneinfo` + +.. + +.. date: 2022-11-02-05-53-25 +.. gh-issue: 83004 +.. nonce: qc_KHr +.. section: Library + +Clean up refleaks on failed module initialisation in in :mod:`_pickle` + +.. + +.. date: 2022-11-02-05-52-36 +.. gh-issue: 83004 +.. nonce: LBl79O +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_io`. + +.. + +.. date: 2022-10-31-12-34-03 +.. gh-issue: 98897 +.. nonce: rgNn4x +.. section: Library + +Fix memory leak in :func:`math.dist` when both points don't have the same +dimension. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-30-12-22-24 +.. gh-issue: 98706 +.. nonce: v1Kuy5 +.. section: Library + +[3.11] Applied changes from importlib_metadata `4.11.4 through 4.13 +<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-13-0>`_, +including compatibility and robustness fixes for ``Distribution`` objects +without ``_normalized_name``, disallowing invalid inputs to +``Distribution.from_name``, and refined behaviors in +``PathDistribution._name_from_stem`` and +``PathDistribution._normalized_name``. + +.. + +.. date: 2022-10-29-03-40-18 +.. gh-issue: 98793 +.. nonce: WSPB4A +.. section: Library + +Fix argument typechecks in :func:`!_overlapped.WSAConnect` and +:func:`!_overlapped.Overlapped.WSASendTo` functions. + +.. + +.. date: 2022-10-28-23-44-17 +.. gh-issue: 98744 +.. nonce: sGHDWm +.. section: Library + +Prevent crashing in :mod:`traceback` when retrieving the byte-offset for +some source files that contain certain unicode characters. + +.. + +.. date: 2022-10-27-12-56-38 +.. gh-issue: 98740 +.. nonce: ZoqqGM +.. section: Library + +Fix internal error in the :mod:`re` module which in very rare circumstances +prevented compilation of a regular expression containing a :ref:`conditional +expression <re-conditional-expression>` without the "else" branch. + +.. + +.. date: 2022-10-26-07-51-55 +.. gh-issue: 98703 +.. nonce: 0hW773 +.. section: Library + +Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` +callback only once on Windows. + +.. + +.. date: 2022-10-25-20-17-34 +.. gh-issue: 98624 +.. nonce: YQUPFy +.. section: Library + +Add a mutex to unittest.mock.NonCallableMock to protect concurrent access to +mock attributes. + +.. + +.. date: 2022-10-23-18-30-39 +.. gh-issue: 89237 +.. nonce: kBui30 +.. section: Library + +Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with +:class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-19-18-31-53 +.. gh-issue: 98458 +.. nonce: vwyq7O +.. section: Library + +Fix infinite loop in unittest when a self-referencing chained exception is +raised + +.. + +.. date: 2022-10-19-09-29-12 +.. gh-issue: 97928 +.. nonce: xj3im7 +.. section: Library + +:meth:`tkinter.Text.count` raises now an exception for options starting with +"-" instead of silently ignoring them. + +.. + +.. date: 2022-10-16-18-52-00 +.. gh-issue: 97966 +.. nonce: humlhz +.. section: Library + +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. + +.. + +.. date: 2022-10-16-06-18-59 +.. gh-issue: 98307 +.. nonce: b2_CDu +.. section: Library + +A :meth:`~logging.handlers.SysLogHandler.createSocket` method was added to +:class:`~logging.handlers.SysLogHandler`. + +.. + +.. date: 2022-10-14-19-57-37 +.. gh-issue: 96035 +.. nonce: 0xcX-p +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers +containing whitespace, underscores, plus and minus signs, or non-ASCII +digits to be incorrectly accepted. + +.. + +.. date: 2022-10-14-11-46-31 +.. gh-issue: 98251 +.. nonce: Uxc9al +.. section: Library + +Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +and ``pip`` when they do not impact path resolution + +.. + +.. date: 2022-10-12-10-00-40 +.. gh-issue: 98178 +.. nonce: hspH51 +.. section: Library + +On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded +applications. On macOS, the libc ``syslog()`` function is not thread-safe, +so :func:`syslog.syslog` no longer releases the GIL to call it. Patch by +Victor Stinner. + +.. + +.. date: 2022-10-10-07-07-31 +.. gh-issue: 96151 +.. nonce: K9fwoq +.. section: Library + +Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. + +.. + +.. date: 2022-10-09-12-12-38 +.. gh-issue: 87730 +.. nonce: ClgP3f +.. section: Library + +Wrap network errors consistently in urllib FTP support, so the test suite +doesn't fail when a network is available but the public internet is not +reachable. + +.. + +.. date: 2022-10-08-19-39-27 +.. gh-issue: 98086 +.. nonce: y---WC +.. section: Library + +Make sure ``patch.dict()`` can be applied on async functions. + +.. + +.. date: 2022-10-06-23-42-00 +.. gh-issue: 90985 +.. nonce: s280JY +.. section: Library + +Earlier in 3.11 we deprecated ``asyncio.Task.cancel("message")``. We +realized we were too harsh, and have undeprecated it. + +.. + +.. date: 2022-10-04-21-21-41 +.. gh-issue: 97837 +.. nonce: 19q-eg +.. section: Library + +Change deprecate warning message in :mod:`unittest` from + +``It is deprecated to return a value!=None`` + +to + +``It is deprecated to return a value that is not None from a test case`` + +.. + +.. date: 2022-10-04-07-55-19 +.. gh-issue: 97825 +.. nonce: mNdv1l +.. section: Library + +Fixes :exc:`AttributeError` when :meth:`subprocess.check_output` is used +with argument ``input=None`` and either of the arguments *encoding* or +*errors* are used. + +.. + +.. date: 2022-10-02-12-38-22 +.. gh-issue: 82836 +.. nonce: OvYLmC +.. section: Library + +Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the +:mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would +return True from this method; now they correctly return False. + +.. + +.. date: 2022-09-30-15-56-20 +.. gh-issue: 96827 +.. nonce: lzy1iw +.. section: Library + +Avoid spurious tracebacks from :mod:`asyncio` when default executor cleanup +is delayed until after the event loop is closed (e.g. as the result of a +keyboard interrupt). + +.. + +.. date: 2022-09-29-23-22-24 +.. gh-issue: 97592 +.. nonce: tpJg_J +.. section: Library + +Avoid a crash in the C version of +:meth:`asyncio.Future.remove_done_callback` when an evil argument is passed. + +.. + +.. date: 2022-09-29-08-15-55 +.. gh-issue: 97639 +.. nonce: JSjWYW +.. section: Library + +Remove ``tokenize.NL`` check from :mod:`tabnanny`. + +.. + +.. date: 2022-09-25-20-42-33 +.. gh-issue: 73588 +.. nonce: uVtjEA +.. section: Library + +Fix generation of the default name of :class:`tkinter.Checkbutton`. +Previously, checkbuttons in different parent widgets could have the same +short name and share the same state if arguments "name" and "variable" are +not specified. Now they are globally unique. + +.. + +.. date: 2022-09-22-14-35-02 +.. gh-issue: 97005 +.. nonce: yf21Q7 +.. section: Library + +Update bundled libexpat to 2.4.9 + +.. + +.. date: 2022-09-22-11-50-29 +.. gh-issue: 85760 +.. nonce: DETTPd +.. section: Library + +Fix race condition in :mod:`asyncio` where +:meth:`~asyncio.SubprocessProtocol.process_exited` called before the +:meth:`~asyncio.SubprocessProtocol.pipe_data_received` leading to +inconsistent output. Patch by Kumar Aditya. + +.. + +.. date: 2022-09-17-13-15-10 +.. gh-issue: 96819 +.. nonce: 6RfqM7 +.. section: Library + +Fixed check in :mod:`multiprocessing.resource_tracker` that guarantees that +the length of a write to a pipe is not greater than ``PIPE_BUF``. + +.. + +.. date: 2022-09-15-00-37-33 +.. gh-issue: 96741 +.. nonce: 4b6czN +.. section: Library + +Corrected type annotation for dataclass attribute +``pstats.FunctionProfile.ncalls`` to be ``str``. + +.. + +.. date: 2022-08-30-11-46-36 +.. gh-issue: 95987 +.. nonce: CV7_u4 +.. section: Library + +Fix ``repr`` of ``Any`` subclasses. + +.. + +.. date: 2022-08-29-16-54-36 +.. gh-issue: 96388 +.. nonce: dCpJcu +.. section: Library + +Work around missing socket functions in :class:`~socket.socket`'s +``__repr__``. + +.. + +.. date: 2022-08-29-12-35-28 +.. gh-issue: 96073 +.. nonce: WaGstf +.. section: Library + +In :mod:`inspect`, fix overeager replacement of "``typing.``" in formatting +annotations. + +.. + +.. date: 2022-08-23-03-13-18 +.. gh-issue: 96192 +.. nonce: TJywOF +.. section: Library + +Fix handling of ``bytes`` :term:`path-like objects <path-like object>` in +:func:`os.ismount()`. + +.. + +.. date: 2022-08-20-10-31-01 +.. gh-issue: 96052 +.. nonce: a6FhaD +.. section: Library + +Fix handling compiler warnings (SyntaxWarning and DeprecationWarning) in +:func:`codeop.compile_command` when checking for incomplete input. +Previously it emitted warnings and raised a SyntaxError. Now it always +returns ``None`` for incomplete input without emitting any warnings. + +.. + +.. date: 2022-08-06-12-18-07 +.. gh-issue: 88863 +.. nonce: NnqsuJ +.. section: Library + +To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, +break reference cycles generated by local exception and future instances +(which has exception instance as its member var). Patch by Dong Uk, Kang. + +.. + +.. date: 2022-07-22-09-09-08 +.. gh-issue: 91212 +.. nonce: 53O8Ab +.. section: Library + +Fixed flickering of the turtle window when the tracer is turned off. Patch +by Shin-myoung-serp. + +.. + +.. date: 2022-07-08-08-39-35 +.. gh-issue: 88050 +.. nonce: 0aOC_m +.. section: Library + +Fix :mod:`asyncio` subprocess transport to kill process cleanly when process +is blocked and avoid ``RuntimeError`` when loop is closed. Patch by Kumar +Aditya. + +.. + +.. date: 2022-06-17-12-02-30 +.. gh-issue: 93858 +.. nonce: R49ARc +.. section: Library + +Prevent error when activating venv in nested fish instances. + +.. + +.. date: 2022-04-23-03-46-37 +.. gh-issue: 91078 +.. nonce: 87-hkp +.. section: Library + +:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. + +.. + +.. bpo: 47220 +.. date: 2022-04-04-22-54-11 +.. nonce: L9jYu4 +.. section: Library + +Document the optional *callback* parameter of :class:`WeakMethod`. Patch by +G?ry Ogam. + +.. + +.. bpo: 46364 +.. date: 2022-01-14-10-49-20 +.. nonce: SzhlU9 +.. section: Library + +Restrict use of sockets instead of pipes for stdin of subprocesses created +by :mod:`asyncio` to AIX platform only. + +.. + +.. bpo: 38523 +.. date: 2020-10-23-22-20-52 +.. nonce: CrkxLh +.. section: Library + +:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument +recursively. + +.. + +.. bpo: 36267 +.. date: 2019-09-03-15-45-19 +.. nonce: z42Ex7 +.. section: Library + +Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` +action is given an explicit argument. + +.. + +.. date: 2022-11-16-12-52-23 +.. gh-issue: 92892 +.. nonce: TS-P0j +.. section: Documentation + +Document that calling variadic functions with ctypes requires special care +on macOS/arm64 (and possibly other platforms). + +.. + +.. date: 2022-10-16-17-34-45 +.. gh-issue: 85525 +.. nonce: DvkD0v +.. section: Documentation + +Remove extra row + +.. + +.. date: 2022-08-12-01-12-52 +.. gh-issue: 95588 +.. nonce: PA0FI7 +.. section: Documentation + +Clarified the conflicting advice given in the :mod:`ast` documentation about +:func:`ast.literal_eval` being "safe" for use on untrusted input while at +the same time warning that it can crash the process. The latter statement is +true and is deemed unfixable without a large amount of work unsuitable for a +bugfix. So we keep the warning and no longer claim that ``literal_eval`` is +safe. + +.. + +.. bpo: 41825 +.. date: 2020-09-22-12-32-16 +.. nonce: npcaCb +.. section: Documentation + +Restructured the documentation for the :func:`os.wait* <os.wait>` family of +functions, and improved the docs for :func:`os.waitid` with more explanation +of the possible argument constants. + +.. + +.. date: 2022-12-05-16-12-56 +.. gh-issue: 99892 +.. nonce: sz_eW8 +.. section: Tests + +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. + +.. + +.. date: 2022-12-01-18-55-18 +.. gh-issue: 99934 +.. nonce: Ox3Fqf +.. section: Tests + +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. + +.. + +.. date: 2022-11-21-19-21-30 +.. gh-issue: 99659 +.. nonce: 4gP0nm +.. section: Tests + +Optional big memory tests in ``test_sqlite3`` now catch the correct +:exc:`sqlite.DataError` exception type in case of too large strings and/or +blobs passed. + +.. + +.. date: 2022-10-26-15-19-20 +.. gh-issue: 98713 +.. nonce: Lnu32R +.. section: Tests + +Fix a bug in the :mod:`typing` tests where a test relying on +CPython-specific implementation details was not decorated with +``@cpython_only`` and was not skipped on other implementations. + +.. + +.. date: 2022-10-15-07-46-48 +.. gh-issue: 87390 +.. nonce: asR-Zo +.. section: Tests + +Add tests for star-unpacking with PEP 646, and some other miscellaneous PEP +646 tests. + +.. + +.. date: 2022-10-12-14-57-06 +.. gh-issue: 96853 +.. nonce: ANe-bw +.. section: Tests + +Added explicit coverage of ``Py_Initialize`` (and hence ``Py_InitializeEx``) +back to the embedding tests (all other embedding tests migrated to +``Py_InitializeFromConfig`` in Python 3.11) + +.. + +.. bpo: 34272 +.. date: 2018-07-29-15-59-51 +.. nonce: lVX2uR +.. section: Tests + +Some C API tests were moved into the new Lib/test/test_capi/ directory. + +.. + +.. date: 2022-11-24-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and +``-Wimplicit-function-declaration`` compiler warnings in +:program:`configure` checks. + +.. + +.. date: 2022-11-15-08-40-22 +.. gh-issue: 99337 +.. nonce: 5LoQDE +.. section: Build + +Fix a compilation issue with GCC 12 on macOS. + +.. + +.. date: 2022-11-04-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for +``PTHREAD_SCOPE_SYSTEM``. + +.. + +.. date: 2022-11-03-08-10-49 +.. gh-issue: 98872 +.. nonce: gdsR8X +.. section: Build + +Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python +3.11. + +.. + +.. date: 2022-11-02-19-25-07 +.. gh-issue: 99016 +.. nonce: R05NkD +.. section: Build + +Fix build with ``PYTHON_FOR_REGEN=python3.8``. + +.. + +.. date: 2022-11-02-18-45-35 +.. gh-issue: 97731 +.. nonce: zKpTlj +.. section: Build + +Specify the full path to the source location for ``make docclean`` (needed +for cross-builds). + +.. + +.. date: 2022-10-26-12-37-52 +.. gh-issue: 98707 +.. nonce: eVXGEx +.. section: Build + +Don't use vendored ``libmpdec`` headers if :option:`--with-system-libmpdec` +is passed to :program:`configure`. Don't use vendored ``libexpat`` headers +if :option:`--with-system-expat` is passed to :program:`!configure`. + +.. + +.. date: 2022-09-20-12-43-44 +.. gh-issue: 96761 +.. nonce: IF29kR +.. section: Build + +Fix the build process of clang compiler for :program:`_bootstrap_python` if +LTO optimization is applied. Patch by Matthias G?rgens and Dong-hee Na. + +.. + +.. date: 2022-09-17-11-19-24 +.. gh-issue: 96883 +.. nonce: p_gr62 +.. section: Build + +``wasm32-emscripten`` builds for browsers now include +:mod:`concurrent.futures` for :mod:`asyncio` and :mod:`unittest.mock`. + +.. + +.. date: 2022-08-26-11-09-11 +.. gh-issue: 84461 +.. nonce: Nsdn_R +.. section: Build + +``wasm32-emscripten`` platform no longer builds :mod:`resource` module, +:func:`~os.getresuid`, :func:`~os.getresgid`, and their setters. The APIs +are stubs and not functional. + +.. + +.. date: 2022-06-25-23-25-47 +.. gh-issue: 94280 +.. nonce: YhEyW_ +.. section: Build + +Updated pegen regeneration script on Windows to find and use Python 3.9 or +higher. Prior to this, pegen regeneration already required 3.9 or higher, +but the script may have used lower versions of Python. + +.. + +.. date: 2022-11-23-17-17-16 +.. gh-issue: 99345 +.. nonce: jOa3-f +.. section: Windows + +Use faster initialization functions to detect install location for Windows +Store package + +.. + +.. date: 2022-11-21-19-50-18 +.. gh-issue: 98629 +.. nonce: tMmB_B +.. section: Windows + +Fix initialization of :data:`sys.version` and ``sys._git`` on Windows + +.. + +.. date: 2022-11-16-19-03-21 +.. gh-issue: 99442 +.. nonce: 6Dgk3Q +.. section: Windows + +Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file +extension. + +.. + +.. date: 2022-11-01-11-07-33 +.. gh-issue: 98689 +.. nonce: 0f6e_N +.. section: Windows + +Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +vulnerable ``inflateGetHeader`` API is not used by Python. + +.. + +.. date: 2022-11-01-00-37-13 +.. gh-issue: 98790 +.. nonce: fpaPAx +.. section: Windows + +Assumes that a missing ``DLLs`` directory means that standard extension +modules are in the executable's directory. + +.. + +.. date: 2022-10-27-20-30-16 +.. gh-issue: 98745 +.. nonce: v06p4r +.. section: Windows + +Update :file:`py.exe` launcher to install 3.11 by default and 3.12 on +request. + +.. + +.. date: 2022-10-26-17-43-09 +.. gh-issue: 98692 +.. nonce: bOopfZ +.. section: Windows + +Fix the :ref:`launcher` ignoring unrecognized shebang lines instead of +treating them as local paths + +.. + +.. date: 2022-10-25-10-34-17 +.. gh-issue: 94328 +.. nonce: 19NhdU +.. section: Windows + +Update Windows installer to use SQLite 3.39.4. + +.. + +.. date: 2022-10-02-11-59-23 +.. gh-issue: 97728 +.. nonce: dIdlPE +.. section: Windows + +Fix possible crashes caused by the use of uninitialized variables when pass +invalid arguments in :func:`os.system` on Windows and in Windows-specific +modules (like ``winreg``). + +.. + +.. date: 2022-09-23-15-40-04 +.. gh-issue: 96965 +.. nonce: CsnEGs +.. section: Windows + +Update libffi to 3.4.3 + +.. + +.. date: 2022-08-30-12-01-51 +.. gh-issue: 94781 +.. nonce: OxO-Gr +.. section: Windows + +Fix :file:`pcbuild.proj` to clean previous instances of ouput files in +``Python\deepfreeze`` and ``Python\frozen_modules`` directories on Windows. +Patch by Charlie Zhao. + +.. + +.. bpo: 40882 +.. date: 2020-06-06-15-10-37 +.. nonce: UvNbdj +.. section: Windows + +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. + +.. + +.. date: 2022-11-25-09-23-20 +.. gh-issue: 87235 +.. nonce: SifjCD +.. section: macOS + +On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script +longer than a couple of bytes. + +.. + +.. date: 2022-11-01-10-32-23 +.. gh-issue: 98940 +.. nonce: W3YzC_ +.. section: macOS + +Fix ``Mac/Extras.install.py`` file filter bug. + +.. + +.. date: 2022-10-25-10-32-23 +.. gh-issue: 94328 +.. nonce: W3YNC_ +.. section: macOS + +Update macOS installer to SQLite 3.39.4. + +.. + +.. date: 2022-10-15-21-20-40 +.. gh-issue: 97527 +.. nonce: otAHJM +.. section: IDLE + +Fix a bug in the previous bugfix that caused IDLE to not start when run with +3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed +without the Lib/test package. 3.11.0 was never affected. + +.. + +.. date: 2022-08-29-17-25-13 +.. gh-issue: 95853 +.. nonce: Ce17cT +.. section: Tools/Demos + +The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for +broken EMSDK versions, and warns about pkg-config env vars. + +.. + +.. date: 2022-08-10-17-08-43 +.. gh-issue: 95853 +.. nonce: HCjC2m +.. section: Tools/Demos + +The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, +and test steps for building CPython on WebAssembly platforms. + +.. + +.. date: 2022-08-05-23-25-59 +.. gh-issue: 95731 +.. nonce: N2KohU +.. section: Tools/Demos + +Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. + +.. + +.. date: 2022-12-05-17-30-13 +.. gh-issue: 98680 +.. nonce: FiMCxZ +.. section: C API + +``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. +These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for +3.11, and are necessary to use the buffer API. + +.. + +.. date: 2022-11-03-17-46-41 +.. gh-issue: 98978 +.. nonce: KJjBvv +.. section: C API + +Fix use-after-free in ``Py_SetPythonHome(NULL)``, +``Py_SetProgramName(NULL)`` and ``_Py_SetProgramFullPath(NULL)`` function +calls. Issue reported by Benedikt Reinartz. Patch by Victor Stinner. + +.. + +.. date: 2022-10-16-15-00-25 +.. gh-issue: 96853 +.. nonce: V0wiXP +.. section: C API + +``Py_InitializeEx`` now correctly calls ``PyConfig_Clear`` after +initializing the interpreter (the omission didn't cause a memory leak only +because none of the dynamically allocated config fields are populated by the +wrapper function) diff --git a/Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst b/Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst deleted file mode 100644 index 1199e842177d..000000000000 --- a/Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Updated pegen regeneration script on Windows to find and use Python 3.9 or -higher. Prior to this, pegen regeneration already required 3.9 or higher, -but the script may have used lower versions of Python. diff --git a/Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst b/Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst deleted file mode 100644 index 134e9645159a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst +++ /dev/null @@ -1,3 +0,0 @@ -``wasm32-emscripten`` platform no longer builds :mod:`resource` module, -:func:`~os.getresuid`, :func:`~os.getresgid`, and their setters. The APIs -are stubs and not functional. diff --git a/Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst b/Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst deleted file mode 100644 index 2258ce8ab9d5..000000000000 --- a/Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst +++ /dev/null @@ -1,2 +0,0 @@ -``wasm32-emscripten`` builds for browsers now include -:mod:`concurrent.futures` for :mod:`asyncio` and :mod:`unittest.mock`. diff --git a/Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst b/Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst deleted file mode 100644 index 18f75ac48910..000000000000 --- a/Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the build process of clang compiler for :program:`_bootstrap_python` if -LTO optimization is applied. Patch by Matthias G?rgens and Dong-hee Na. diff --git a/Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst b/Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst deleted file mode 100644 index 69afa9dea307..000000000000 --- a/Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst +++ /dev/null @@ -1,4 +0,0 @@ -Don't use vendored ``libmpdec`` headers if :option:`--with-system-libmpdec` -is passed to :program:`configure`. -Don't use vendored ``libexpat`` headers if :option:`--with-system-expat` -is passed to :program:`!configure`. diff --git a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst b/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst deleted file mode 100644 index 46b1fb833d47..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specify the full path to the source location for ``make docclean`` (needed for -cross-builds). diff --git a/Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst b/Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst deleted file mode 100644 index df189daca3a2..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst +++ /dev/null @@ -1 +0,0 @@ -Fix build with ``PYTHON_FOR_REGEN=python3.8``. diff --git a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst b/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst deleted file mode 100644 index ad4dc496ee0e..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python 3.11. diff --git a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index e320ecfdfbb7..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for ``PTHREAD_SCOPE_SYSTEM``. diff --git a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst b/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst deleted file mode 100644 index f7396abde13b..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a compilation issue with GCC 12 on macOS. diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index 2dace165ca1a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst b/Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst deleted file mode 100644 index d7e3cc423ac8..000000000000 --- a/Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst +++ /dev/null @@ -1,4 +0,0 @@ -``Py_InitializeEx`` now correctly calls ``PyConfig_Clear`` after initializing -the interpreter (the omission didn't cause a memory leak only because none -of the dynamically allocated config fields are populated by the wrapper -function) diff --git a/Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst b/Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst deleted file mode 100644 index b9672728009a..000000000000 --- a/Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix use-after-free in ``Py_SetPythonHome(NULL)``, -``Py_SetProgramName(NULL)`` and ``_Py_SetProgramFullPath(NULL)`` function -calls. Issue reported by Benedikt Reinartz. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst deleted file mode 100644 index a87090168e06..000000000000 --- a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. -These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for -3.11, and are necessary to use the buffer API. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst deleted file mode 100644 index b5964375962f..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure in :class:`io.FileIO` when the opener -returns an invalid file descriptor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst deleted file mode 100644 index dd96c9e20d87..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of -uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. -Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst deleted file mode 100644 index 8eadab0ad8fb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst +++ /dev/null @@ -1 +0,0 @@ -Allow :mod:`pdb` to locate source for frozen modules in the standard library. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst b/Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst deleted file mode 100644 index 06e414bca0f9..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst +++ /dev/null @@ -1,4 +0,0 @@ -On WASI :data:`~errno.ENOTCAPABLE` is now mapped to :exc:`PermissionError`. -The :mod:`errno` modules exposes the new error number. ``getpath.py`` now -ignores :exc:`PermissionError` when it cannot open landmark files -``pybuilddir.txt`` and ``pyenv.cfg``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst b/Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst deleted file mode 100644 index 37534fa17525..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst +++ /dev/null @@ -1 +0,0 @@ -Disable incorrect pickling of the C implemented classmethod descriptors. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst deleted file mode 100644 index beac84ee822a..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make sure that all frame objects created are created from valid interpreter -frames. Prevents the possibility of invalid frames in backtraces and signal -handlers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst deleted file mode 100644 index bdd33c8d2ca9..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix undefined behaviour in C code of null pointer arithmetic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst deleted file mode 100644 index c0d41ae7d21e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure, fatal error, or :exc:`SystemError` if a -line tracing event raises an exception while opcode tracing is enabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst deleted file mode 100644 index 611ab94bc636..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst +++ /dev/null @@ -1,5 +0,0 @@ -At Python exit, sometimes a thread holding the GIL can wait forever for a -thread (usually a daemon thread) which requested to drop the GIL, whereas -the thread already exited. To fix the race condition, the thread which -requested the GIL drop now resets its request before exiting. Issue -discovered and analyzed by Mingliang ZHAO. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst deleted file mode 100644 index 0c8b704c9510..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix overly-broad source position information for chained comparisons used as -branching conditions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst deleted file mode 100644 index 59e389a64ee0..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the Python path configuration used to initialized :data:`sys.path` at -Python startup. Paths are no longer encoded to UTF-8/strict to avoid encoding -errors if it contains surrogate characters (bytes paths are decoded with the -surrogateescape error handler). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst deleted file mode 100644 index 6f07529f15bb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a missing incref/decref pair in ``Exception.__setstate__()``. -Patch by Ofey Chan. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst deleted file mode 100644 index 611521808865..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure that all Python frame objects are backed by "complete" frames. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst deleted file mode 100644 index 9b4a421a9d47..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Bugfix: :func:`PyFunction_GetAnnotations` should return a borrowed -reference. It was returning a new reference. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst deleted file mode 100644 index 4efc10da293a..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue that could delay the specialization of :opcode:`PRECALL` -instructions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst deleted file mode 100644 index d1f949c6e13a..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`os.sched_yield` now release the GIL while calling sched_yield(2). -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst deleted file mode 100644 index 7142fc619765..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Print exception class name instead of its string representation when raising -errors from :mod:`ctypes` calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst deleted file mode 100644 index af2db1f9965c..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst +++ /dev/null @@ -1 +0,0 @@ -Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by Chaim Sanders diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst deleted file mode 100644 index 56a41e3883d1..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Suppress ImportError for invalid query for help() command. Patch by Dong-hee -Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst deleted file mode 100644 index e8de568e8528..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue where member descriptors (such as those for -:attr:`~object.__slots__`) could behave incorrectly or crash instead of -raising a :exc:`TypeError` when accessed via an instance of an invalid type. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst deleted file mode 100644 index da1e61ea8504..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix multiple crashes in debug mode when ``str`` subclasses -are used instead of ``str`` itself. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst deleted file mode 100644 index 6f23969f4ddc..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Lower the recursion depth for marshal on WASI to support -wasmtime 2.0/main. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst deleted file mode 100644 index 0e15819a862b..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix subscription of type aliases containing bare generic types or types like -:class:`~typing.TypeVar`: for example ``tuple[A, T][int]`` and -``tuple[TypeVar, T][int]``, where ``A`` is a generic type, and ``T`` is a -type variable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst deleted file mode 100644 index f5378eb837d1..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the error reporting positions of specialized traceback anchors when the -source line contains Unicode characters. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst deleted file mode 100644 index 0445afbbc4fb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst +++ /dev/null @@ -1 +0,0 @@ -Fix location of :exc:`SyntaxError` for a :keyword:`try` block with both :keyword:`except` and :keyword:`except* <except_star>`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst deleted file mode 100644 index c72fb21942e6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`faulthandler` to emit an error message with the proper -unexpected signal number. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst deleted file mode 100644 index 571cdd02cd55..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix calculation of :data:`sys._base_executable` when inside a POSIX virtual -environment using copies of the python binary when the base installation does -not provide the executable name used by the venv. Calculation will fall back to -alternative names ("python<MAJOR>", "python<MAJOR>.<MINOR>"). diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst deleted file mode 100644 index aa6160dd5a5e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst +++ /dev/null @@ -1 +0,0 @@ -Fix failure in :keyword:`except* <except_star>` with unhashable exceptions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst deleted file mode 100644 index 8ad0e147c203..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue that prevented :c:type:`PyThreadState` and -:c:type:`PyInterpreterState` memory from being freed properly. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst deleted file mode 100644 index 8908bfaa8e25..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue that could potentially cause incorrect error handling for some -bytecode instructions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst deleted file mode 100644 index 142f91ccd92e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix zip path for venv created from a non-installed python on POSIX -platforms. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst deleted file mode 100644 index 8d9f55d6d9d0..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where an :exc:`ExceptionGroup` subclass can wrap a -:exc:`BaseException`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst deleted file mode 100644 index 8071fd130dd6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the -available tokenizer buffer. Patch by Pablo galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst deleted file mode 100644 index 9321cef77eed..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of the -first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst deleted file mode 100644 index 3fe21a8a21bf..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue that could cause frames to be visible to Python code as they -are being torn down, possibly leading to memory corruption or hard crashes -of the interpreter. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst deleted file mode 100644 index 20cd361affea..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the tokenizer that could cause infinite recursion when showing -syntax warnings that happen in the first line of the source. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst deleted file mode 100644 index 8bdaa9462750..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash when an object which does not have a dictionary frees its instance values. diff --git a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst b/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst deleted file mode 100644 index 390b4a9824c7..000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Restructured the documentation for the :func:`os.wait* <os.wait>` family of functions, -and improved the docs for :func:`os.waitid` with more explanation of the -possible argument constants. diff --git a/Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst b/Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst deleted file mode 100644 index c070bbc19517..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst +++ /dev/null @@ -1,6 +0,0 @@ -Clarified the conflicting advice given in the :mod:`ast` documentation about -:func:`ast.literal_eval` being "safe" for use on untrusted input while at -the same time warning that it can crash the process. The latter statement is -true and is deemed unfixable without a large amount of work unsuitable for a -bugfix. So we keep the warning and no longer claim that ``literal_eval`` is -safe. diff --git a/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst b/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst deleted file mode 100644 index 292e16998acd..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst +++ /dev/null @@ -1 +0,0 @@ -Remove extra row diff --git a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst b/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst deleted file mode 100644 index 54e421d19d9d..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst +++ /dev/null @@ -1 +0,0 @@ -Document that calling variadic functions with ctypes requires special care on macOS/arm64 (and possibly other platforms). diff --git a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst b/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst deleted file mode 100644 index e7fda8974194..000000000000 --- a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the previous bugfix that caused IDLE to not start when run with -3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed -without the Lib/test package. 3.11.0 was never affected. diff --git a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst b/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst deleted file mode 100644 index 7c9b592d6ecd..000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst +++ /dev/null @@ -1 +0,0 @@ -Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` action is given an explicit argument. diff --git a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst b/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst deleted file mode 100644 index 3810e299c78b..000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument -recursively. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst deleted file mode 100644 index d547ffc6f97e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst +++ /dev/null @@ -1 +0,0 @@ -Restrict use of sockets instead of pipes for stdin of subprocesses created by :mod:`asyncio` to AIX platform only. diff --git a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst b/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst deleted file mode 100644 index 6e2af088640b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document the optional *callback* parameter of :class:`WeakMethod`. Patch by -G?ry Ogam. diff --git a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst b/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst deleted file mode 100644 index e05d5e2a1314..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. diff --git a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst b/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst deleted file mode 100644 index 508ba626bab4..000000000000 --- a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent error when activating venv in nested fish instances. diff --git a/Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst b/Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst deleted file mode 100644 index 43c0765d940d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :mod:`asyncio` subprocess transport to kill process cleanly when process is blocked and avoid ``RuntimeError`` when loop is closed. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst b/Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst deleted file mode 100644 index 8552f51196b5..000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed flickering of the turtle window when the tracer is turned off. Patch by Shin-myoung-serp. diff --git a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst b/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst deleted file mode 100644 index 23f8cb01cf0a..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, -break reference cycles generated by local exception and future instances -(which has exception instance as its member var). Patch by Dong Uk, Kang. diff --git a/Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst b/Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst deleted file mode 100644 index c190fb7dbcb9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix handling compiler warnings (SyntaxWarning and DeprecationWarning) in -:func:`codeop.compile_command` when checking for incomplete input. -Previously it emitted warnings and raised a SyntaxError. Now it always -returns ``None`` for incomplete input without emitting any warnings. diff --git a/Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst b/Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst deleted file mode 100644 index 58e51da6ba39..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst +++ /dev/null @@ -1 +0,0 @@ -Fix handling of ``bytes`` :term:`path-like objects <path-like object>` in :func:`os.ismount()`. diff --git a/Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst b/Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst deleted file mode 100644 index 8f20588c4c58..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst +++ /dev/null @@ -1 +0,0 @@ -In :mod:`inspect`, fix overeager replacement of "``typing.``" in formatting annotations. diff --git a/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst b/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst deleted file mode 100644 index 3a35c4734871..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Work around missing socket functions in :class:`~socket.socket`'s -``__repr__``. diff --git a/Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst b/Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst deleted file mode 100644 index 232bba1b9244..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``repr`` of ``Any`` subclasses. diff --git a/Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst b/Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst deleted file mode 100644 index e7f53311e589..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst +++ /dev/null @@ -1 +0,0 @@ -Corrected type annotation for dataclass attribute ``pstats.FunctionProfile.ncalls`` to be ``str``. diff --git a/Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst b/Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst deleted file mode 100644 index 07b62a883b85..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed check in :mod:`multiprocessing.resource_tracker` that guarantees that the length of a write to a pipe is not greater than ``PIPE_BUF``. diff --git a/Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst b/Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst deleted file mode 100644 index af8ae2026f16..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst +++ /dev/null @@ -1 +0,0 @@ -Fix race condition in :mod:`asyncio` where :meth:`~asyncio.SubprocessProtocol.process_exited` called before the :meth:`~asyncio.SubprocessProtocol.pipe_data_received` leading to inconsistent output. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst b/Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst deleted file mode 100644 index d57999aa29b7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.4.9 diff --git a/Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst b/Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst deleted file mode 100644 index d8a0e690e291..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix generation of the default name of :class:`tkinter.Checkbutton`. -Previously, checkbuttons in different parent widgets could have the same -short name and share the same state if arguments "name" and "variable" are -not specified. Now they are globally unique. diff --git a/Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst b/Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst deleted file mode 100644 index 65c3105f3bc3..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``tokenize.NL`` check from :mod:`tabnanny`. diff --git a/Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst b/Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst deleted file mode 100644 index aa245cf94400..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid a crash in the C version of :meth:`asyncio.Future.remove_done_callback` when an evil argument is passed. diff --git a/Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst b/Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst deleted file mode 100644 index 159ab32ffbfc..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid spurious tracebacks from :mod:`asyncio` when default executor cleanup is delayed until after the event loop is closed (e.g. as the result of a keyboard interrupt). diff --git a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst deleted file mode 100644 index dcbea66d66bf..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would return True from this method; now they correctly return False. diff --git a/Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst b/Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst deleted file mode 100644 index 4633dce7b663..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes :exc:`AttributeError` when :meth:`subprocess.check_output` is used with argument ``input=None`` and either of the arguments *encoding* or *errors* are used. diff --git a/Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst b/Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst deleted file mode 100644 index b1350c959e2b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst +++ /dev/null @@ -1,7 +0,0 @@ -Change deprecate warning message in :mod:`unittest` from - -``It is deprecated to return a value!=None`` - -to - -``It is deprecated to return a value that is not None from a test case`` diff --git a/Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst b/Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst deleted file mode 100644 index 964aa3986331..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst +++ /dev/null @@ -1 +0,0 @@ -Earlier in 3.11 we deprecated ``asyncio.Task.cancel("message")``. We realized we were too harsh, and have undeprecated it. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst b/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst deleted file mode 100644 index f4a1d272e13b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst +++ /dev/null @@ -1 +0,0 @@ -Make sure ``patch.dict()`` can be applied on async functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst b/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst deleted file mode 100644 index 6c63fa4928c6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst +++ /dev/null @@ -1,3 +0,0 @@ -Wrap network errors consistently in urllib FTP support, so the test suite -doesn't fail when a network is available but the public internet is not -reachable. diff --git a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst b/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst deleted file mode 100644 index 700c9748735f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. diff --git a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst b/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst deleted file mode 100644 index 833a6e6bb3f7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst +++ /dev/null @@ -1,4 +0,0 @@ -On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded applications. -On macOS, the libc ``syslog()`` function is not thread-safe, so -:func:`syslog.syslog` no longer releases the GIL to call it. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst b/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst deleted file mode 100644 index 1a2b6a2537b9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` and ``pip`` when -they do not impact path resolution diff --git a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst b/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst deleted file mode 100644 index f04a0fd0915e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers -containing whitespace, underscores, plus and minus signs, or non-ASCII digits to be -incorrectly accepted. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst b/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst deleted file mode 100644 index 3fe41d53c980..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst +++ /dev/null @@ -1,2 +0,0 @@ -A :meth:`~logging.handlers.SysLogHandler.createSocket` method was added to -:class:`~logging.handlers.SysLogHandler`. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst deleted file mode 100644 index b725465ae4f0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst +++ /dev/null @@ -1,2 +0,0 @@ -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst b/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst deleted file mode 100644 index cf33db7548f6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`tkinter.Text.count` raises now an exception for options starting with -"-" instead of silently ignoring them. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst deleted file mode 100644 index f74195cc8e7d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst +++ /dev/null @@ -1 +0,0 @@ -Fix infinite loop in unittest when a self-referencing chained exception is raised diff --git a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst b/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst deleted file mode 100644 index 668ea4c7a4ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with :class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst b/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst deleted file mode 100644 index fb3a2b837fc3..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a mutex to unittest.mock.NonCallableMock to protect concurrent access -to mock attributes. diff --git a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst b/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst deleted file mode 100644 index 3107519a8714..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` -callback only once on Windows. diff --git a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst b/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst deleted file mode 100644 index 887d506d4bdd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix internal error in the :mod:`re` module which in very rare circumstances -prevented compilation of a regular expression containing a :ref:`conditional -expression <re-conditional-expression>` without the "else" branch. diff --git a/Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst b/Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst deleted file mode 100644 index cf99ea51bf7c..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prevent crashing in :mod:`traceback` when retrieving the byte-offset for -some source files that contain certain unicode characters. diff --git a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst b/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst deleted file mode 100644 index 7b67af06cf3d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst +++ /dev/null @@ -1 +0,0 @@ -Fix argument typechecks in :func:`!_overlapped.WSAConnect` and :func:`!_overlapped.Overlapped.WSASendTo` functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst b/Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst deleted file mode 100644 index 9b5deb0372f2..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst +++ /dev/null @@ -1,7 +0,0 @@ -[3.11] Applied changes from importlib_metadata `4.11.4 through 4.13 -<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-13-0>`_, -including compatibility and robustness fixes for ``Distribution`` objects -without ``_normalized_name``, disallowing invalid inputs to -``Distribution.from_name``, and refined behaviors in -``PathDistribution._name_from_stem`` and -``PathDistribution._normalized_name``. diff --git a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst b/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst deleted file mode 100644 index f61af2543c7f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in :func:`math.dist` when both points don't have the same dimension. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst deleted file mode 100644 index 4de17abd0634..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_io`. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst deleted file mode 100644 index de0006342063..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleaks on failed module initialisation in in :mod:`_pickle` diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst deleted file mode 100644 index bd54d3eae8c9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` diff --git a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst b/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst deleted file mode 100644 index d9f12a6775fe..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 22.3.1. diff --git a/Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst b/Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst deleted file mode 100644 index 6393ec614353..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst +++ /dev/null @@ -1 +0,0 @@ -``enum.auto()`` is now correctly activated when combined with other assignment values. E.g. ``ONE = auto(), 'some text'`` will now evaluate as ``(1, 'some text')``. diff --git a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst b/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst deleted file mode 100644 index a84caa6ac2ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. diff --git a/Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst b/Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst deleted file mode 100644 index 99bf1d5d08ba..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst +++ /dev/null @@ -1 +0,0 @@ -fix negative numbers failing in verify() diff --git a/Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst b/Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst deleted file mode 100644 index f0a5390b03a7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst +++ /dev/null @@ -1 +0,0 @@ -Remove older version of ``_SSLProtocolTransport.get_write_buffer_limits`` in :mod:`!asyncio.sslproto` diff --git a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst b/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst deleted file mode 100644 index 2bf05a3bdfbd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``SystemError`` in :mod:`ctypes` when exception was not set during -``__initsubclass__``. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst b/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst deleted file mode 100644 index 3102ef41f162..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :func:`inspect.signature` reported incorrect arguments for decorated methods. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst deleted file mode 100644 index 412c8c793435..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.TypeVarTuple` and -:class:`~typing.ParamSpec` together in user generics. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst deleted file mode 100644 index 1950680b1df8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.ParamSpec` followed by -:class:`~typing.TypeVarTuple` in generic aliases. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst deleted file mode 100644 index f153f2fceac8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Check the number of arguments in substitution in user generics containing a -:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst b/Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst deleted file mode 100644 index 0a06e7c5c6ac..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug in :func:`urllib.parse.urlparse` that causes URL schemes that begin -with a digit, a plus sign, or a minus sign to be parsed incorrectly. diff --git a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst b/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst deleted file mode 100644 index 451561c579da..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when -changing line numbers. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst b/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst deleted file mode 100644 index 014161cf7b1d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst +++ /dev/null @@ -1 +0,0 @@ -Release the GIL when calling termios APIs to avoid blocking threads. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst b/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst deleted file mode 100644 index f6ee449891d9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now -``addClassCleanup()`` uses separate lists for different ``TestCase`` -subclasses, and ``doClassCleanups()`` only cleans up the particular class. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst deleted file mode 100644 index 63fe7b8a3a32..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug when calling trace.CoverageResults with valid infile. diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst deleted file mode 100644 index dd4f43351ac7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`asyncio.get_event_loop` now only emits a deprecation warning when a -new event loop was created implicitly. It no longer emits a deprecation -warning if the current event loop was set. diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst deleted file mode 100644 index e305352c7a55..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst +++ /dev/null @@ -1,5 +0,0 @@ -Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so -that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex -replacement we put in or the characters r"\x" came through in the original -request line. diff --git a/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst b/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst deleted file mode 100644 index 2f113492d42d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a shell code injection vulnerability in the ``get-remote-certificate.py`` -example script. The script no longer uses a shell to run ``openssl`` commands. -Issue reported and initial fix by Caleb Shortt. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst deleted file mode 100644 index 479299e54547..000000000000 --- a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst +++ /dev/null @@ -1 +0,0 @@ -Some C API tests were moved into the new Lib/test/test_capi/ directory. diff --git a/Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst b/Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst deleted file mode 100644 index 89958c577219..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added explicit coverage of ``Py_Initialize`` (and hence ``Py_InitializeEx``) -back to the embedding tests (all other embedding tests migrated to -``Py_InitializeFromConfig`` in Python 3.11) diff --git a/Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst b/Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst deleted file mode 100644 index 181e12c7430b..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst +++ /dev/null @@ -1 +0,0 @@ -Add tests for star-unpacking with PEP 646, and some other miscellaneous PEP 646 tests. diff --git a/Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst b/Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst deleted file mode 100644 index 57e58943df49..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the :mod:`typing` tests where a test relying on CPython-specific -implementation details was not decorated with ``@cpython_only`` and was not -skipped on other implementations. diff --git a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst b/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst deleted file mode 100644 index 3db1ec12b520..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optional big memory tests in ``test_sqlite3`` now catch the correct -:exc:`sqlite.DataError` exception type in case of too large strings and/or -blobs passed. diff --git a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst deleted file mode 100644 index 1b1287c33dbd..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst +++ /dev/null @@ -1 +0,0 @@ -Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst deleted file mode 100644 index eded0361fbeb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip test_normalization() of test_unicodedata if it fails to download -NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst deleted file mode 100644 index 6b214616c0a9..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst deleted file mode 100644 index c38db3af425e..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst +++ /dev/null @@ -1,2 +0,0 @@ -The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and -test steps for building CPython on WebAssembly platforms. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst deleted file mode 100644 index 1cd1ce14fac0..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for -broken EMSDK versions, and warns about pkg-config env vars. diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst deleted file mode 100644 index 2670aeef9a25..000000000000 --- a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on -Windows. diff --git a/Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst b/Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst deleted file mode 100644 index d343173d40d6..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :file:`pcbuild.proj` to clean previous instances of ouput files in ``Python\deepfreeze`` and -``Python\frozen_modules`` directories on Windows. Patch by Charlie Zhao. diff --git a/Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst b/Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst deleted file mode 100644 index de7aff0563e5..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst +++ /dev/null @@ -1 +0,0 @@ -Update libffi to 3.4.3 diff --git a/Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst b/Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst deleted file mode 100644 index 2a6a253a52ae..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix possible crashes caused by the use of uninitialized variables when pass -invalid arguments in :func:`os.system` on Windows and in Windows-specific -modules (like ``winreg``). diff --git a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst b/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst deleted file mode 100644 index eb48ff9b6ec6..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.39.4. diff --git a/Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst b/Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst deleted file mode 100644 index 3a5efd9a1cfa..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :ref:`launcher` ignoring unrecognized shebang lines instead of -treating them as local paths diff --git a/Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst b/Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst deleted file mode 100644 index caf73db280f3..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst +++ /dev/null @@ -1 +0,0 @@ -Update :file:`py.exe` launcher to install 3.11 by default and 3.12 on request. diff --git a/Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst b/Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst deleted file mode 100644 index e8c5ac452858..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Assumes that a missing ``DLLs`` directory means that standard extension -modules are in the executable's directory. diff --git a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst b/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst deleted file mode 100644 index 295debb81369..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but -the vulnerable ``inflateGetHeader`` API is not used by Python. diff --git a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst b/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst deleted file mode 100644 index 8e19366c4297..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file -extension. diff --git a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst b/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst deleted file mode 100644 index 46cbf998eb20..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst +++ /dev/null @@ -1 +0,0 @@ -Fix initialization of :data:`sys.version` and ``sys._git`` on Windows diff --git a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst b/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst deleted file mode 100644 index 99db0c55a67e..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use faster initialization functions to detect install location for Windows -Store package diff --git a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst b/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst deleted file mode 100644 index cbef54d17a82..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.39.4. diff --git a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst b/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst deleted file mode 100644 index 18ef0b0e2523..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``Mac/Extras.install.py`` file filter bug. diff --git a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst b/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst deleted file mode 100644 index 3111e4975e87..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst +++ /dev/null @@ -1 +0,0 @@ -On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script longer than a couple of bytes. diff --git a/README.rst b/README.rst index 335041972059..8d9ace93b6c9 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.11.0 +This is Python version 3.11.1 ============================= .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From webhook-mailer at python.org Tue Dec 6 16:19:03 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 21:19:03 -0000 Subject: [Python-checkins] Python 3.10.9 Message-ID: <mailman.2832.1670361544.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1dd9be6584413fbfa823f37a224f101b819505d1 commit: 1dd9be6584413fbfa823f37a224f101b819505d1 branch: 3.10 author: Pablo Galindo <pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T18:31:21Z summary: Python 3.10.9 files: A Misc/NEWS.d/3.10.9.rst D Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst D Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst D Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst D Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst D Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst D Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst D Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst D Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst D Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst D Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst D Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst D Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst D Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst D Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst D Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst D Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst D Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst D Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst D Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst D Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst D Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst D Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst D Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst D Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst D Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst D Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst D Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst D Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst D Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst D Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst D Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst D Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst D Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst D Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst D Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst D Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst D Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst D Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst D Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst D Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst D Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst D Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst D Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst D Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst D Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst D Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst D Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst D Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst D Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 6db63f934c16..48a34d0ef5fc 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 10 -#define PY_MICRO_VERSION 8 +#define PY_MICRO_VERSION 9 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.10.8+" +#define PY_VERSION "3.10.9" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 34bb19eaf50c..e8bedaae60be 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Oct 11 12:21:26 2022 +# Autogenerated by Sphinx on Tue Dec 6 18:31:02 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -358,7 +358,7 @@ 'yield_expression)]\n' '\n' 'The difference from normal Assignment statements is that only ' - 'single\n' + 'a single\n' 'target is allowed.\n' '\n' 'For simple names as assignment targets, if in class or module ' @@ -408,12 +408,13 @@ 'analysis\n' ' tools and IDEs.\n' '\n' - 'Changed in version 3.8: Now annotated assignments allow same\n' - 'expressions in the right hand side as the regular ' - 'assignments.\n' - 'Previously, some expressions (like un-parenthesized tuple ' - 'expressions)\n' - 'caused a syntax error.\n', + 'Changed in version 3.8: Now annotated assignments allow the ' + 'same\n' + 'expressions in the right hand side as regular assignments. ' + 'Previously,\n' + 'some expressions (like un-parenthesized tuple expressions) ' + 'caused a\n' + 'syntax error.\n', 'async': 'Coroutines\n' '**********\n' '\n' @@ -2041,7 +2042,7 @@ '\n' '* Mappings (instances of "dict") compare equal if and only if ' 'they\n' - ' have equal *(key, value)* pairs. Equality comparison of the ' + ' have equal "(key, value)" pairs. Equality comparison of the ' 'keys and\n' ' values enforces reflexivity.\n' '\n' @@ -7225,7 +7226,7 @@ 'the clauses had been separated out into individual import ' 'statements.\n' '\n' - 'The details of the first step, finding and loading modules are\n' + 'The details of the first step, finding and loading modules, are\n' 'described in greater detail in the section on the import system, ' 'which\n' 'also describes the various types of packages and modules that can ' @@ -10999,8 +11000,9 @@ 'y)" is\n' 'typically invalid without special support in "MyClass". To ' 'be able to\n' - 'use that kind of patterns, the class needs to define a\n' - '*__match_args__* attribute.\n' + 'use that kind of pattern, the class needs to define a ' + '*__match_args__*\n' + 'attribute.\n' '\n' 'object.__match_args__\n' '\n' @@ -11205,6 +11207,10 @@ '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' + ' If *sub* is empty, returns the number of empty strings ' + 'between\n' + ' characters which is the length of the string plus one.\n' + '\n' "str.encode(encoding='utf-8', errors='strict')\n" '\n' ' Return an encoded version of the string as a bytes ' @@ -11711,7 +11717,7 @@ 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11752,7 +11758,7 @@ " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -13733,17 +13739,11 @@ 'dictionaries or\n' 'other mutable types (that are compared by value rather than ' 'by object\n' - 'identity) may not be used as keys. Numeric types used for ' - 'keys obey\n' - 'the normal rules for numeric comparison: if two numbers ' - 'compare equal\n' - '(such as "1" and "1.0") then they can be used ' - 'interchangeably to index\n' - 'the same dictionary entry. (Note however, that since ' - 'computers store\n' - 'floating-point numbers as approximations it is usually ' - 'unwise to use\n' - 'them as dictionary keys.)\n' + 'identity) may not be used as keys. Values that compare equal ' + '(such as\n' + '"1", "1.0", and "True") can be used interchangeably to index ' + 'the same\n' + 'dictionary entry.\n' '\n' 'class dict(**kwargs)\n' 'class dict(mapping, **kwargs)\n' diff --git a/Misc/NEWS.d/3.10.9.rst b/Misc/NEWS.d/3.10.9.rst new file mode 100644 index 000000000000..7dbc291d857a --- /dev/null +++ b/Misc/NEWS.d/3.10.9.rst @@ -0,0 +1,656 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-09-07-10-42-00 +.. gh-issue: 97514 +.. nonce: Yggdsl +.. section: Security + +On Linux the :mod:`multiprocessing` module returns to using filesystem +backed unix domain sockets for communication with the *forkserver* process +instead of the Linux abstract socket namespace. Only code that chooses to +use the :ref:`"forkserver" start method <multiprocessing-start-methods>` is +affected. + +Abstract sockets have no permissions and could allow any user on the system +in the same `network namespace +<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often +the whole system) to inject code into the multiprocessing *forkserver* +process. This was a potential privilege escalation. Filesystem based socket +permissions restrict this to the *forkserver* process user as was the +default in Python 3.8 and earlier. + +This prevents Linux `CVE-2022-42919 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. + +.. + +.. date: 2022-11-21-11-27-14 +.. gh-issue: 99578 +.. nonce: DcKoBJ +.. section: Core and Builtins + +Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by +Victor Stinner. + +.. + +.. date: 2022-11-19-22-27-52 +.. gh-issue: 99581 +.. nonce: yKYPbf +.. section: Core and Builtins + +Fixed a bug that was causing a buffer overflow if the tokenizer copies a +line missing the newline caracter from a file that is as long as the +available tokenizer buffer. Patch by Pablo galindo + +.. + +.. date: 2022-11-06-22-59-02 +.. gh-issue: 96055 +.. nonce: TmQuJn +.. section: Core and Builtins + +Update :mod:`faulthandler` to emit an error message with the proper +unexpected signal number. Patch by Dong-hee Na. + +.. + +.. date: 2022-10-31-21-01-35 +.. gh-issue: 98852 +.. nonce: MYaRN6 +.. section: Core and Builtins + +Fix subscription of :class:`types.GenericAlias` instances containing bare +generic types: for example ``tuple[A, T][int]``, where ``A`` is a generic +type, and ``T`` is a type variable. + +.. + +.. date: 2022-10-19-01-01-08 +.. gh-issue: 98415 +.. nonce: ZS2eWh +.. section: Core and Builtins + +Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by +Chaim Sanders + +.. + +.. date: 2022-10-15-23-15-14 +.. gh-issue: 92119 +.. nonce: PMSwwG +.. section: Core and Builtins + +Print exception class name instead of its string representation when raising +errors from :mod:`ctypes` calls. + +.. + +.. date: 2022-06-10-16-37-44 +.. gh-issue: 93696 +.. nonce: 65BI2R +.. section: Core and Builtins + +Allow :mod:`pdb` to locate source for frozen modules in the standard +library. + +.. + +.. bpo: 31718 +.. date: 2020-02-23-23-48-15 +.. nonce: sXko5e +.. section: Core and Builtins + +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. + +.. + +.. bpo: 38031 +.. date: 2019-09-04-19-09-49 +.. nonce: Yq4L72 +.. section: Core and Builtins + +Fix a possible assertion failure in :class:`io.FileIO` when the opener +returns an invalid file descriptor. + +.. + +.. date: 2022-12-05-13-40-15 +.. gh-issue: 100001 +.. nonce: 78ReYp +.. section: Library + +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. + +.. + +.. date: 2022-12-02-13-05-00 +.. gh-issue: 93453 +.. nonce: EFj1NN +.. section: Library + +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. + +.. + +.. date: 2022-11-21-17-56-18 +.. gh-issue: 51524 +.. nonce: nTykx8 +.. section: Library + +Fix bug when calling trace.CoverageResults with valid infile. + +.. + +.. date: 2022-11-21-13-49-03 +.. gh-issue: 99645 +.. nonce: 9w1QKq +.. section: Library + +Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now +``addClassCleanup()`` uses separate lists for different ``TestCase`` +subclasses, and ``doClassCleanups()`` only cleans up the particular class. + +.. + +.. date: 2022-11-15-10-55-24 +.. gh-issue: 97001 +.. nonce: KeQuVF +.. section: Library + +Release the GIL when calling termios APIs to avoid blocking threads. + +.. + +.. date: 2022-11-13-02-06-56 +.. gh-issue: 99341 +.. nonce: 8-OlwB +.. section: Library + +Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when +changing line numbers. + +.. + +.. date: 2022-11-09-20-48-42 +.. gh-issue: 74044 +.. nonce: zBj26K +.. section: Library + +Fixed bug where :func:`inspect.signature` reported incorrect arguments for +decorated methods. + +.. + +.. date: 2022-11-09-12-16-35 +.. gh-issue: 99275 +.. nonce: klOqoL +.. section: Library + +Fix ``SystemError`` in :mod:`ctypes` when exception was not set during +``__initsubclass__``. + +.. + +.. date: 2022-11-06-12-44-51 +.. gh-issue: 99155 +.. nonce: vLZOzi +.. section: Library + +Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. + +.. + +.. date: 2022-11-05-17-16-40 +.. gh-issue: 99134 +.. nonce: Msgspf +.. section: Library + +Update the bundled copy of pip to version 22.3.1. + +.. + +.. date: 2022-11-05-11-42-15 +.. gh-issue: 99130 +.. nonce: 91MMXu +.. section: Library + +Apply bugfixes from `importlib_metadata 4.11.4 +<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-11-4>`_, +namely: In ``PathDistribution._name_from_stem``, avoid including parts of +the extension in the result. In ``PathDistribution._normalized_name``, +ensure names loaded from the stem of the filename are also normalized, +ensuring duplicate entry points by packages varying only by non-normalized +name are hidden. + +.. + +.. date: 2022-11-02-05-54-02 +.. gh-issue: 83004 +.. nonce: 0v8iyw +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_zoneinfo` + +.. + +.. date: 2022-11-02-05-53-25 +.. gh-issue: 83004 +.. nonce: qc_KHr +.. section: Library + +Clean up refleaks on failed module initialisation in in :mod:`_pickle` + +.. + +.. date: 2022-11-02-05-52-36 +.. gh-issue: 83004 +.. nonce: LBl79O +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_io`. + +.. + +.. date: 2022-10-31-12-34-03 +.. gh-issue: 98897 +.. nonce: rgNn4x +.. section: Library + +Fix memory leak in :func:`math.dist` when both points don't have the same +dimension. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-29-03-40-18 +.. gh-issue: 98793 +.. nonce: WSPB4A +.. section: Library + +Fix argument typechecks in :func:`!_overlapped.WSAConnect` and +:func:`!_overlapped.Overlapped.WSASendTo` functions. + +.. + +.. date: 2022-10-27-12-56-38 +.. gh-issue: 98740 +.. nonce: ZoqqGM +.. section: Library + +Fix internal error in the :mod:`re` module which in very rare circumstances +prevented compilation of a regular expression containing a :ref:`conditional +expression <re-conditional-expression>` without the "else" branch. + +.. + +.. date: 2022-10-26-07-51-55 +.. gh-issue: 98703 +.. nonce: 0hW773 +.. section: Library + +Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` +callback only once on Windows. + +.. + +.. date: 2022-10-25-20-17-34 +.. gh-issue: 98624 +.. nonce: YQUPFy +.. section: Library + +Add a mutex to unittest.mock.NonCallableMock to protect concurrent access to +mock attributes. + +.. + +.. date: 2022-10-23-18-30-39 +.. gh-issue: 89237 +.. nonce: kBui30 +.. section: Library + +Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with +:class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-19-18-31-53 +.. gh-issue: 98458 +.. nonce: vwyq7O +.. section: Library + +Fix infinite loop in unittest when a self-referencing chained exception is +raised + +.. + +.. date: 2022-10-19-09-29-12 +.. gh-issue: 97928 +.. nonce: xj3im7 +.. section: Library + +:meth:`tkinter.Text.count` raises now an exception for options starting with +"-" instead of silently ignoring them. + +.. + +.. date: 2022-10-16-18-52-00 +.. gh-issue: 97966 +.. nonce: humlhz +.. section: Library + +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. + +.. + +.. date: 2022-10-16-15-31-50 +.. gh-issue: 98331 +.. nonce: Y5kPOX +.. section: Library + +Update the bundled copies of pip and setuptools to versions 22.3 and 65.5.0 +respectively. + +.. + +.. date: 2022-10-14-19-57-37 +.. gh-issue: 96035 +.. nonce: 0xcX-p +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers +containing whitespace, underscores, plus and minus signs, or non-ASCII +digits to be incorrectly accepted. + +.. + +.. date: 2022-10-14-11-46-31 +.. gh-issue: 98251 +.. nonce: Uxc9al +.. section: Library + +Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +and ``pip`` when they do not impact path resolution + +.. + +.. date: 2022-10-12-10-00-40 +.. gh-issue: 98178 +.. nonce: hspH51 +.. section: Library + +On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded +applications. On macOS, the libc ``syslog()`` function is not thread-safe, +so :func:`syslog.syslog` no longer releases the GIL to call it. Patch by +Victor Stinner. + +.. + +.. date: 2022-10-10-07-07-31 +.. gh-issue: 96151 +.. nonce: K9fwoq +.. section: Library + +Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. + +.. + +.. date: 2022-10-08-19-39-27 +.. gh-issue: 98086 +.. nonce: y---WC +.. section: Library + +Make sure ``patch.dict()`` can be applied on async functions. + +.. + +.. date: 2022-08-06-12-18-07 +.. gh-issue: 88863 +.. nonce: NnqsuJ +.. section: Library + +To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, +break reference cycles generated by local exception and future instances +(which has exception instance as its member var). Patch by Dong Uk, Kang. + +.. + +.. date: 2022-06-17-12-02-30 +.. gh-issue: 93858 +.. nonce: R49ARc +.. section: Library + +Prevent error when activating venv in nested fish instances. + +.. + +.. bpo: 46364 +.. date: 2022-01-14-10-49-20 +.. nonce: SzhlU9 +.. section: Library + +Restrict use of sockets instead of pipes for stdin of subprocesses created +by :mod:`asyncio` to AIX platform only. + +.. + +.. bpo: 38523 +.. date: 2020-10-23-22-20-52 +.. nonce: CrkxLh +.. section: Library + +:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument +recursively. + +.. + +.. bpo: 36267 +.. date: 2019-09-03-15-45-19 +.. nonce: z42Ex7 +.. section: Library + +Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` +action is given an explicit argument. + +.. + +.. date: 2022-11-16-12-52-23 +.. gh-issue: 92892 +.. nonce: TS-P0j +.. section: Documentation + +Document that calling variadic functions with ctypes requires special care +on macOS/arm64 (and possibly other platforms). + +.. + +.. date: 2022-12-05-16-12-56 +.. gh-issue: 99892 +.. nonce: sz_eW8 +.. section: Tests + +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. + +.. + +.. bpo: 34272 +.. date: 2018-07-29-15-59-51 +.. nonce: lVX2uR +.. section: Tests + +Some C API tests were moved into the new Lib/test/test_capi/ directory. + +.. + +.. date: 2022-11-24-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and +``-Wimplicit-function-declaration`` compiler warnings in +:program:`configure` checks. + +.. + +.. date: 2022-11-04-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for +``PTHREAD_SCOPE_SYSTEM``. + +.. + +.. date: 2022-11-02-18-45-35 +.. gh-issue: 97731 +.. nonce: zKpTlj +.. section: Build + +Specify the full path to the source location for ``make docclean`` (needed +for cross-builds). + +.. + +.. date: 2022-10-25-14-43-00 +.. gh-issue: 98671 +.. nonce: a42a6d +.. section: Build + +Fix ``NO_MISALIGNED_ACCESSES`` being not defined for the SHA3 extension when +``HAVE_ALIGNED_REQUIRED`` is set. Allowing builds on hardware that unaligned +memory accesses are not allowed. + +.. + +.. date: 2022-11-23-17-17-16 +.. gh-issue: 99345 +.. nonce: jOa3-f +.. section: Windows + +Use faster initialization functions to detect install location for Windows +Store package + +.. + +.. date: 2022-11-01-11-07-33 +.. gh-issue: 98689 +.. nonce: 0f6e_N +.. section: Windows + +Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +vulnerable ``inflateGetHeader`` API is not used by Python. + +.. + +.. date: 2022-10-25-10-34-17 +.. gh-issue: 94328 +.. nonce: 19NhdU +.. section: Windows + +Update Windows installer to use SQLite 3.39.4. + +.. + +.. bpo: 40882 +.. date: 2020-06-06-15-10-37 +.. nonce: UvNbdj +.. section: Windows + +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. + +.. + +.. date: 2022-10-25-10-32-23 +.. gh-issue: 94328 +.. nonce: W3YNC_ +.. section: macOS + +Update macOS installer to SQLite 3.39.4. + +.. + +.. date: 2022-10-15-21-20-40 +.. gh-issue: 97527 +.. nonce: otAHJM +.. section: IDLE + +Fix a bug in the previous bugfix that caused IDLE to not start when run with +3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed +without the Lib/test package. 3.11.0 was never affected. + +.. + +.. date: 2022-08-05-23-25-59 +.. gh-issue: 95731 +.. nonce: N2KohU +.. section: Tools/Demos + +Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. diff --git a/Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst b/Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst deleted file mode 100644 index 4602d53fcdb7..000000000000 --- a/Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``NO_MISALIGNED_ACCESSES`` being not defined for the SHA3 extension -when ``HAVE_ALIGNED_REQUIRED`` is set. Allowing builds on hardware that -unaligned memory accesses are not allowed. diff --git a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst b/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst deleted file mode 100644 index 46b1fb833d47..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specify the full path to the source location for ``make docclean`` (needed for -cross-builds). diff --git a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index e320ecfdfbb7..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for ``PTHREAD_SCOPE_SYSTEM``. diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index 2dace165ca1a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst deleted file mode 100644 index b5964375962f..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure in :class:`io.FileIO` when the opener -returns an invalid file descriptor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst deleted file mode 100644 index dd96c9e20d87..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of -uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. -Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst deleted file mode 100644 index 8eadab0ad8fb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst +++ /dev/null @@ -1 +0,0 @@ -Allow :mod:`pdb` to locate source for frozen modules in the standard library. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst deleted file mode 100644 index 7142fc619765..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Print exception class name instead of its string representation when raising -errors from :mod:`ctypes` calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst deleted file mode 100644 index af2db1f9965c..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst +++ /dev/null @@ -1 +0,0 @@ -Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by Chaim Sanders diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst deleted file mode 100644 index 25c473717ca2..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix subscription of :class:`types.GenericAlias` instances containing bare -generic types: for example ``tuple[A, T][int]``, -where ``A`` is a generic type, and ``T`` is a type variable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst deleted file mode 100644 index c72fb21942e6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`faulthandler` to emit an error message with the proper -unexpected signal number. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst deleted file mode 100644 index 8071fd130dd6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the -available tokenizer buffer. Patch by Pablo galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst deleted file mode 100644 index 9321cef77eed..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of the -first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst b/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst deleted file mode 100644 index 54e421d19d9d..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst +++ /dev/null @@ -1 +0,0 @@ -Document that calling variadic functions with ctypes requires special care on macOS/arm64 (and possibly other platforms). diff --git a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst b/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst deleted file mode 100644 index e7fda8974194..000000000000 --- a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the previous bugfix that caused IDLE to not start when run with -3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed -without the Lib/test package. 3.11.0 was never affected. diff --git a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst b/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst deleted file mode 100644 index 7c9b592d6ecd..000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst +++ /dev/null @@ -1 +0,0 @@ -Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` action is given an explicit argument. diff --git a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst b/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst deleted file mode 100644 index 3810e299c78b..000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument -recursively. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst deleted file mode 100644 index d547ffc6f97e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst +++ /dev/null @@ -1 +0,0 @@ -Restrict use of sockets instead of pipes for stdin of subprocesses created by :mod:`asyncio` to AIX platform only. diff --git a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst b/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst deleted file mode 100644 index 508ba626bab4..000000000000 --- a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent error when activating venv in nested fish instances. diff --git a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst b/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst deleted file mode 100644 index 23f8cb01cf0a..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, -break reference cycles generated by local exception and future instances -(which has exception instance as its member var). Patch by Dong Uk, Kang. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst b/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst deleted file mode 100644 index f4a1d272e13b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst +++ /dev/null @@ -1 +0,0 @@ -Make sure ``patch.dict()`` can be applied on async functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst b/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst deleted file mode 100644 index 700c9748735f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. diff --git a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst b/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst deleted file mode 100644 index 833a6e6bb3f7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst +++ /dev/null @@ -1,4 +0,0 @@ -On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded applications. -On macOS, the libc ``syslog()`` function is not thread-safe, so -:func:`syslog.syslog` no longer releases the GIL to call it. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst b/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst deleted file mode 100644 index 1a2b6a2537b9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` and ``pip`` when -they do not impact path resolution diff --git a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst b/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst deleted file mode 100644 index f04a0fd0915e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers -containing whitespace, underscores, plus and minus signs, or non-ASCII digits to be -incorrectly accepted. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst b/Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst deleted file mode 100644 index b4cf94310af6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copies of pip and setuptools to versions 22.3 and 65.5.0 respectively. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst deleted file mode 100644 index b725465ae4f0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst +++ /dev/null @@ -1,2 +0,0 @@ -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst b/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst deleted file mode 100644 index cf33db7548f6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`tkinter.Text.count` raises now an exception for options starting with -"-" instead of silently ignoring them. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst deleted file mode 100644 index f74195cc8e7d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst +++ /dev/null @@ -1 +0,0 @@ -Fix infinite loop in unittest when a self-referencing chained exception is raised diff --git a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst b/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst deleted file mode 100644 index 668ea4c7a4ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with :class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst b/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst deleted file mode 100644 index fb3a2b837fc3..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a mutex to unittest.mock.NonCallableMock to protect concurrent access -to mock attributes. diff --git a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst b/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst deleted file mode 100644 index 3107519a8714..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` -callback only once on Windows. diff --git a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst b/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst deleted file mode 100644 index 887d506d4bdd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix internal error in the :mod:`re` module which in very rare circumstances -prevented compilation of a regular expression containing a :ref:`conditional -expression <re-conditional-expression>` without the "else" branch. diff --git a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst b/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst deleted file mode 100644 index 7b67af06cf3d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst +++ /dev/null @@ -1 +0,0 @@ -Fix argument typechecks in :func:`!_overlapped.WSAConnect` and :func:`!_overlapped.Overlapped.WSASendTo` functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst b/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst deleted file mode 100644 index f61af2543c7f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in :func:`math.dist` when both points don't have the same dimension. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst deleted file mode 100644 index 4de17abd0634..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_io`. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst deleted file mode 100644 index de0006342063..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleaks on failed module initialisation in in :mod:`_pickle` diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst deleted file mode 100644 index bd54d3eae8c9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` diff --git a/Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst b/Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst deleted file mode 100644 index 124d5e478f8f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst +++ /dev/null @@ -1,7 +0,0 @@ -Apply bugfixes from `importlib_metadata 4.11.4 -<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-11-4>`_, -namely: In ``PathDistribution._name_from_stem``, avoid including parts of -the extension in the result. In ``PathDistribution._normalized_name``, -ensure names loaded from the stem of the filename are also normalized, -ensuring duplicate entry points by packages varying only by non-normalized -name are hidden. diff --git a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst b/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst deleted file mode 100644 index d9f12a6775fe..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 22.3.1. diff --git a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst b/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst deleted file mode 100644 index a84caa6ac2ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst b/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst deleted file mode 100644 index 2bf05a3bdfbd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``SystemError`` in :mod:`ctypes` when exception was not set during -``__initsubclass__``. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst b/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst deleted file mode 100644 index 3102ef41f162..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :func:`inspect.signature` reported incorrect arguments for decorated methods. diff --git a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst b/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst deleted file mode 100644 index 451561c579da..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when -changing line numbers. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst b/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst deleted file mode 100644 index 014161cf7b1d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst +++ /dev/null @@ -1 +0,0 @@ -Release the GIL when calling termios APIs to avoid blocking threads. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst b/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst deleted file mode 100644 index f6ee449891d9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now -``addClassCleanup()`` uses separate lists for different ``TestCase`` -subclasses, and ``doClassCleanups()`` only cleans up the particular class. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst deleted file mode 100644 index 63fe7b8a3a32..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug when calling trace.CoverageResults with valid infile. diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst deleted file mode 100644 index dd4f43351ac7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`asyncio.get_event_loop` now only emits a deprecation warning when a -new event loop was created implicitly. It no longer emits a deprecation -warning if the current event loop was set. diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst deleted file mode 100644 index e305352c7a55..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst +++ /dev/null @@ -1,5 +0,0 @@ -Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so -that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex -replacement we put in or the characters r"\x" came through in the original -request line. diff --git a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst deleted file mode 100644 index 02d95b570520..000000000000 --- a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst +++ /dev/null @@ -1,15 +0,0 @@ -On Linux the :mod:`multiprocessing` module returns to using filesystem backed -unix domain sockets for communication with the *forkserver* process instead of -the Linux abstract socket namespace. Only code that chooses to use the -:ref:`"forkserver" start method <multiprocessing-start-methods>` is affected. - -Abstract sockets have no permissions and could allow any user on the system in -the same `network namespace -<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often the -whole system) to inject code into the multiprocessing *forkserver* process. -This was a potential privilege escalation. Filesystem based socket permissions -restrict this to the *forkserver* process user as was the default in Python 3.8 -and earlier. - -This prevents Linux `CVE-2022-42919 -<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst deleted file mode 100644 index 479299e54547..000000000000 --- a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst +++ /dev/null @@ -1 +0,0 @@ -Some C API tests were moved into the new Lib/test/test_capi/ directory. diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst deleted file mode 100644 index eded0361fbeb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip test_normalization() of test_unicodedata if it fails to download -NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst deleted file mode 100644 index 6b214616c0a9..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst deleted file mode 100644 index 2670aeef9a25..000000000000 --- a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on -Windows. diff --git a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst b/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst deleted file mode 100644 index eb48ff9b6ec6..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.39.4. diff --git a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst b/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst deleted file mode 100644 index 295debb81369..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but -the vulnerable ``inflateGetHeader`` API is not used by Python. diff --git a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst b/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst deleted file mode 100644 index 99db0c55a67e..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use faster initialization functions to detect install location for Windows -Store package diff --git a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst b/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst deleted file mode 100644 index cbef54d17a82..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.39.4. diff --git a/README.rst b/README.rst index 986940dbb5d8..4e40478cf9af 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.10.8 +This is Python version 3.10.9 ============================= .. image:: https://travis-ci.com/python/cpython.svg?branch=master From webhook-mailer at python.org Tue Dec 6 16:24:09 2022 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 06 Dec 2022 21:24:09 -0000 Subject: [Python-checkins] Python 3.12.0a3 Message-ID: <mailman.2833.1670361850.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b6bd7ffcbc1ffaa68e3423e7415ef8ba0f8a188d commit: b6bd7ffcbc1ffaa68e3423e7415ef8ba0f8a188d branch: main author: Thomas Wouters <thomas at python.org> committer: Yhg1s <thomas at python.org> date: 2022-12-06T19:33:02+01:00 summary: Python 3.12.0a3 files: A Misc/NEWS.d/3.12.0a3.rst D Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst D Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst D Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst D Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst D Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst D Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst D Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst D Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst D Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst D Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst D Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst D Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst D Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst D Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst D Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst D Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst D Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst D Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst D Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst D Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst D Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst D Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst D Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst D Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst D Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst D Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst D Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst D Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst D Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst D Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst D Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst D Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst D Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst D Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst D Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst D Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst D Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst D Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst D Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst D Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst D Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst D Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst D Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst D Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst D Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst D Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst D Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst D Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst D Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst D Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst D Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst D Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst D Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst D Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst D Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst D Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst D Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst D Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst D Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst D Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index a16b8d7104e3..1ae4889c7573 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.12.0a2+" +#define PY_VERSION "3.12.0a3" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index a817dc3547fa..4ba680cceda9 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Nov 14 12:13:19 2022 +# Autogenerated by Sphinx on Tue Dec 6 19:31:49 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -11109,8 +11109,9 @@ 'y)" is\n' 'typically invalid without special support in "MyClass". To ' 'be able to\n' - 'use that kind of patterns, the class needs to define a\n' - '*__match_args__* attribute.\n' + 'use that kind of pattern, the class needs to define a ' + '*__match_args__*\n' + 'attribute.\n' '\n' 'object.__match_args__\n' '\n' @@ -11510,7 +11511,7 @@ 'property\n' ' being one of ?Lm?, ?Lt?, ?Lu?, ?Ll?, or ?Lo?. Note ' 'that this is\n' - ' different from the ?Alphabetic? property defined in the ' + ' different from the Alphabetic property defined in the ' 'Unicode\n' ' Standard.\n' '\n' @@ -11559,9 +11560,9 @@ 'according to the\n' ' language definition, section Identifiers and keywords.\n' '\n' - ' Call "keyword.iskeyword()" to test whether string "s" ' - 'is a reserved\n' - ' identifier, such as "def" and "class".\n' + ' "keyword.iskeyword()" can be used to test whether ' + 'string "s" is a\n' + ' reserved identifier, such as "def" and "class".\n' '\n' ' Example:\n' '\n' diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst new file mode 100644 index 000000000000..3d1e43350d13 --- /dev/null +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -0,0 +1,836 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-30-11-09-40 +.. gh-issue: 99891 +.. nonce: 9VomwB +.. section: Core and Builtins + +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo + +.. + +.. date: 2022-11-27-13-50-13 +.. gh-issue: 91054 +.. nonce: oox_kW +.. section: Core and Builtins + +Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to +register callbacks to receive notification on creation and destruction of +code objects. + +.. + +.. date: 2022-11-26-04-00-41 +.. gh-issue: 99729 +.. nonce: A3ovwQ +.. section: Core and Builtins + +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. + +.. + +.. date: 2022-11-23-18-16-18 +.. gh-issue: 99708 +.. nonce: 7MuaiR +.. section: Core and Builtins + +Fix bug where compiler crashes on an if expression with an empty body block. + +.. + +.. date: 2022-11-21-11-27-14 +.. gh-issue: 99578 +.. nonce: DcKoBJ +.. section: Core and Builtins + +Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by +Victor Stinner. + +.. + +.. date: 2022-11-19-22-27-52 +.. gh-issue: 99581 +.. nonce: yKYPbf +.. section: Core and Builtins + +Fixed a bug that was causing a buffer overflow if the tokenizer copies a +line missing the newline caracter from a file that is as long as the +available tokenizer buffer. Patch by Pablo galindo + +.. + +.. date: 2022-11-18-11-24-25 +.. gh-issue: 99553 +.. nonce: F64h-n +.. section: Core and Builtins + +Fix bug where an :exc:`ExceptionGroup` subclass can wrap a +:exc:`BaseException`. + +.. + +.. date: 2022-11-16-21-35-30 +.. gh-issue: 99547 +.. nonce: p_c_bp +.. section: Core and Builtins + +Add a function to os.path to check if a path is a junction: isjunction. Add +similar functionality to pathlib.Path as is_junction. + +.. + +.. date: 2022-11-12-01-39-57 +.. gh-issue: 99370 +.. nonce: _cu32j +.. section: Core and Builtins + +Fix zip path for venv created from a non-installed python on POSIX +platforms. + +.. + +.. date: 2022-11-11-14-04-01 +.. gh-issue: 99377 +.. nonce: -CJvWn +.. section: Core and Builtins + +Add audit events for thread creation and clear operations. + +.. + +.. date: 2022-11-10-17-09-16 +.. gh-issue: 98686 +.. nonce: bmAKwr +.. section: Core and Builtins + +Remove the ``BINARY_OP_GENERIC`` and ``COMPARE_OP_GENERIC`` +"specializations". + +.. + +.. date: 2022-11-10-16-53-40 +.. gh-issue: 99298 +.. nonce: HqRJES +.. section: Core and Builtins + +Remove the remaining error paths for attribute specializations, and refuse +to specialize attribute accesses on types that haven't had +:c:func:`PyType_Ready` called on them yet. + +.. + +.. date: 2022-11-05-22-26-35 +.. gh-issue: 99127 +.. nonce: Btk7ih +.. section: Core and Builtins + +Allow some features of :mod:`syslog` to the main interpreter only. Patch by +Dong-hee Na. + +.. + +.. date: 2022-10-05-11-44-52 +.. gh-issue: 91053 +.. nonce: f5Bo3p +.. section: Core and Builtins + +Optimizing interpreters and JIT compilers may need to invalidate internal +metadata when functions are modified. This change adds the ability to +provide a callback that will be invoked each time a function is created, +modified, or destroyed. + +.. + +.. date: 2022-09-17-17-08-01 +.. gh-issue: 90994 +.. nonce: f0H2Yd +.. section: Core and Builtins + +Improve error messages when there's a syntax error with call arguments. The +following three cases are covered: - No value is assigned to a named +argument, eg ``foo(a=)``. - A value is assigned to a star argument, eg +``foo(*args=[0])``. - A value is assigned to a double-star keyword argument, +eg ``foo(**kwarg={'a': 0})``. + +.. + +.. bpo: 45026 +.. date: 2021-08-29-15-55-19 +.. nonce: z7nTA3 +.. section: Core and Builtins + +Optimize the :class:`range` object iterator. It is now smaller, faster +iteration of ranges containing large numbers. Smaller pickles, faster +unpickling. + +.. + +.. bpo: 31718 +.. date: 2020-02-23-23-48-15 +.. nonce: sXko5e +.. section: Core and Builtins + +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. + +.. + +.. bpo: 38031 +.. date: 2019-09-04-19-09-49 +.. nonce: Yq4L72 +.. section: Core and Builtins + +Fix a possible assertion failure in :class:`io.FileIO` when the opener +returns an invalid file descriptor. + +.. + +.. date: 2022-12-05-13-40-15 +.. gh-issue: 100001 +.. nonce: 78ReYp +.. section: Library + +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. + +.. + +.. date: 2022-12-03-05-58-48 +.. gh-issue: 99957 +.. nonce: jLYYgN +.. section: Library + +Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. + +.. + +.. date: 2022-11-22-19-31-26 +.. gh-issue: 79033 +.. nonce: MW6kHq +.. section: Library + +Fix :func:`asyncio.Server.wait_closed` to actually do what the docs promise +-- wait for all existing connections to complete, after closing the server. + +.. + +.. date: 2022-11-21-17-56-18 +.. gh-issue: 51524 +.. nonce: nTykx8 +.. section: Library + +Fix bug when calling trace.CoverageResults with valid infile. + +.. + +.. date: 2022-11-21-13-49-03 +.. gh-issue: 99645 +.. nonce: 9w1QKq +.. section: Library + +Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now +``addClassCleanup()`` uses separate lists for different ``TestCase`` +subclasses, and ``doClassCleanups()`` only cleans up the particular class. + +.. + +.. date: 2022-11-21-10-45-54 +.. gh-issue: 99508 +.. nonce: QqVbby +.. section: Library + +Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling +``_imp.source_hash()``. + +.. + +.. date: 2022-11-17-10-56-47 +.. gh-issue: 66285 +.. nonce: KvjlaB +.. section: Library + +Fix :mod:`asyncio` to not share event loop and signal wakeupfd in forked +processes. Patch by Kumar Aditya. + +.. + +.. date: 2022-11-15-10-55-24 +.. gh-issue: 97001 +.. nonce: KeQuVF +.. section: Library + +Release the GIL when calling termios APIs to avoid blocking threads. + +.. + +.. date: 2022-11-15-04-08-25 +.. gh-issue: 92647 +.. nonce: cZcjnJ +.. section: Library + +Use final status of an enum to determine lookup or creation branch of +functional API. + +.. + +.. date: 2022-11-14-08-21-56 +.. gh-issue: 99388 +.. nonce: UWSlwp +.. section: Library + +Add *loop_factory* parameter to :func:`asyncio.run` to allow specifying a +custom event loop factory. Patch by Kumar Aditya. + +.. + +.. date: 2022-11-13-02-06-56 +.. gh-issue: 99341 +.. nonce: 8-OlwB +.. section: Library + +Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when +changing line numbers. + +.. + +.. date: 2022-11-12-12-15-30 +.. gh-issue: 99382 +.. nonce: dKg_rW +.. section: Library + +Check the number of arguments in substitution in user generics containing a +:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. + +.. + +.. date: 2022-11-12-12-10-23 +.. gh-issue: 99379 +.. nonce: bcGhxF +.. section: Library + +Fix substitution of :class:`~typing.ParamSpec` followed by +:class:`~typing.TypeVarTuple` in generic aliases. + +.. + +.. date: 2022-11-12-12-08-34 +.. gh-issue: 99344 +.. nonce: 7M_u8G +.. section: Library + +Fix substitution of :class:`~typing.TypeVarTuple` and +:class:`~typing.ParamSpec` together in user generics. + +.. + +.. date: 2022-11-09-12-36-12 +.. gh-issue: 99284 +.. nonce: 9p4J2l +.. section: Library + +Remove ``_use_broken_old_ctypes_structure_semantics_`` old untested and +undocumented hack from :mod:`ctypes`. + +.. + +.. date: 2022-11-09-03-34-29 +.. gh-issue: 99201 +.. nonce: lDJ7xI +.. section: Library + +Fix :exc:`IndexError` when initializing the config variables on Windows if +``HAVE_DYNAMIC_LOADING`` is not set. + +.. + +.. date: 2022-11-08-15-54-43 +.. gh-issue: 99240 +.. nonce: MhYwcz +.. section: Library + +Fix double-free bug in Argument Clinic ``str_converter`` by extracting +memory clean up to a new ``post_parsing`` section. + +.. + +.. date: 2022-11-08-11-18-51 +.. gh-issue: 64490 +.. nonce: VcBgrN +.. section: Library + +Fix refcount error when arguments are packed to tuple in Argument Clinic. + +.. + +.. date: 2022-11-02-23-47-07 +.. gh-issue: 99029 +.. nonce: 7uCiIB +.. section: Library + +:meth:`pathlib.PurePath.relative_to()` now treats naked Windows drive paths +as relative. This brings its behaviour in line with other parts of pathlib. + +.. + +.. date: 2022-10-24-11-01-05 +.. gh-issue: 98253 +.. nonce: HVd5v4 +.. section: Library + +The implementation of the typing module is now more resilient to reference +leaks in binary extension modules. + +Previously, a reference leak in a typed C API-based extension module could +leak internals of the typing module, which could in turn introduce leaks in +essentially any other package with typed function signatures. Although the +typing package is not the original source of the problem, such non-local +dependences exacerbate debugging of large-scale projects, and the +implementation was therefore changed to reduce harm by providing better +isolation. + +.. + +.. date: 2022-10-19-18-31-53 +.. gh-issue: 98458 +.. nonce: vwyq7O +.. section: Library + +Fix infinite loop in unittest when a self-referencing chained exception is +raised + +.. + +.. date: 2022-10-19-13-37-23 +.. gh-issue: 93453 +.. nonce: wTB_sH +.. section: Library + +:func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like +:func:`asyncio.ensure_future`, :func:`asyncio.shield` or +:func:`asyncio.gather`, and also the +:meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of +:class:`asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` +if called when there is no running event loop and the current event loop was +not set. Previously they implicitly created and set a new current event +loop. :exc:`DeprecationWarning` is no longer emitted if there is no running +event loop but the current event loop was set. + +.. + +.. date: 2022-10-16-18-52-00 +.. gh-issue: 97966 +.. nonce: humlhz +.. section: Library + +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. + +.. + +.. date: 2022-10-13-22-13-54 +.. gh-issue: 98248 +.. nonce: lwyygy +.. section: Library + +Provide informative error messages in :func:`struct.pack` when its integral +arguments are not in range. + +.. + +.. date: 2022-10-08-19-20-33 +.. gh-issue: 98108 +.. nonce: WUObqM +.. section: Library + +``zipfile.Path`` is now pickleable if its initialization parameters were +pickleable (e.g. for file system paths). + +.. + +.. date: 2022-10-08-15-41-00 +.. gh-issue: 98098 +.. nonce: DugpWi +.. section: Library + +Created packages from zipfile and test_zipfile modules, separating +``zipfile.Path`` functionality. + +.. + +.. date: 2022-10-02-12-38-22 +.. gh-issue: 82836 +.. nonce: OvYLmC +.. section: Library + +Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the +:mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would +return True from this method; now they correctly return False. + +.. + +.. date: 2022-09-14-21-56-15 +.. gh-issue: 96828 +.. nonce: ZoOY5G +.. section: Library + +Add an :data:`~ssl.OP_ENABLE_KTLS` option for enabling the use of the kernel +TLS (kTLS). Patch by Illia Volochii. + +.. + +.. date: 2022-08-06-12-18-07 +.. gh-issue: 88863 +.. nonce: NnqsuJ +.. section: Library + +To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, +break reference cycles generated by local exception and future instances +(which has exception instance as its member var). Patch by Dong Uk, Kang. + +.. + +.. date: 2022-04-23-03-46-37 +.. gh-issue: 91078 +.. nonce: 87-hkp +.. section: Library + +:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. + +.. + +.. bpo: 47220 +.. date: 2022-04-04-22-54-11 +.. nonce: L9jYu4 +.. section: Library + +Document the optional *callback* parameter of :class:`WeakMethod`. Patch by +G?ry Ogam. + +.. + +.. bpo: 44817 +.. date: 2021-08-03-05-31-00 +.. nonce: wOW_Qn +.. section: Library + +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) and +161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). + +.. + +.. bpo: 41260 +.. date: 2020-08-02-23-46-22 +.. nonce: Q2BNzY +.. section: Library + +Rename the *fmt* parameter of the pure Python implementation of +:meth:`datetime.date.strftime` to *format*. + +.. + +.. bpo: 15999 +.. date: 2019-08-30-10-48-53 +.. nonce: QqsRRi +.. section: Library + +All built-in functions now accept arguments of any type instead of just +``bool`` and ``int`` for boolean parameters. + +.. + +.. date: 2022-12-02-17-08-08 +.. gh-issue: 99931 +.. nonce: wC46hE +.. section: Documentation + +Use `sphinxext-opengraph <https://sphinxext-opengraph.readthedocs.io/>`__ to +generate `OpenGraph metadata <https://ogp.me/>`__. + +.. + +.. date: 2022-11-26-21-43-05 +.. gh-issue: 89682 +.. nonce: DhKoTM +.. section: Documentation + +Reworded docstring of the default ``__contains__`` to clarify that it +returns a :class:`bool`. + +.. + +.. date: 2022-11-26-15-51-23 +.. gh-issue: 88330 +.. nonce: B_wFq8 +.. section: Documentation + +Improved the description of what a resource is in importlib.resources docs. + +.. + +.. date: 2022-11-16-12-52-23 +.. gh-issue: 92892 +.. nonce: TS-P0j +.. section: Documentation + +Document that calling variadic functions with ctypes requires special care +on macOS/arm64 (and possibly other platforms). + +.. + +.. bpo: 41825 +.. date: 2020-09-22-12-32-16 +.. nonce: npcaCb +.. section: Documentation + +Restructured the documentation for the :func:`os.wait* <os.wait>` family of +functions, and improved the docs for :func:`os.waitid` with more explanation +of the possible argument constants. + +.. + +.. date: 2022-12-05-16-12-56 +.. gh-issue: 99892 +.. nonce: sz_eW8 +.. section: Tests + +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. + +.. + +.. date: 2022-12-01-18-55-18 +.. gh-issue: 99934 +.. nonce: Ox3Fqf +.. section: Tests + +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. + +.. + +.. date: 2022-11-23-18-32-16 +.. gh-issue: 99741 +.. nonce: q4R7NH +.. section: Tests + +We've implemented multi-phase init (PEP 489/630/687) for the internal (for +testing) _xxsubinterpreters module. + +.. + +.. date: 2022-11-21-19-21-30 +.. gh-issue: 99659 +.. nonce: 4gP0nm +.. section: Tests + +Optional big memory tests in ``test_sqlite3`` now catch the correct +:exc:`sqlite.DataError` exception type in case of too large strings and/or +blobs passed. + +.. + +.. date: 2022-11-19-13-34-28 +.. gh-issue: 99593 +.. nonce: 8ZfCkj +.. section: Tests + +Cover the Unicode C API with tests. + +.. + +.. date: 2022-08-22-15-49-14 +.. gh-issue: 96002 +.. nonce: 4UE9UE +.. section: Tests + +Add functional test for Argument Clinic. + +.. + +.. date: 2022-11-24-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and +``-Wimplicit-function-declaration`` compiler warnings in +:program:`configure` checks. + +.. + +.. date: 2022-11-15-08-40-22 +.. gh-issue: 99337 +.. nonce: 5LoQDE +.. section: Build + +Fix a compilation issue with GCC 12 on macOS. + +.. + +.. date: 2022-11-09-14-42-48 +.. gh-issue: 99289 +.. nonce: X7wFE1 +.. section: Build + +Add a ``COMPILEALL_OPTS`` variable in Makefile to override :mod:`compileall` +options (default: ``-j0``) in ``make install``. Also merged the +``compileall`` commands into a single command building .pyc files for the +all optimization levels (0, 1, 2) at once. Patch by Victor Stinner. + +.. + +.. date: 2022-11-03-08-10-49 +.. gh-issue: 98872 +.. nonce: gdsR8X +.. section: Build + +Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python +3.11. + +.. + +.. date: 2022-10-16-12-49-24 +.. gh-issue: 88226 +.. nonce: BsnQ4k +.. section: Build + +Always define ``TARGET_*`` labels in ``Python/ceval.c``, even if +``USE_COMPUTED_GOTOS`` is disabled. This allows breakpoints to be set at +those labels in (for instance) ``gdb``. + +.. + +.. date: 2022-11-23-17-17-16 +.. gh-issue: 99345 +.. nonce: jOa3-f +.. section: Windows + +Use faster initialization functions to detect install location for Windows +Store package + +.. + +.. date: 2022-11-21-19-50-18 +.. gh-issue: 98629 +.. nonce: tMmB_B +.. section: Windows + +Fix initialization of :data:`sys.version` and ``sys._git`` on Windows + +.. + +.. date: 2022-11-16-19-03-21 +.. gh-issue: 99442 +.. nonce: 6Dgk3Q +.. section: Windows + +Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file +extension. + +.. + +.. bpo: 40882 +.. date: 2020-06-06-15-10-37 +.. nonce: UvNbdj +.. section: Windows + +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. + +.. + +.. date: 2022-11-25-09-23-20 +.. gh-issue: 87235 +.. nonce: SifjCD +.. section: macOS + +On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script +longer than a couple of bytes. + +.. + +.. date: 2022-11-01-10-32-23 +.. gh-issue: 98940 +.. nonce: W3YzC_ +.. section: macOS + +Fix ``Mac/Extras.install.py`` file filter bug. + +.. + +.. date: 2022-08-11-09-58-15 +.. gh-issue: 64490 +.. nonce: PjwhM4 +.. section: Tools/Demos + +Argument Clinic varargs bugfixes + +* Fix out-of-bounds error in :c:func:`!_PyArg_UnpackKeywordsWithVararg`. +* Fix incorrect check which allowed more than one varargs in clinic.py. +* Fix miscalculation of ``noptargs`` in generated code. +* Do not generate ``noptargs`` when there is a vararg argument and no optional argument. + +.. + +.. date: 2022-12-05-17-30-13 +.. gh-issue: 98680 +.. nonce: FiMCxZ +.. section: C API + +``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. +These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for +3.11, and are necessary to use the buffer API. + +.. + +.. date: 2022-11-20-09-52-50 +.. gh-issue: 99612 +.. nonce: eBHksg +.. section: C API + +Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: +``*consumed`` was not set. + +.. + +.. date: 2022-11-02-16-51-24 +.. gh-issue: 47146 +.. nonce: dsYDtI +.. section: C API + +The ``structmember.h`` header is deprecated. Its non-deprecated contents are +now available just by including ``Python.h``, with a ``Py_`` prefix added if +it was missing. (Deprecated contents are :c:macro:`T_OBJECT`, +:c:macro:`T_NONE`, and no-op flags.) Patch by Petr Viktorin, based on +earlier work by Alexander Belopolsky and Matthias Braun. diff --git a/Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst b/Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst deleted file mode 100644 index 5f32091739a2..000000000000 --- a/Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst +++ /dev/null @@ -1,3 +0,0 @@ -Always define ``TARGET_*`` labels in ``Python/ceval.c``, even if -``USE_COMPUTED_GOTOS`` is disabled. This allows breakpoints to be -set at those labels in (for instance) ``gdb``. diff --git a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst b/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst deleted file mode 100644 index ad4dc496ee0e..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python 3.11. diff --git a/Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst b/Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst deleted file mode 100644 index 86ce4c79d17c..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add a ``COMPILEALL_OPTS`` variable in Makefile to override :mod:`compileall` -options (default: ``-j0``) in ``make install``. Also merged the ``compileall`` -commands into a single command building .pyc files for the all optimization levels -(0, 1, 2) at once. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst b/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst deleted file mode 100644 index f7396abde13b..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a compilation issue with GCC 12 on macOS. diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index 2dace165ca1a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst b/Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst deleted file mode 100644 index 0f419427925d..000000000000 --- a/Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst +++ /dev/null @@ -1,5 +0,0 @@ -The ``structmember.h`` header is deprecated. Its non-deprecated contents are -now available just by including ``Python.h``, with a ``Py_`` prefix added if -it was missing. (Deprecated contents are :c:macro:`T_OBJECT`, -:c:macro:`T_NONE`, and no-op flags.) Patch by Petr Viktorin, based on -earlier work by Alexander Belopolsky and Matthias Braun. diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst deleted file mode 100644 index 40e3c8db5403..000000000000 --- a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: -``*consumed`` was not set. diff --git a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst deleted file mode 100644 index a87090168e06..000000000000 --- a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. -These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for -3.11, and are necessary to use the buffer API. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst deleted file mode 100644 index b5964375962f..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure in :class:`io.FileIO` when the opener -returns an invalid file descriptor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst deleted file mode 100644 index dd96c9e20d87..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of -uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. -Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst deleted file mode 100644 index 481ab53e4f51..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optimize the :class:`range` object iterator. It is now smaller, faster -iteration of ranges containing large numbers. Smaller pickles, faster -unpickling. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst deleted file mode 100644 index 5edc1ea65d8b..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve error messages when there's a syntax error with call arguments. The following three cases are covered: -- No value is assigned to a named argument, eg ``foo(a=)``. -- A value is assigned to a star argument, eg ``foo(*args=[0])``. -- A value is assigned to a double-star keyword argument, eg ``foo(**kwarg={'a': 0})``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst deleted file mode 100644 index 59bb12caef74..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst +++ /dev/null @@ -1,4 +0,0 @@ -Optimizing interpreters and JIT compilers may need to invalidate internal -metadata when functions are modified. This change adds the ability to -provide a callback that will be invoked each time a function is created, -modified, or destroyed. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst deleted file mode 100644 index e93ae4e7b127..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst +++ /dev/null @@ -1 +0,0 @@ -Allow some features of :mod:`syslog` to the main interpreter only. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst deleted file mode 100644 index 79a5b3ba98cf..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove the remaining error paths for attribute specializations, and refuse -to specialize attribute accesses on types that haven't had -:c:func:`PyType_Ready` called on them yet. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst deleted file mode 100644 index b4a40d168429..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove the ``BINARY_OP_GENERIC`` and ``COMPARE_OP_GENERIC`` -"specializations". diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst deleted file mode 100644 index 631b9ca23044..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst +++ /dev/null @@ -1 +0,0 @@ -Add audit events for thread creation and clear operations. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst deleted file mode 100644 index 142f91ccd92e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix zip path for venv created from a non-installed python on POSIX -platforms. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst deleted file mode 100644 index 7e3c52924213..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst +++ /dev/null @@ -1 +0,0 @@ -Add a function to os.path to check if a path is a junction: isjunction. Add similar functionality to pathlib.Path as is_junction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst deleted file mode 100644 index 8d9f55d6d9d0..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where an :exc:`ExceptionGroup` subclass can wrap a -:exc:`BaseException`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst deleted file mode 100644 index 8071fd130dd6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the -available tokenizer buffer. Patch by Pablo galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst deleted file mode 100644 index 9321cef77eed..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of the -first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst deleted file mode 100644 index 47f385c2eefe..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug where compiler crashes on an if expression with an empty body block. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst deleted file mode 100644 index 3fe21a8a21bf..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue that could cause frames to be visible to Python code as they -are being torn down, possibly leading to memory corruption or hard crashes -of the interpreter. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst deleted file mode 100644 index c46459c15b9e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to -register callbacks to receive notification on creation and destruction of -code objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst deleted file mode 100644 index 20cd361affea..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the tokenizer that could cause infinite recursion when showing -syntax warnings that happen in the first line of the source. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst b/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst deleted file mode 100644 index 390b4a9824c7..000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Restructured the documentation for the :func:`os.wait* <os.wait>` family of functions, -and improved the docs for :func:`os.waitid` with more explanation of the -possible argument constants. diff --git a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst b/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst deleted file mode 100644 index 54e421d19d9d..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst +++ /dev/null @@ -1 +0,0 @@ -Document that calling variadic functions with ctypes requires special care on macOS/arm64 (and possibly other platforms). diff --git a/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst b/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst deleted file mode 100644 index 0f242eecc312..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst +++ /dev/null @@ -1 +0,0 @@ -Improved the description of what a resource is in importlib.resources docs. diff --git a/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst b/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst deleted file mode 100644 index 46be065b6539..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst +++ /dev/null @@ -1 +0,0 @@ -Reworded docstring of the default ``__contains__`` to clarify that it returns a :class:`bool`. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst deleted file mode 100644 index 0c01a2cb2cfa..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use `sphinxext-opengraph <https://sphinxext-opengraph.readthedocs.io/>`__ -to generate `OpenGraph metadata <https://ogp.me/>`__. diff --git a/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst b/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst deleted file mode 100644 index 9650c694cc67..000000000000 --- a/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst +++ /dev/null @@ -1,2 +0,0 @@ -All built-in functions now accept arguments of any type instead of just -``bool`` and ``int`` for boolean parameters. diff --git a/Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst b/Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst deleted file mode 100644 index ae2fdd9b84a0..000000000000 --- a/Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Rename the *fmt* parameter of the pure Python implementation of -:meth:`datetime.date.strftime` to *format*. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst deleted file mode 100644 index 79f8c506b54f..000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) -and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). diff --git a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst b/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst deleted file mode 100644 index 6e2af088640b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document the optional *callback* parameter of :class:`WeakMethod`. Patch by -G?ry Ogam. diff --git a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst b/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst deleted file mode 100644 index e05d5e2a1314..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. diff --git a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst b/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst deleted file mode 100644 index 23f8cb01cf0a..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, -break reference cycles generated by local exception and future instances -(which has exception instance as its member var). Patch by Dong Uk, Kang. diff --git a/Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst b/Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst deleted file mode 100644 index d8a448851f47..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add an :data:`~ssl.OP_ENABLE_KTLS` option for enabling the use of the kernel -TLS (kTLS). Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst deleted file mode 100644 index dcbea66d66bf..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would return True from this method; now they correctly return False. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst b/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst deleted file mode 100644 index 202275e16ea0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Created packages from zipfile and test_zipfile modules, separating -``zipfile.Path`` functionality. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst b/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst deleted file mode 100644 index 7e962580dda2..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst +++ /dev/null @@ -1,2 +0,0 @@ -``zipfile.Path`` is now pickleable if its initialization parameters were -pickleable (e.g. for file system paths). diff --git a/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst b/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst deleted file mode 100644 index 347f6e160335..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst +++ /dev/null @@ -1 +0,0 @@ -Provide informative error messages in :func:`struct.pack` when its integral arguments are not in range. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst deleted file mode 100644 index b725465ae4f0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst +++ /dev/null @@ -1,2 +0,0 @@ -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst b/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst deleted file mode 100644 index 5aee4b4f0aa7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst +++ /dev/null @@ -1,9 +0,0 @@ -:func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like -:func:`asyncio.ensure_future`, :func:`asyncio.shield` or -:func:`asyncio.gather`, and also the -:meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of -:class:`asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` -if called when there is no running event loop and the current event loop was -not set. Previously they implicitly created and set a new current event -loop. :exc:`DeprecationWarning` is no longer emitted if there is no running -event loop but the current event loop was set. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst deleted file mode 100644 index f74195cc8e7d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst +++ /dev/null @@ -1 +0,0 @@ -Fix infinite loop in unittest when a self-referencing chained exception is raised diff --git a/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst b/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst deleted file mode 100644 index 00df0070f3b9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst +++ /dev/null @@ -1,10 +0,0 @@ -The implementation of the typing module is now more resilient to reference -leaks in binary extension modules. - -Previously, a reference leak in a typed C API-based extension module could leak -internals of the typing module, which could in turn introduce leaks in -essentially any other package with typed function signatures. Although the -typing package is not the original source of the problem, such non-local -dependences exacerbate debugging of large-scale projects, and the -implementation was therefore changed to reduce harm by providing better -isolation. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst b/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst deleted file mode 100644 index 0bfba5e1e326..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`pathlib.PurePath.relative_to()` now treats naked Windows drive paths -as relative. This brings its behaviour in line with other parts of pathlib. diff --git a/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst b/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst deleted file mode 100644 index f98c181cc9c5..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst +++ /dev/null @@ -1 +0,0 @@ -Fix refcount error when arguments are packed to tuple in Argument Clinic. diff --git a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst deleted file mode 100644 index 0a4db052755f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix double-free bug in Argument Clinic ``str_converter`` by -extracting memory clean up to a new ``post_parsing`` section. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst b/Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst deleted file mode 100644 index 6d03574fdaf5..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :exc:`IndexError` when initializing the config variables on Windows if -``HAVE_DYNAMIC_LOADING`` is not set. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst b/Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst deleted file mode 100644 index 48576bd457aa..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove ``_use_broken_old_ctypes_structure_semantics_`` -old untested and undocumented hack from :mod:`ctypes`. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst deleted file mode 100644 index 412c8c793435..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.TypeVarTuple` and -:class:`~typing.ParamSpec` together in user generics. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst deleted file mode 100644 index 1950680b1df8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.ParamSpec` followed by -:class:`~typing.TypeVarTuple` in generic aliases. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst deleted file mode 100644 index f153f2fceac8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Check the number of arguments in substitution in user generics containing a -:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. diff --git a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst b/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst deleted file mode 100644 index 451561c579da..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when -changing line numbers. diff --git a/Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst b/Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst deleted file mode 100644 index f35799d45457..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add *loop_factory* parameter to :func:`asyncio.run` to allow specifying a custom event loop factory. -Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst b/Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst deleted file mode 100644 index c6e2a0ca25ff..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst +++ /dev/null @@ -1 +0,0 @@ -Use final status of an enum to determine lookup or creation branch of functional API. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst b/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst deleted file mode 100644 index 014161cf7b1d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst +++ /dev/null @@ -1 +0,0 @@ -Release the GIL when calling termios APIs to avoid blocking threads. diff --git a/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst b/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst deleted file mode 100644 index ebd821738827..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :mod:`asyncio` to not share event loop and signal wakeupfd in forked processes. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst deleted file mode 100644 index 82720d17bcaf..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling -``_imp.source_hash()``. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst b/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst deleted file mode 100644 index f6ee449891d9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now -``addClassCleanup()`` uses separate lists for different ``TestCase`` -subclasses, and ``doClassCleanups()`` only cleans up the particular class. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst deleted file mode 100644 index 63fe7b8a3a32..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug when calling trace.CoverageResults with valid infile. diff --git a/Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst b/Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst deleted file mode 100644 index 4b12fd9c8d79..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`asyncio.Server.wait_closed` to actually do what the docs promise -- wait for all existing connections to complete, after closing the server. diff --git a/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst b/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst deleted file mode 100644 index 4fd7b6b85cef..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst deleted file mode 100644 index e305352c7a55..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst +++ /dev/null @@ -1,5 +0,0 @@ -Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so -that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex -replacement we put in or the characters r"\x" came through in the original -request line. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst deleted file mode 100644 index dc86e1d70f12..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst +++ /dev/null @@ -1 +0,0 @@ -Add functional test for Argument Clinic. diff --git a/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst b/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst deleted file mode 100644 index ec4cda208032..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst +++ /dev/null @@ -1 +0,0 @@ -Cover the Unicode C API with tests. diff --git a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst b/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst deleted file mode 100644 index 3db1ec12b520..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optional big memory tests in ``test_sqlite3`` now catch the correct -:exc:`sqlite.DataError` exception type in case of too large strings and/or -blobs passed. diff --git a/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst deleted file mode 100644 index 791726b52bfb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst +++ /dev/null @@ -1,2 +0,0 @@ -We've implemented multi-phase init (PEP 489/630/687) -for the internal (for testing) _xxsubinterpreters module. diff --git a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst deleted file mode 100644 index 1b1287c33dbd..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst +++ /dev/null @@ -1 +0,0 @@ -Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst deleted file mode 100644 index eded0361fbeb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip test_normalization() of test_unicodedata if it fails to download -NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst deleted file mode 100644 index 4a308a930605..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst +++ /dev/null @@ -1,7 +0,0 @@ -Argument Clinic varargs bugfixes - -* Fix out-of-bounds error in :c:func:`!_PyArg_UnpackKeywordsWithVararg`. -* Fix incorrect check which allowed more than one varargs in clinic.py. -* Fix miscalculation of ``noptargs`` in generated code. -* Do not generate ``noptargs`` when there is a vararg argument and no optional argument. - diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst deleted file mode 100644 index 2670aeef9a25..000000000000 --- a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on -Windows. diff --git a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst b/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst deleted file mode 100644 index 8e19366c4297..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file -extension. diff --git a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst b/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst deleted file mode 100644 index 46cbf998eb20..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst +++ /dev/null @@ -1 +0,0 @@ -Fix initialization of :data:`sys.version` and ``sys._git`` on Windows diff --git a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst b/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst deleted file mode 100644 index 99db0c55a67e..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use faster initialization functions to detect install location for Windows -Store package diff --git a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst b/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst deleted file mode 100644 index 18ef0b0e2523..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``Mac/Extras.install.py`` file filter bug. diff --git a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst b/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst deleted file mode 100644 index 3111e4975e87..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst +++ /dev/null @@ -1 +0,0 @@ -On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script longer than a couple of bytes. diff --git a/README.rst b/README.rst index 1fa019473643..ab9d3a6ea71c 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 2 +This is Python version 3.12.0 alpha 3 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From webhook-mailer at python.org Tue Dec 6 16:44:03 2022 From: webhook-mailer at python.org (ethanfurman) Date: Tue, 06 Dec 2022 21:44:03 -0000 Subject: [Python-checkins] gh-94943: [Enum] improve repr() when inheriting from a dataclass (GH-99740) Message-ID: <mailman.2834.1670363044.3313.python-checkins@python.org> https://github.com/python/cpython/commit/679efbb080242fc5be63ad873968f05faeef889f commit: 679efbb080242fc5be63ad873968f05faeef889f branch: main author: Ethan Furman <ethan at stoneleaf.us> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-06T13:43:41-08:00 summary: gh-94943: [Enum] improve repr() when inheriting from a dataclass (GH-99740) Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: A Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst M Doc/howto/enum.rst M Doc/library/enum.rst M Lib/enum.py M Lib/test/test_enum.py diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 98d9f4febe2d..3155c6cb977b 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -459,6 +459,31 @@ sense to allow sharing some common behavior between a group of enumerations. (See `OrderedEnum`_ for an example.) +.. _enum-dataclass-support: + +Dataclass support +----------------- + +When inheriting from a :class:`~dataclasses.dataclass`, +the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: + + >>> @dataclass + ... class CreatureDataMixin: + ... size: str + ... legs: int + ... tail: bool = field(repr=False, default=True) + ... + >>> class Creature(CreatureDataMixin, Enum): + ... BEETLE = 'small', 6 + ... DOG = 'medium', 4 + ... + >>> Creature.DOG + <Creature.DOG: size='medium', legs=4> + +Use the :func:`!dataclass` argument ``repr=False`` +to use the standard :func:`repr`. + + Pickling -------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 208aecf11c80..f8875440baf5 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -389,6 +389,8 @@ Data Types Using :class:`auto` with :class:`Enum` results in integers of increasing value, starting with ``1``. + .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. class:: IntEnum diff --git a/Lib/enum.py b/Lib/enum.py index 1b683c702d59..f07b821f1a60 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -955,7 +955,15 @@ def _find_data_repr_(mcls, class_name, bases): return base._value_repr_ elif '__repr__' in base.__dict__: # this is our data repr - return base.__dict__['__repr__'] + # double-check if a dataclass with a default __repr__ + if ( + '__dataclass_fields__' in base.__dict__ + and '__dataclass_params__' in base.__dict__ + and base.__dict__['__dataclass_params__'].repr + ): + return _dataclass_repr + else: + return base.__dict__['__repr__'] return None @classmethod @@ -1551,6 +1559,14 @@ def _power_of_two(value): return False return value == 2 ** _high_bit(value) +def _dataclass_repr(self): + dcf = self.__dataclass_fields__ + return ', '.join( + '%s=%r' % (k, getattr(self, k)) + for k in dcf.keys() + if dcf[k].repr + ) + def global_enum_repr(self): """ use module.enum_name instead of class.enum_name diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index b6082cf02b18..146ef39ca384 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2717,17 +2717,67 @@ def upper(self): def test_repr_with_dataclass(self): "ensure dataclass-mixin has correct repr()" - from dataclasses import dataclass - @dataclass + # + # check overridden dataclass __repr__ is used + # + from dataclasses import dataclass, field + @dataclass(repr=False) class Foo: __qualname__ = 'Foo' a: int + def __repr__(self): + return 'ha hah!' class Entries(Foo, Enum): ENTRY1 = 1 self.assertTrue(isinstance(Entries.ENTRY1, Foo)) self.assertTrue(Entries._member_type_ is Foo, Entries._member_type_) self.assertTrue(Entries.ENTRY1.value == Foo(1), Entries.ENTRY1.value) - self.assertEqual(repr(Entries.ENTRY1), '<Entries.ENTRY1: Foo(a=1)>') + self.assertEqual(repr(Entries.ENTRY1), '<Entries.ENTRY1: ha hah!>') + # + # check auto-generated dataclass __repr__ is not used + # + @dataclass + class CreatureDataMixin: + __qualname__ = 'CreatureDataMixin' + size: str + legs: int + tail: bool = field(repr=False, default=True) + class Creature(CreatureDataMixin, Enum): + __qualname__ = 'Creature' + BEETLE = ('small', 6) + DOG = ('medium', 4) + self.assertEqual(repr(Creature.DOG), "<Creature.DOG: size='medium', legs=4>") + # + # check inherited repr used + # + class Huh: + def __repr__(self): + return 'inherited' + @dataclass(repr=False) + class CreatureDataMixin(Huh): + __qualname__ = 'CreatureDataMixin' + size: str + legs: int + tail: bool = field(repr=False, default=True) + class Creature(CreatureDataMixin, Enum): + __qualname__ = 'Creature' + BEETLE = ('small', 6) + DOG = ('medium', 4) + self.assertEqual(repr(Creature.DOG), "<Creature.DOG: inherited>") + # + # check default object.__repr__ used if nothing provided + # + @dataclass(repr=False) + class CreatureDataMixin: + __qualname__ = 'CreatureDataMixin' + size: str + legs: int + tail: bool = field(repr=False, default=True) + class Creature(CreatureDataMixin, Enum): + __qualname__ = 'Creature' + BEETLE = ('small', 6) + DOG = ('medium', 4) + self.assertRegex(repr(Creature.DOG), "<Creature.DOG: .*CreatureDataMixin object at .*>") def test_repr_with_init_data_type_mixin(self): # non-data_type is a mixin that doesn't define __new__ diff --git a/Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst b/Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst new file mode 100644 index 000000000000..ed4754e49bd2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst @@ -0,0 +1,5 @@ +Add :ref:`enum-dataclass-support` to the +:class:`~enum.Enum` :meth:`~enum.Enum.__repr__`. +When inheriting from a :class:`~dataclasses.dataclass`, +only show the field names in the value section of the member :func:`repr`, +and not the dataclass' class name. From webhook-mailer at python.org Tue Dec 6 17:40:10 2022 From: webhook-mailer at python.org (vstinner) Date: Tue, 06 Dec 2022 22:40:10 -0000 Subject: [Python-checkins] PyUnicode_KIND() uses _Py_RVALUE() (#100060) Message-ID: <mailman.2835.1670366411.3313.python-checkins@python.org> https://github.com/python/cpython/commit/abbe4482ab914d6da8b1678ad46174cb875ed7f6 commit: abbe4482ab914d6da8b1678ad46174cb875ed7f6 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-06T23:40:05+01:00 summary: PyUnicode_KIND() uses _Py_RVALUE() (#100060) The PyUnicode_KIND() macro is modified to use _Py_RVALUE(), so it can no longer be used as a l-value. files: M Include/cpython/unicodeobject.h diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index a75336f590e8..75a74ffa2f9d 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -231,7 +231,7 @@ enum PyUnicode_Kind { // new compiler warnings on "kind < PyUnicode_KIND(str)" (compare signed and // unsigned numbers) where kind type is an int or on // "unsigned int kind = PyUnicode_KIND(str)" (cast signed to unsigned). -#define PyUnicode_KIND(op) (_PyASCIIObject_CAST(op)->state.kind) +#define PyUnicode_KIND(op) _Py_RVALUE(_PyASCIIObject_CAST(op)->state.kind) /* Return a void pointer to the raw unicode buffer. */ static inline void* _PyUnicode_COMPACT_DATA(PyObject *op) { From webhook-mailer at python.org Tue Dec 6 18:10:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 06 Dec 2022 23:10:02 -0000 Subject: [Python-checkins] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) Message-ID: <mailman.2836.1670368203.3313.python-checkins@python.org> https://github.com/python/cpython/commit/97e7004cfe48305bcd642c653b406dc7470e196d commit: 97e7004cfe48305bcd642c653b406dc7470e196d branch: main author: Pablo Galindo Salgado <Pablogsal at gmail.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-06T15:09:56-08:00 summary: gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) Automerge-Triggered-By: GH:pablogsal files: A Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst M Lib/test/test_syntax.py M Parser/pegen_errors.c diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 78cac231929a..cb284195d976 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2145,6 +2145,22 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + def test_error_string_literal(self): + + self._check_error("'blech", "unterminated string literal") + self._check_error('"blech', "unterminated string literal") + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst new file mode 100644 index 000000000000..8e7c72d804f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst @@ -0,0 +1,2 @@ +Honor existing errors obtained when searching for mismatching parentheses in +the tokenizer. Patch by Pablo Galindo diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 7738cbaf9ef3..6ea7600119b6 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -169,6 +169,10 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { for (;;) { switch (_PyTokenizer_Get(p->tok, &new_token)) { case ERRORTOKEN: + if (PyErr_Occurred()) { + ret = -1; + goto exit; + } if (p->tok->level != 0) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; if (current_err_line > error_lineno) { From webhook-mailer at python.org Tue Dec 6 21:44:58 2022 From: webhook-mailer at python.org (ethanfurman) Date: Wed, 07 Dec 2022 02:44:58 -0000 Subject: [Python-checkins] [Enum] Remove unused code from `test_enum.py` (GH-96986) Message-ID: <mailman.2837.1670381100.3313.python-checkins@python.org> https://github.com/python/cpython/commit/889b0b9bf95651fc05ad532cc4a66c0f8ff29fc2 commit: 889b0b9bf95651fc05ad532cc4a66c0f8ff29fc2 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-06T18:44:47-08:00 summary: [Enum] Remove unused code from `test_enum.py` (GH-96986) files: M Lib/test/test_enum.py diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 146ef39ca384..376347e4c111 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4553,11 +4553,6 @@ class Quadruple(Enum): COMPLEX_A = 2j COMPLEX_B = 3j -class _ModuleWrapper: - """We use this class as a namespace for swapping modules.""" - def __init__(self, module): - self.__dict__.update(module.__dict__) - class TestConvert(unittest.TestCase): def tearDown(self): # Reset the module-level test variables to their original integer @@ -4597,12 +4592,6 @@ def test_convert_int(self): self.assertEqual(test_type.CONVERT_TEST_NAME_D, 5) self.assertEqual(test_type.CONVERT_TEST_NAME_E, 5) # Ensure that test_type only picked up names matching the filter. - int_dir = dir(int) + [ - 'CONVERT_TEST_NAME_A', 'CONVERT_TEST_NAME_B', 'CONVERT_TEST_NAME_C', - 'CONVERT_TEST_NAME_D', 'CONVERT_TEST_NAME_E', 'CONVERT_TEST_NAME_F', - 'CONVERT_TEST_SIGABRT', 'CONVERT_TEST_SIGIOT', - 'CONVERT_TEST_EIO', 'CONVERT_TEST_EBUS', - ] extra = [name for name in dir(test_type) if name not in enum_dir(test_type)] missing = [name for name in enum_dir(test_type) if name not in dir(test_type)] self.assertEqual( @@ -4644,7 +4633,6 @@ def test_convert_str(self): self.assertEqual(test_type.CONVERT_STR_TEST_1, 'hello') self.assertEqual(test_type.CONVERT_STR_TEST_2, 'goodbye') # Ensure that test_type only picked up names matching the filter. - str_dir = dir(str) + ['CONVERT_STR_TEST_1', 'CONVERT_STR_TEST_2'] extra = [name for name in dir(test_type) if name not in enum_dir(test_type)] missing = [name for name in enum_dir(test_type) if name not in dir(test_type)] self.assertEqual( @@ -4712,8 +4700,6 @@ def member_dir(member): allowed.add(name) return sorted(allowed) -missing = object() - if __name__ == '__main__': unittest.main() From webhook-mailer at python.org Wed Dec 7 01:55:59 2022 From: webhook-mailer at python.org (corona10) Date: Wed, 07 Dec 2022 06:55:59 -0000 Subject: [Python-checkins] gh-93018: Fix for the compatibility problems with expat (gh-93900) Message-ID: <mailman.2838.1670396160.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7031275776f43c76231318c2158a7a2753bc1fba commit: 7031275776f43c76231318c2158a7a2753bc1fba branch: main author: Mat?j Cepl <mcepl at cepl.eu> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-07T15:55:49+09:00 summary: gh-93018: Fix for the compatibility problems with expat (gh-93900) files: A Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst M Lib/test/test_minidom.py diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index ef38c362103f..2ca3908bd1ca 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -1163,14 +1163,10 @@ def testEncodings(self): # Verify that character decoding errors raise exceptions instead # of crashing - if pyexpat.version_info >= (2, 4, 5): - self.assertRaises(ExpatError, parseString, - b'<fran\xe7ais></fran\xe7ais>') - self.assertRaises(ExpatError, parseString, - b'<franais>Comment \xe7a va ? Tr\xe8s bien ?</franais>') - else: - self.assertRaises(UnicodeDecodeError, parseString, - b'<fran\xe7ais>Comment \xe7a va ? Tr\xe8s bien ?</fran\xe7ais>') + with self.assertRaises((UnicodeDecodeError, ExpatError)): + parseString( + b'<fran\xe7ais>Comment \xe7a va ? Tr\xe8s bien ?</fran\xe7ais>' + ) doc.unlink() @@ -1631,13 +1627,11 @@ def testEmptyXMLNSValue(self): self.confirm(doc2.namespaceURI == xml.dom.EMPTY_NAMESPACE) def testExceptionOnSpacesInXMLNSValue(self): - if pyexpat.version_info >= (2, 4, 5): - context = self.assertRaisesRegex(ExpatError, 'syntax error') - else: - context = self.assertRaisesRegex(ValueError, 'Unsupported syntax') - - with context: - parseString('<element xmlns:abc="http:abc.com/de f g/hi/j k"><abc:foo /></element>') + with self.assertRaises((ValueError, ExpatError)): + parseString( + '<element xmlns:abc="http:abc.com/de f g/hi/j k">' + + '<abc:foo /></element>' + ) def testDocRemoveChild(self): doc = parse(tstfile) diff --git a/Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst b/Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst new file mode 100644 index 000000000000..a8fb98048e40 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst @@ -0,0 +1 @@ +Make two tests forgiving towards host system libexpat with backported security fixes applied. From webhook-mailer at python.org Wed Dec 7 04:18:41 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 07 Dec 2022 09:18:41 -0000 Subject: [Python-checkins] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) Message-ID: <mailman.2839.1670404722.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b97ddd5122c016bd090ed187479606205b6b1a4 commit: 2b97ddd5122c016bd090ed187479606205b6b1a4 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T01:18:00-08:00 summary: gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) (cherry picked from commit 97e7004cfe48305bcd642c653b406dc7470e196d) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> Automerge-Triggered-By: GH:pablogsal files: A Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst M Lib/test/test_syntax.py M Parser/pegen_errors.c diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 400092ee2c89..42d36e0b9d81 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2095,6 +2095,22 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + def test_error_string_literal(self): + + self._check_error("'blech", "unterminated string literal") + self._check_error('"blech', "unterminated string literal") + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst new file mode 100644 index 000000000000..8e7c72d804f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst @@ -0,0 +1,2 @@ +Honor existing errors obtained when searching for mismatching parentheses in +the tokenizer. Patch by Pablo Galindo diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index a0f4b9809e21..3d8cccb0a974 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -170,6 +170,10 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { const char *end; switch (_PyTokenizer_Get(p->tok, &start, &end)) { case ERRORTOKEN: + if (PyErr_Occurred()) { + ret = -1; + goto exit; + } if (p->tok->level != 0) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; if (current_err_line > error_lineno) { From webhook-mailer at python.org Wed Dec 7 08:52:30 2022 From: webhook-mailer at python.org (pablogsal) Date: Wed, 07 Dec 2022 13:52:30 -0000 Subject: [Python-checkins] [3.10] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) (#100073) Message-ID: <mailman.2840.1670421151.3313.python-checkins@python.org> https://github.com/python/cpython/commit/72cfe5b1b9ad1bbe9da38a5af605e5d03ad88147 commit: 72cfe5b1b9ad1bbe9da38a5af605e5d03ad88147 branch: 3.10 author: Pablo Galindo Salgado <Pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-07T13:52:05Z summary: [3.10] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) (#100073) Automerge-Triggered-By: GH:pablogsal. (cherry picked from commit 97e7004cfe48305bcd642c653b406dc7470e196d) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst M Lib/test/test_syntax.py M Parser/pegen.c diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 5f2528b904b5..cd09d1eabdf5 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1587,6 +1587,22 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + def test_error_string_literal(self): + + self._check_error("'blech", "unterminated string literal") + self._check_error('"blech', "unterminated string literal") + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + def test_match_call_does_not_raise_syntax_error(self): code = """ def match(x): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst new file mode 100644 index 000000000000..8e7c72d804f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst @@ -0,0 +1,2 @@ +Honor existing errors obtained when searching for mismatching parentheses in +the tokenizer. Patch by Pablo Galindo diff --git a/Parser/pegen.c b/Parser/pegen.c index c12adedebba3..95c2273e4b42 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1328,6 +1328,10 @@ _PyPegen_check_tokenizer_errors(Parser *p) { const char *end; switch (PyTokenizer_Get(p->tok, &start, &end)) { case ERRORTOKEN: + if (PyErr_Occurred()) { + ret = -1; + goto exit; + } if (p->tok->level != 0) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; if (current_err_line > error_lineno) { From webhook-mailer at python.org Wed Dec 7 09:22:44 2022 From: webhook-mailer at python.org (vstinner) Date: Wed, 07 Dec 2022 14:22:44 -0000 Subject: [Python-checkins] gh-98724: Fix Py_CLEAR() macro side effects (#99100) (#100070) Message-ID: <mailman.2841.1670422965.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b11a384dc7471ffc16de4b86e8f5fdeef151f348 commit: b11a384dc7471ffc16de4b86e8f5fdeef151f348 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-07T15:22:38+01:00 summary: gh-98724: Fix Py_CLEAR() macro side effects (#99100) (#100070) The Py_CLEAR(), Py_SETREF() and Py_XSETREF() macros now only evaluate their arguments once. If an argument has side effects, these side effects are no longer duplicated. Use temporary variables to avoid duplicating side effects of macro arguments. If available, use _Py_TYPEOF() to avoid type punning. Otherwise, use memcpy() for the assignment to prevent a miscompilation with strict aliasing caused by type punning. Add _Py_TYPEOF() macro: __typeof__() on GCC and clang. Add test_py_clear() and test_py_setref() unit tests to _testcapi. files: A Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst M Doc/c-api/refcounting.rst M Doc/whatsnew/3.12.rst M Include/cpython/object.h M Include/object.h M Include/pyport.h M Modules/_testcapimodule.c diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index cd1f2ef70768..d8e9c2da6f3f 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -7,8 +7,8 @@ Reference Counting ****************** -The macros in this section are used for managing reference counts of Python -objects. +The functions and macros in this section are used for managing reference counts +of Python objects. .. c:function:: Py_ssize_t Py_REFCNT(PyObject *o) @@ -129,6 +129,11 @@ objects. It is a good idea to use this macro whenever decrementing the reference count of an object that might be traversed during garbage collection. + .. versionchanged:: 3.12 + The macro argument is now only evaluated once. If the argument has side + effects, these are no longer duplicated. + + .. c:function:: void Py_IncRef(PyObject *o) Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. @@ -139,3 +144,40 @@ objects. Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. + + +.. c:macro:: Py_SETREF(dst, src) + + Macro safely decrementing the `dst` reference count and setting `dst` to + `src`. + + As in case of :c:func:`Py_CLEAR`, "the obvious" code can be deadly:: + + Py_DECREF(dst); + dst = src; + + The safe way is:: + + Py_SETREF(dst, src); + + That arranges to set `dst` to `src` _before_ decrementing reference count of + *dst* old value, so that any code triggered as a side-effect of `dst` + getting torn down no longer believes `dst` points to a valid object. + + .. versionadded:: 3.6 + + .. versionchanged:: 3.12 + The macro arguments are now only evaluated once. If an argument has side + effects, these are no longer duplicated. + + +.. c:macro:: Py_XSETREF(dst, src) + + Variant of :c:macro:`Py_SETREF` macro that uses :c:func:`Py_XDECREF` instead + of :c:func:`Py_DECREF`. + + .. versionadded:: 3.6 + + .. versionchanged:: 3.12 + The macro arguments are now only evaluated once. If an argument has side + effects, these are no longer duplicated. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index e9e25b9dc466..73dc462f0b33 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -851,6 +851,11 @@ Porting to Python 3.12 :class:`bytes` type is accepted for bytes strings. (Contributed by Victor Stinner in :gh:`98393`.) +* The :c:macro:`Py_CLEAR`, :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` + macros now only evaluate their arguments once. If an argument has side + effects, these side effects are no longer duplicated. + (Contributed by Victor Stinner in :gh:`98724`.) + Deprecated ---------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3abfcb7d44f0..426337086130 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -305,38 +305,69 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *, PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); -/* Safely decref `op` and set `op` to `op2`. +/* Safely decref `dst` and set `dst` to `src`. * * As in case of Py_CLEAR "the obvious" code can be deadly: * - * Py_DECREF(op); - * op = op2; + * Py_DECREF(dst); + * dst = src; * * The safe way is: * - * Py_SETREF(op, op2); + * Py_SETREF(dst, src); * - * That arranges to set `op` to `op2` _before_ decref'ing, so that any code - * triggered as a side-effect of `op` getting torn down no longer believes - * `op` points to a valid object. + * That arranges to set `dst` to `src` _before_ decref'ing, so that any code + * triggered as a side-effect of `dst` getting torn down no longer believes + * `dst` points to a valid object. * - * Py_XSETREF is a variant of Py_SETREF that uses Py_XDECREF instead of - * Py_DECREF. + * Temporary variables are used to only evalutate macro arguments once and so + * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to + * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for + * implementation details about type punning. + * + * The memcpy() implementation does not emit a compiler warning if 'src' has + * not the same type than 'src': any pointer type is accepted for 'src'. */ - -#define Py_SETREF(op, op2) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - (op) = (op2); \ - Py_DECREF(_py_tmp); \ +#ifdef _Py_TYPEOF +#define Py_SETREF(dst, src) \ + do { \ + _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \ + _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_DECREF(_tmp_old_dst); \ } while (0) +#else +#define Py_SETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ + PyObject *_tmp_src = _PyObject_CAST(src); \ + memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_dst); \ + } while (0) +#endif -#define Py_XSETREF(op, op2) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - (op) = (op2); \ - Py_XDECREF(_py_tmp); \ +/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of + * Py_DECREF(). + */ +#ifdef _Py_TYPEOF +#define Py_XSETREF(dst, src) \ + do { \ + _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \ + _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_XDECREF(_tmp_old_dst); \ + } while (0) +#else +#define Py_XSETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ + PyObject *_tmp_src = _PyObject_CAST(src); \ + memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ + Py_XDECREF(_tmp_old_dst); \ } while (0) +#endif PyAPI_DATA(PyTypeObject) _PyNone_Type; diff --git a/Include/object.h b/Include/object.h index 75624fe8c77a..3774f1267300 100644 --- a/Include/object.h +++ b/Include/object.h @@ -598,15 +598,44 @@ static inline void Py_DECREF(PyObject *op) * one of those can't cause problems -- but in part that relies on that * Python integers aren't currently weakly referencable. Best practice is * to use Py_CLEAR() even if you can't think of a reason for why you need to. + * + * gh-98724: Use a temporary variable to only evaluate the macro argument once, + * to avoid the duplication of side effects if the argument has side effects. + * + * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, + * the code can be miscompiled with strict aliasing because of type punning. + * With strict aliasing, a compiler considers that two pointers of different + * types cannot read or write the same memory which enables optimization + * opportunities. + * + * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, + * and so avoid type punning. Otherwise, use memcpy() which causes type erasure + * and so prevents the compiler to reuse an old cached 'op' value after + * Py_CLEAR(). */ -#define Py_CLEAR(op) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - if (_py_tmp != NULL) { \ - (op) = NULL; \ - Py_DECREF(_py_tmp); \ - } \ +#ifdef _Py_TYPEOF +#define Py_CLEAR(op) \ + do { \ + _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ + _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + *_tmp_op_ptr = _Py_NULL; \ + Py_DECREF(_tmp_old_op); \ + } \ } while (0) +#else +#define Py_CLEAR(op) \ + do { \ + PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ + PyObject *_tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + PyObject *_null_ptr = _Py_NULL; \ + memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#endif + /* Function to use in case the object pointer can be NULL: */ static inline void Py_XINCREF(PyObject *op) diff --git a/Include/pyport.h b/Include/pyport.h index b3ff2f4882e9..b1b2a7477969 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -698,6 +698,15 @@ extern char * _getpty(int *, int, mode_t, int); # define _Py__has_builtin(x) 0 #endif +// _Py_TYPEOF(expr) gets the type of an expression. +// +// Example: _Py_TYPEOF(x) x_copy = (x); +// +// The macro is only defined if GCC or clang compiler is used. +#if defined(__GNUC__) || defined(__clang__) +# define _Py_TYPEOF(expr) __typeof__(expr) +#endif + /* A convenient way for code to know if sanitizers are enabled. */ #if defined(__has_feature) diff --git a/Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst b/Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst new file mode 100644 index 000000000000..e6c26cde168d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst @@ -0,0 +1,3 @@ +The :c:macro:`Py_CLEAR`, :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros +now only evaluate their arguments once. If an argument has side effects, these +side effects are no longer duplicated. Patch by Victor Stinner. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3617fafe9b4f..83eef73a875d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2589,6 +2589,91 @@ test_set_type_size(PyObject *self, PyObject *Py_UNUSED(ignored)) } +// Test Py_CLEAR() macro +static PyObject* +test_py_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // simple case with a variable + PyObject *obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + Py_CLEAR(obj); + assert(obj == NULL); + + // gh-98724: complex case, Py_CLEAR() argument has a side effect + PyObject* array[1]; + array[0] = PyList_New(0); + if (array[0] == NULL) { + return NULL; + } + + PyObject **p = array; + Py_CLEAR(*p++); + assert(array[0] == NULL); + assert(p == array + 1); + + Py_RETURN_NONE; +} + + +// Test Py_SETREF() and Py_XSETREF() macros, similar to test_py_clear() +static PyObject* +test_py_setref(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // Py_SETREF() simple case with a variable + PyObject *obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + Py_SETREF(obj, NULL); + assert(obj == NULL); + + // Py_XSETREF() simple case with a variable + PyObject *obj2 = PyList_New(0); + if (obj2 == NULL) { + return NULL; + } + Py_XSETREF(obj2, NULL); + assert(obj2 == NULL); + // test Py_XSETREF() when the argument is NULL + Py_XSETREF(obj2, NULL); + assert(obj2 == NULL); + + // gh-98724: complex case, Py_SETREF() argument has a side effect + PyObject* array[1]; + array[0] = PyList_New(0); + if (array[0] == NULL) { + return NULL; + } + + PyObject **p = array; + Py_SETREF(*p++, NULL); + assert(array[0] == NULL); + assert(p == array + 1); + + // gh-98724: complex case, Py_XSETREF() argument has a side effect + PyObject* array2[1]; + array2[0] = PyList_New(0); + if (array2[0] == NULL) { + return NULL; + } + + PyObject **p2 = array2; + Py_XSETREF(*p2++, NULL); + assert(array2[0] == NULL); + assert(p2 == array2 + 1); + + // test Py_XSETREF() when the argument is NULL + p2 = array2; + Py_XSETREF(*p2++, NULL); + assert(array2[0] == NULL); + assert(p2 == array2 + 1); + + Py_RETURN_NONE; +} + + #define TEST_REFCOUNT() \ do { \ PyObject *obj = PyList_New(0); \ @@ -3252,6 +3337,8 @@ static PyMethodDef TestMethods[] = { {"pynumber_tobase", pynumber_tobase, METH_VARARGS}, {"without_gc", without_gc, METH_O}, {"test_set_type_size", test_set_type_size, METH_NOARGS}, + {"test_py_clear", test_py_clear, METH_NOARGS}, + {"test_py_setref", test_py_setref, METH_NOARGS}, {"test_refcount_macros", test_refcount_macros, METH_NOARGS}, {"test_refcount_funcs", test_refcount_funcs, METH_NOARGS}, {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, From webhook-mailer at python.org Wed Dec 7 11:55:43 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 07 Dec 2022 16:55:43 -0000 Subject: [Python-checkins] gh-83035: handle decorator with nested parens in inspect.getsource (#99654) Message-ID: <mailman.2842.1670432145.3313.python-checkins@python.org> https://github.com/python/cpython/commit/68e41295b8611a990de68f15c89f1eb3dea51867 commit: 68e41295b8611a990de68f15c89f1eb3dea51867 branch: main author: Carl Meyer <carl at oddbird.net> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-07T08:55:12-08:00 summary: gh-83035: handle decorator with nested parens in inspect.getsource (#99654) files: A Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst M Lib/inspect.py M Lib/test/inspect_fodder2.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 31ac888126b5..e165937e448a 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1160,7 +1160,6 @@ def __init__(self): self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 self.body_col0 = None @@ -1175,13 +1174,6 @@ def tokeneater(self, type, token, srowcol, erowcol, line): self.islambda = True self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] @@ -1189,7 +1181,7 @@ def tokeneater(self, type, token, srowcol, erowcol, line): raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass diff --git a/Lib/test/inspect_fodder2.py b/Lib/test/inspect_fodder2.py index e7d4b53ebefc..2dc49817087c 100644 --- a/Lib/test/inspect_fodder2.py +++ b/Lib/test/inspect_fodder2.py @@ -259,3 +259,17 @@ def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): #line 259 def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + + at deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 3f5c299ce681..2b7977b1648f 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -886,6 +886,12 @@ def test_class(self): self.assertSourceEqual(self.fodderModule.X, 1, 2) +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + class _BrokenDataDescriptor(object): """ A broken data descriptor. See bug #1785. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst new file mode 100644 index 000000000000..629d9aefb2d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst @@ -0,0 +1 @@ +Fix :func:`inspect.getsource` handling of decorator calls with nested parentheses. From webhook-mailer at python.org Wed Dec 7 12:25:03 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 07 Dec 2022 17:25:03 -0000 Subject: [Python-checkins] gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) Message-ID: <mailman.2843.1670433904.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fe7c309a8c5d8abedd97dd271e94eb2c0f858c12 commit: fe7c309a8c5d8abedd97dd271e94eb2c0f858c12 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T09:24:41-08:00 summary: gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) (cherry picked from commit 68e41295b8611a990de68f15c89f1eb3dea51867) Co-authored-by: Carl Meyer <carl at oddbird.net> files: A Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst M Lib/inspect.py M Lib/test/inspect_fodder2.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index ad16f5c46e3b..417371f5cf14 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1052,7 +1052,6 @@ def __init__(self): self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 self.body_col0 = None @@ -1067,13 +1066,6 @@ def tokeneater(self, type, token, srowcol, erowcol, line): self.islambda = True self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] @@ -1081,7 +1073,7 @@ def tokeneater(self, type, token, srowcol, erowcol, line): raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass diff --git a/Lib/test/inspect_fodder2.py b/Lib/test/inspect_fodder2.py index e7d4b53ebefc..2dc49817087c 100644 --- a/Lib/test/inspect_fodder2.py +++ b/Lib/test/inspect_fodder2.py @@ -259,3 +259,17 @@ def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): #line 259 def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + + at deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 03cb3bdddc2c..5aa4a60de0bf 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -825,6 +825,12 @@ def test_class(self): self.assertSourceEqual(self.fodderModule.X, 1, 2) +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + class _BrokenDataDescriptor(object): """ A broken data descriptor. See bug #1785. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst new file mode 100644 index 000000000000..629d9aefb2d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst @@ -0,0 +1 @@ +Fix :func:`inspect.getsource` handling of decorator calls with nested parentheses. From webhook-mailer at python.org Wed Dec 7 12:26:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 07 Dec 2022 17:26:17 -0000 Subject: [Python-checkins] gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) Message-ID: <mailman.2844.1670433979.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2997f3913a7f644352a1d4013588451837d2ac58 commit: 2997f3913a7f644352a1d4013588451837d2ac58 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T09:26:11-08:00 summary: gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) (cherry picked from commit 68e41295b8611a990de68f15c89f1eb3dea51867) Co-authored-by: Carl Meyer <carl at oddbird.net> files: A Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst M Lib/inspect.py M Lib/test/inspect_fodder2.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index d1a9daf28913..cab92b6b8fb8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1175,7 +1175,6 @@ def __init__(self): self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 self.body_col0 = None @@ -1190,13 +1189,6 @@ def tokeneater(self, type, token, srowcol, erowcol, line): self.islambda = True self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] @@ -1204,7 +1196,7 @@ def tokeneater(self, type, token, srowcol, erowcol, line): raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass diff --git a/Lib/test/inspect_fodder2.py b/Lib/test/inspect_fodder2.py index e7d4b53ebefc..2dc49817087c 100644 --- a/Lib/test/inspect_fodder2.py +++ b/Lib/test/inspect_fodder2.py @@ -259,3 +259,17 @@ def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): #line 259 def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + + at deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index a54e6eb53e68..6c5e674aa69f 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -886,6 +886,12 @@ def test_class(self): self.assertSourceEqual(self.fodderModule.X, 1, 2) +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + class _BrokenDataDescriptor(object): """ A broken data descriptor. See bug #1785. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst new file mode 100644 index 000000000000..629d9aefb2d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst @@ -0,0 +1 @@ +Fix :func:`inspect.getsource` handling of decorator calls with nested parentheses. From webhook-mailer at python.org Wed Dec 7 13:09:34 2022 From: webhook-mailer at python.org (iritkatriel) Date: Wed, 07 Dec 2022 18:09:34 -0000 Subject: [Python-checkins] gh-100077: make test_code.test_invalid_bytecode more robust and maintainable (#100078) Message-ID: <mailman.2845.1670436575.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f3e97c90ed6f82fce67b0e8757eec54908ba49ce commit: f3e97c90ed6f82fce67b0e8757eec54908ba49ce branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-07T18:09:05Z summary: gh-100077: make test_code.test_invalid_bytecode more robust and maintainable (#100078) files: M Lib/test/test_code.py diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 4e4d82314a9f..02ab8fbcdb07 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -143,7 +143,7 @@ gc_collect) from test.support.script_helper import assert_python_ok from test.support import threading_helper -from opcode import opmap +from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -339,15 +339,19 @@ def func(): self.assertEqual(list(new_code.co_lines()), []) def test_invalid_bytecode(self): - def foo(): pass - foo.__code__ = co = foo.__code__.replace(co_code=b'\xee\x00d\x00S\x00') + def foo(): + pass - with self.assertRaises(SystemError) as se: - foo() - self.assertEqual( - f"{co.co_filename}:{co.co_firstlineno}: unknown opcode 238", - str(se.exception)) + # assert that opcode 238 is invalid + self.assertEqual(opname[238], '<238>') + # change first opcode to 0xee (=238) + foo.__code__ = foo.__code__.replace( + co_code=b'\xee' + foo.__code__.co_code[1:]) + + msg = f"unknown opcode 238" + with self.assertRaisesRegex(SystemError, msg): + foo() @requires_debug_ranges() def test_co_positions_artificial_instructions(self): From webhook-mailer at python.org Wed Dec 7 14:08:01 2022 From: webhook-mailer at python.org (hugovk) Date: Wed, 07 Dec 2022 19:08:01 -0000 Subject: [Python-checkins] gh-100072: only trigger netlify builds for doc changes (#100074) Message-ID: <mailman.2846.1670440082.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d92407ed497e3fc5acacb0294ab6095013e600f4 commit: d92407ed497e3fc5acacb0294ab6095013e600f4 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-07T21:07:30+02:00 summary: gh-100072: only trigger netlify builds for doc changes (#100074) Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> files: M netlify.toml diff --git a/netlify.toml b/netlify.toml index 52675b3d701e..f5790fc5fec7 100644 --- a/netlify.toml +++ b/netlify.toml @@ -2,7 +2,10 @@ base = "Doc/" command = "make html" publish = "build/html" + # Do not trigger netlify builds if docs were not changed. + # Changed files should be in sync with `.github/workflows/doc.yml` + ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF . ../netlify.toml" [build.environment] PYTHON_VERSION = "3.8" - IS_DEPLOYMENT_PREVIEW = "true" \ No newline at end of file + IS_DEPLOYMENT_PREVIEW = "true" From webhook-mailer at python.org Wed Dec 7 17:02:58 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Wed, 07 Dec 2022 22:02:58 -0000 Subject: [Python-checkins] gh-90110: Clean Up the C-analyzer Globals Lists (gh-100091) Message-ID: <mailman.2847.1670450580.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d47ffeb9e35dbc7264ffa12fddaa6e0d3ba767a4 commit: d47ffeb9e35dbc7264ffa12fddaa6e0d3ba767a4 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-07T15:02:47-07:00 summary: gh-90110: Clean Up the C-analyzer Globals Lists (gh-100091) https://github.com/python/cpython/issues/90110 files: M Parser/action_helpers.c M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv M Tools/c-analyzer/table-file.py diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 27c093332f67..f12dad095aca 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -13,6 +13,7 @@ void * _PyPegen_dummy_name(Parser *p, ...) { // XXX This leaks memory from the initial arena. + // Use a statically allocated variable instead of a pointer? static void *cache = NULL; if (cache != NULL) { @@ -1287,4 +1288,4 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq _PyPegen_get_last_comprehension_item(last_comprehension), "Generator expression must be parenthesized" ); -} \ No newline at end of file +} diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index adbb319a1b6f..af5b7eb8eaf0 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -4,10 +4,10 @@ filename funcname name reason # These are all variables that we will be making non-global. ################################## -# global objects to fix in core code +## global objects to fix in core code -#----------------------- -# exported builtin types (C-API) +##----------------------- +## exported builtin types (C-API) Objects/boolobject.c - PyBool_Type - Objects/bytearrayobject.c - PyByteArrayIter_Type - @@ -102,8 +102,8 @@ Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - Python/traceback.c - PyTraceBack_Type - -#----------------------- -# other exported builtin types +##----------------------- +## other exported builtin types # Not in a .h file: Objects/codeobject.c - _PyLineIterator - @@ -126,8 +126,8 @@ Python/hamt.c - _PyHamt_CollisionNode_Type - Python/hamt.c - _PyHamt_Type - Python/symtable.c - PySTEntry_Type - -#----------------------- -# private static builtin types +##----------------------- +## private static builtin types Objects/setobject.c - _PySetDummy_Type - Objects/stringlib/unicode_format.h - PyFormatterIter_Type - @@ -136,8 +136,8 @@ Objects/unicodeobject.c - EncodingMapType - #Objects/unicodeobject.c - PyFieldNameIter_Type - #Objects/unicodeobject.c - PyFormatterIter_Type - -#----------------------- -# static builtin structseq +##----------------------- +## static builtin structseq Objects/floatobject.c - FloatInfoType - Objects/longobject.c - Int_InfoType - @@ -148,8 +148,8 @@ Python/sysmodule.c - Hash_InfoType - Python/sysmodule.c - VersionInfoType - Python/thread.c - ThreadInfoType - -#----------------------- -# builtin exception types +##----------------------- +## builtin exception types Objects/exceptions.c - _PyExc_BaseException - Objects/exceptions.c - _PyExc_BaseExceptionGroup - @@ -286,8 +286,8 @@ Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - -#----------------------- -# singletons +##----------------------- +## singletons Objects/boolobject.c - _Py_FalseStruct - Objects/boolobject.c - _Py_TrueStruct - @@ -300,26 +300,26 @@ Objects/sliceobject.c - _Py_EllipsisObject - ################################## -# global non-objects to fix in core code +## global non-objects to fix in core code -#----------------------- -# effectively-const but initialized lazily +##----------------------- +## effectively-const but initialized lazily -# idempotent +## idempotent Python/dtoa.c - p5s - Objects/obmalloc.c new_arena debug_stats - -# others +## others Python/perf_trampoline.c - perf_map_file - Objects/unicodeobject.c - ucnhash_capi - -#----------------------- -# state +##----------------------- +## state -# local buffer +## local buffer Python/suggestions.c levenshtein_distance buffer - -# other +## other Objects/object.c - _Py_RefTotal - Python/perf_trampoline.c - perf_status - Python/perf_trampoline.c - extra_code_index - @@ -329,10 +329,10 @@ Python/thread_pthread_stubs.h - py_tls_entries - ################################## -# global objects to fix in builtin modules +## global objects to fix in builtin modules -#----------------------- -# static types +##----------------------- +## static types Modules/_collectionsmodule.c - defdict_type - Modules/_collectionsmodule.c - deque_type - @@ -381,10 +381,10 @@ Modules/itertoolsmodule.c - tee_type - Modules/itertoolsmodule.c - teedataobject_type - Modules/itertoolsmodule.c - ziplongest_type - -#----------------------- -# other +##----------------------- +## other -# state +## state Modules/faulthandler.c - fatal_error - Modules/faulthandler.c - thread - Modules/faulthandler.c - user_signals - @@ -393,10 +393,10 @@ Modules/faulthandler.c - old_stack - ################################## -# global non-objects to fix in builtin modules +## global non-objects to fix in builtin modules -#----------------------- -# initialized once +##----------------------- +## initialized once Modules/_io/bufferedio.c _PyIO_trap_eintr eintr_int - Modules/posixmodule.c os_dup2_impl dup3_works - @@ -405,8 +405,8 @@ Modules/posixmodule.c - ticks_per_second - Modules/timemodule.c _PyTime_GetClockWithInfo initialized - Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - -#----------------------- -# state +##----------------------- +## state Modules/_tracemalloc.c - allocators - Modules/_tracemalloc.c - tables_lock - @@ -420,7 +420,6 @@ Modules/_tracemalloc.c - tracemalloc_traces - Modules/_tracemalloc.c - tracemalloc_domains - Modules/_tracemalloc.c - tracemalloc_reentrant_key - Modules/faulthandler.c faulthandler_dump_traceback reentrant - -Modules/posixmodule.c - environ - Modules/signalmodule.c - is_tripped - Modules/signalmodule.c - signal_global_state - Modules/signalmodule.c - wakeup - @@ -428,10 +427,10 @@ Modules/signalmodule.c - Handlers - ################################## -# global objects to fix in extension modules +## global objects to fix in extension modules -#----------------------- -# static types +##----------------------- +## static types Modules/_asynciomodule.c - FutureIterType - Modules/_asynciomodule.c - FutureType - @@ -506,10 +505,10 @@ Modules/xxmodule.c - Xxo_Type - Modules/xxsubtype.c - spamdict_type - Modules/xxsubtype.c - spamlist_type - -#----------------------- -# non-static types - initialized once +##----------------------- +## non-static types - initialized once -# heap types +## heap types Modules/_decimal/_decimal.c - DecimalTuple - Modules/_decimal/_decimal.c - PyDecSignalDict_Type - Modules/_tkinter.c - PyTclObject_Type - @@ -517,7 +516,7 @@ Modules/_tkinter.c - Tkapp_Type - Modules/_tkinter.c - Tktt_Type - Modules/xxlimited_35.c - Xxo_Type - -# exception types +## exception types Modules/_ctypes/_ctypes.c - PyExc_ArgError - Modules/_cursesmodule.c - PyCursesError - Modules/_decimal/_decimal.c - DecimalException - @@ -528,15 +527,15 @@ Modules/socketmodule.c - socket_gaierror - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - -#----------------------- -# cached - initialized once +##----------------------- +## cached - initialized once -# manually cached PyUnicodeOjbect +## manually cached PyUnicodeOjbect Modules/_asynciomodule.c - context_kwname - Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - Modules/_ctypes/_ctypes.c CreateSwappedType suffix - -# other - during module init +## other - during module init Modules/_asynciomodule.c - asyncio_mod - Modules/_asynciomodule.c - traceback_extract_stack - Modules/_asynciomodule.c - asyncio_future_repr_func - @@ -551,10 +550,10 @@ Modules/_zoneinfo.c - io_open - Modules/_zoneinfo.c - _tzpath_find_tzfile - Modules/_zoneinfo.c - _common_mod - -#----------------------- -# other +##----------------------- +## other -# initialized once +## initialized once Modules/_ctypes/_ctypes.c - _unpickle - Modules/_ctypes/_ctypes.c PyCArrayType_from_ctype cache - Modules/_cursesmodule.c - ModDict - @@ -577,7 +576,7 @@ Modules/_decimal/_decimal.c - Rational - Modules/_decimal/_decimal.c - SignalTuple - Modules/arraymodule.c array_array___reduce_ex___impl array_reconstructor - -# state +## state Modules/_asynciomodule.c - cached_running_holder - Modules/_asynciomodule.c - fi_freelist - Modules/_asynciomodule.c - fi_freelist_len - @@ -592,20 +591,19 @@ Modules/_tkinter.c - valInCmd - Modules/_tkinter.c - trbInCmd - Modules/_zoneinfo.c - TIMEDELTA_CACHE - Modules/_zoneinfo.c - ZONEINFO_WEAK_CACHE - -Modules/syslogmodule.c - S_ident_o - ################################## -# global non-objects to fix in extension modules +## global non-objects to fix in extension modules -#----------------------- -# initialized once +##----------------------- +## initialized once -# pre-allocated buffer +## pre-allocated buffer Modules/nismodule.c nisproc_maplist_2 res - Modules/pyexpat.c PyUnknownEncodingHandler template_buffer - -# other +## other Include/datetime.h - PyDateTimeAPI - Modules/_asynciomodule.c - module_initialized - Modules/_ctypes/cfield.c _ctypes_get_fielddesc initialized - @@ -650,8 +648,8 @@ Modules/readline.c - libedit_history_start - Modules/socketmodule.c - accept4_works - Modules/socketmodule.c - sock_cloexec_works - -#----------------------- -# state +##----------------------- +## state Modules/_asynciomodule.c - cached_running_holder_tsid - Modules/_asynciomodule.c - task_name_counter - @@ -681,4 +679,3 @@ Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - Modules/socketmodule.c - defaulttimeout - -Modules/syslogmodule.c - S_log_open - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 242deace8c94..128336a997eb 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -5,52 +5,61 @@ filename funcname name reason ################################## -# process-global resources +## process-global values - set once -# Initialization for these should be idempotent. +# These will never re-initialize (but would be idempotent). +# These are effectively const. -#----------------------- -# effectively const, set once before/during first init +##----------------------- +## process-global resources - set during first init -Modules/getbuildinfo.c - buildinfo - -Modules/getbuildinfo.c - initialized - -Python/getversion.c - initialized - -Python/getversion.c - version - - -#----------------------- -# effectively const, set once during first init +## indicators for resource availability/capability +Python/bootstrap_hash.c py_getrandom getrandom_works - +Python/fileutils.c - _Py_open_cloexec_works - +Python/fileutils.c set_inheritable ioctl_works - -Python/bootstrap_hash.c - _Py_HashSecret_Initialized - -Python/pyhash.c - _Py_HashSecret - +## resource init Python/thread.c - initialized - Python/thread_pthread.h - condattr_monotonic - - # safe static buffer used during one-time initialization Python/thread_pthread.h init_condattr ca - -# indicators for process-global resource availability/capability -Python/bootstrap_hash.c py_getrandom getrandom_works - -Python/fileutils.c - _Py_open_cloexec_works - -Python/fileutils.c set_inheritable ioctl_works - - -#----------------------- -# effectively const but set once lazily (*after* first init) +##----------------------- +## other values (not Python-specific) +## cached computed data - set lazily (*after* first init) +# XXX Are these safe relative to write races? Objects/longobject.c long_from_non_binary_base log_base_BASE - Objects/longobject.c long_from_non_binary_base convwidth_base - Objects/longobject.c long_from_non_binary_base convmultmax_base - Objects/unicodeobject.c - bloom_linebreak - +# This is safe: Objects/unicodeobject.c _init_global_state initialized - -# XXX Move to _PyRuntimeState? +##----------------------- +## other values (Python-specific) + +## internal state - set before/during first init +Modules/getbuildinfo.c - buildinfo - +Modules/getbuildinfo.c - initialized - +Python/getversion.c - initialized - +Python/getversion.c - version - + +## public C-API - set during first init +Python/bootstrap_hash.c - _Py_HashSecret_Initialized - +Python/pyhash.c - _Py_HashSecret - + +## internal state - set lazily (*after* first init) +# XXX Move to _PyRuntimeState (i.e. tie to init/fini cycle)? Parser/action_helpers.c _PyPegen_dummy_name cache - ################################## -# state tied to C main() (only in main thread) +## state tied to Py_Main() +# (only in main thread) -#----------------------- -# handling C argv +##----------------------- +## handling C argv Python/getopt.c - _PyOS_optarg - Python/getopt.c - _PyOS_opterr - @@ -58,8 +67,8 @@ Python/getopt.c - _PyOS_optind - Python/getopt.c - opt_ptr - Python/pathconfig.c - _Py_path_config - -#----------------------- -# REPL +##----------------------- +## REPL Parser/myreadline.c - _PyOS_ReadlineLock - Parser/myreadline.c - _PyOS_ReadlineTState - @@ -68,28 +77,29 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - ################################## -# state tied to each runtime init/fini cycle +## runtime-global values - set once with each init -Python/pylifecycle.c - _PyRuntime - -Python/pylifecycle.c - runtime_initialized - +# These are effectively const. -# All uses of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py. +##----------------------- +## set by embedders before init +# (whether directly or through a call) -#----------------------- -# effectively const once init finishes - -# set by embedders before init (whether directly or through a call) Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - +# XXX This only gets cleared by Py_Main(). Python/initconfig.c - orig_argv - -# deprecated +##----------------------- +## public C-API + +## deprecated Python/preconfig.c - Py_FileSystemDefaultEncoding - Python/preconfig.c - Py_HasFileSystemDefaultEncoding - Python/preconfig.c - Py_FileSystemDefaultEncodeErrors - Python/preconfig.c - _Py_HasFileSystemDefaultEncodeErrors - -# legacy config flags +## legacy config flags Python/initconfig.c - Py_UTF8Mode - Python/initconfig.c - Py_DebugFlag - Python/initconfig.c - Py_VerboseFlag - @@ -109,373 +119,70 @@ Python/initconfig.c - Py_IsolatedFlag - Python/initconfig.c - Py_LegacyWindowsFSEncodingFlag - Python/initconfig.c - Py_LegacyWindowsStdioFlag - -# initialized statically, customized by embedders +##----------------------- +## initialized statically, may be customized by embedders + Python/frozen.c - PyImport_FrozenModules - Python/import.c - inittab_copy - Python/import.c - PyImport_Inittab - -# used temporarily during init -Python/sysmodule.c - _preinit_warnoptions - -Python/sysmodule.c - _preinit_xoptions - - ################################## -# special-use diagnistic state +## runtime-global state -Parser/pegen.c - memo_statistics - +##----------------------- +## tied to each init/fini cycle +## the consolidated runtime state +Python/pylifecycle.c - _PyRuntime - +Python/pylifecycle.c - runtime_initialized - -################################## -# one-off temporary state +# All cases of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py. -# This is safe enough. -Python/pylifecycle.c _Py_FatalErrorFormat reentrant - -Python/pylifecycle.c fatal_error reentrant - +## main interp state in stdlib modules +Modules/syslogmodule.c - S_ident_o - +Modules/syslogmodule.c - S_log_open - +##----------------------- +## *not* tied to init/fini cycle -################################## -# not used (kept for compatibility) +# These do not ge reset with each init/fini cycle. +# XXX These should probably be tied to init/fini. Move to _PyRuntimeState? -Python/pyfpe.c - PyFPE_counter - +# special-use diagnistic state +Parser/pegen.c - memo_statistics - + +##----------------------- +## one-off temporary state + +# used during runtime init +Python/sysmodule.c - _preinit_warnoptions - +Python/sysmodule.c - _preinit_xoptions - + +# thread-safety +# XXX need race protection? +Python/pylifecycle.c _Py_FatalErrorFormat reentrant - +Python/pylifecycle.c fatal_error reentrant - ################################## -# The analyzer should have ignored these. -# XXX Fix the analyzer. +## not significant -Modules/_io/_iomodule.c - _PyIO_Module - -Modules/_sqlite/module.c - _sqlite3module - +##----------------------- +## not used (kept for compatibility) -# forward/extern references +Python/pyfpe.c - PyFPE_counter - -Include/py_curses.h - PyCurses_API - -Include/pydecimal.h - _decimal_api - -Modules/_blake2/blake2module.c - blake2b_type_spec - -Modules/_blake2/blake2module.c - blake2s_type_spec - -Modules/_io/fileio.c - _Py_open_cloexec_works - -Modules/_io/_iomodule.h - PyIOBase_Type - -Modules/_io/_iomodule.h - PyRawIOBase_Type - -Modules/_io/_iomodule.h - PyBufferedIOBase_Type - -Modules/_io/_iomodule.h - PyTextIOBase_Type - -Modules/_io/_iomodule.h - PyFileIO_Type - -Modules/_io/_iomodule.h - PyBytesIO_Type - -Modules/_io/_iomodule.h - PyStringIO_Type - -Modules/_io/_iomodule.h - PyBufferedReader_Type - -Modules/_io/_iomodule.h - PyBufferedWriter_Type - -Modules/_io/_iomodule.h - PyBufferedRWPair_Type - -Modules/_io/_iomodule.h - PyBufferedRandom_Type - -Modules/_io/_iomodule.h - PyTextIOWrapper_Type - -Modules/_io/_iomodule.h - PyIncrementalNewlineDecoder_Type - -Modules/_io/_iomodule.h - _PyBytesIOBuffer_Type - -Modules/_io/_iomodule.h - _PyIO_Module - -Modules/_io/_iomodule.h - _PyIO_str_close - -Modules/_io/_iomodule.h - _PyIO_str_closed - -Modules/_io/_iomodule.h - _PyIO_str_decode - -Modules/_io/_iomodule.h - _PyIO_str_encode - -Modules/_io/_iomodule.h - _PyIO_str_fileno - -Modules/_io/_iomodule.h - _PyIO_str_flush - -Modules/_io/_iomodule.h - _PyIO_str_getstate - -Modules/_io/_iomodule.h - _PyIO_str_isatty - -Modules/_io/_iomodule.h - _PyIO_str_newlines - -Modules/_io/_iomodule.h - _PyIO_str_nl - -Modules/_io/_iomodule.h - _PyIO_str_peek - -Modules/_io/_iomodule.h - _PyIO_str_read - -Modules/_io/_iomodule.h - _PyIO_str_read1 - -Modules/_io/_iomodule.h - _PyIO_str_readable - -Modules/_io/_iomodule.h - _PyIO_str_readall - -Modules/_io/_iomodule.h - _PyIO_str_readinto - -Modules/_io/_iomodule.h - _PyIO_str_readline - -Modules/_io/_iomodule.h - _PyIO_str_reset - -Modules/_io/_iomodule.h - _PyIO_str_seek - -Modules/_io/_iomodule.h - _PyIO_str_seekable - -Modules/_io/_iomodule.h - _PyIO_str_setstate - -Modules/_io/_iomodule.h - _PyIO_str_tell - -Modules/_io/_iomodule.h - _PyIO_str_truncate - -Modules/_io/_iomodule.h - _PyIO_str_writable - -Modules/_io/_iomodule.h - _PyIO_str_write - -Modules/_io/_iomodule.h - _PyIO_empty_str - -Modules/_io/_iomodule.h - _PyIO_empty_bytes - -Modules/_multiprocessing/multiprocessing.h - _PyMp_SemLockType - -Modules/_sqlite/module.c - _pysqlite_converters - -Modules/_sqlite/module.c - _pysqlite_enable_callback_tracebacks - -Modules/_sqlite/module.c - pysqlite_BaseTypeAdapted - -Modules/_sqlite/module.h - pysqlite_global_state - -Modules/_testcapimodule.c - _PyBytesIOBuffer_Type - -Modules/posixmodule.c - _Py_open_cloexec_works - -Objects/object.c - _Py_GenericAliasIterType - -Objects/object.c - _PyMemoryIter_Type - -Objects/object.c - _PyLineIterator - -Objects/object.c - _PyPositionsIterator - -Python/perf_trampoline.c - _Py_trampoline_func_start - -Python/perf_trampoline.c - _Py_trampoline_func_end - -Python/importdl.h - _PyImport_DynLoadFiletab - +##----------------------- +## should be const +# XXX Make them const. -Modules/expat/xmlrole.c - prolog0 - -Modules/expat/xmlrole.c - prolog1 - -Modules/expat/xmlrole.c - prolog2 - -Modules/expat/xmlrole.c - doctype0 - -Modules/expat/xmlrole.c - doctype1 - -Modules/expat/xmlrole.c - doctype2 - -Modules/expat/xmlrole.c - doctype3 - -Modules/expat/xmlrole.c - doctype4 - -Modules/expat/xmlrole.c - doctype5 - -Modules/expat/xmlrole.c - internalSubset - -Modules/expat/xmlrole.c - entity0 - -Modules/expat/xmlrole.c - entity1 - -Modules/expat/xmlrole.c - entity2 - -Modules/expat/xmlrole.c - entity3 - -Modules/expat/xmlrole.c - entity4 - -Modules/expat/xmlrole.c - entity5 - -Modules/expat/xmlrole.c - entity6 - -Modules/expat/xmlrole.c - entity7 - -Modules/expat/xmlrole.c - entity8 - -Modules/expat/xmlrole.c - entity9 - -Modules/expat/xmlrole.c - entity10 - -Modules/expat/xmlrole.c - notation0 - -Modules/expat/xmlrole.c - notation1 - -Modules/expat/xmlrole.c - notation2 - -Modules/expat/xmlrole.c - notation3 - -Modules/expat/xmlrole.c - notation4 - -Modules/expat/xmlrole.c - attlist0 - -Modules/expat/xmlrole.c - attlist1 - -Modules/expat/xmlrole.c - attlist2 - -Modules/expat/xmlrole.c - attlist3 - -Modules/expat/xmlrole.c - attlist4 - -Modules/expat/xmlrole.c - attlist5 - -Modules/expat/xmlrole.c - attlist6 - -Modules/expat/xmlrole.c - attlist7 - -Modules/expat/xmlrole.c - attlist8 - -Modules/expat/xmlrole.c - attlist9 - -Modules/expat/xmlrole.c - element0 - -Modules/expat/xmlrole.c - element1 - -Modules/expat/xmlrole.c - element2 - -Modules/expat/xmlrole.c - element3 - -Modules/expat/xmlrole.c - element4 - -Modules/expat/xmlrole.c - element5 - -Modules/expat/xmlrole.c - element6 - -Modules/expat/xmlrole.c - element7 - -Modules/expat/xmlrole.c - externalSubset0 - -Modules/expat/xmlrole.c - externalSubset1 - -Modules/expat/xmlrole.c - condSect0 - -Modules/expat/xmlrole.c - condSect1 - -Modules/expat/xmlrole.c - condSect2 - -Modules/expat/xmlrole.c - declClose - -Modules/expat/xmlrole.c - error - +# These are all variables that we will be leaving global. +# All module defs, type defs, etc. are handled in c-analyzr/cpython/_analyzer.py. +# All kwlist arrays are handled in c-analyzr/cpython/_analyzer.py. -################################## -# test code - -Modules/_ctypes/_ctypes_test.c - _ctypes_test_slots - -Modules/_ctypes/_ctypes_test.c - _ctypes_testmodule - -Modules/_ctypes/_ctypes_test.c - _xxx_lib - -Modules/_ctypes/_ctypes_test.c - an_integer - -Modules/_ctypes/_ctypes_test.c - bottom - -Modules/_ctypes/_ctypes_test.c - last_tf_arg_s - -Modules/_ctypes/_ctypes_test.c - last_tf_arg_u - -Modules/_ctypes/_ctypes_test.c - last_tfrsuv_arg - -Modules/_ctypes/_ctypes_test.c - left - -Modules/_ctypes/_ctypes_test.c - module_methods - -Modules/_ctypes/_ctypes_test.c - my_eggs - -Modules/_ctypes/_ctypes_test.c - my_spams - -Modules/_ctypes/_ctypes_test.c - right - -Modules/_ctypes/_ctypes_test.c - top - -Modules/_testbuffer.c - NDArray_Type - -Modules/_testbuffer.c - StaticArray_Type - -Modules/_testbuffer.c - Struct - -Modules/_testbuffer.c - _testbuffer_functions - -Modules/_testbuffer.c - _testbuffermodule - -Modules/_testbuffer.c - calcsize - -Modules/_testbuffer.c - infobuf - -Modules/_testbuffer.c - ndarray_as_buffer - -Modules/_testbuffer.c - ndarray_as_mapping - -Modules/_testbuffer.c - ndarray_as_sequence - -Modules/_testbuffer.c - ndarray_getset - -Modules/_testbuffer.c - ndarray_methods - -Modules/_testbuffer.c - simple_fmt - -Modules/_testbuffer.c - simple_format - -Modules/_testbuffer.c - static_buffer - -Modules/_testbuffer.c - static_mem - -Modules/_testbuffer.c - static_shape - -Modules/_testbuffer.c - static_strides - -Modules/_testbuffer.c - staticarray_as_buffer - -Modules/_testbuffer.c - structmodule - -Modules/_testbuffer.c ndarray_init kwlist - -Modules/_testbuffer.c ndarray_memoryview_from_buffer format - -Modules/_testbuffer.c ndarray_memoryview_from_buffer info - -Modules/_testbuffer.c ndarray_memoryview_from_buffer shape - -Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - -Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - -Modules/_testbuffer.c ndarray_push kwlist - -Modules/_testbuffer.c staticarray_init kwlist - -Modules/_testcapi/heaptype.c - _testcapimodule - -Modules/_testcapi/unicode.c - _testcapimodule - -Modules/_testcapimodule.c - ContainerNoGC_members - -Modules/_testcapimodule.c - ContainerNoGC_type - -Modules/_testcapimodule.c - FmData - -Modules/_testcapimodule.c - FmHook - -Modules/_testcapimodule.c - GenericAlias_Type - -Modules/_testcapimodule.c - Generic_Type - -Modules/_testcapimodule.c - HeapCTypeSetattr_slots - -Modules/_testcapimodule.c - HeapCTypeSetattr_spec - -Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_slots - -Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_spec - -Modules/_testcapimodule.c - HeapCTypeSubclass_slots - -Modules/_testcapimodule.c - HeapCTypeSubclass_spec - -Modules/_testcapimodule.c - HeapCTypeWithBuffer_slots - -Modules/_testcapimodule.c - HeapCTypeWithBuffer_spec - -Modules/_testcapimodule.c - HeapCTypeWithDict_slots - -Modules/_testcapimodule.c - HeapCTypeWithDict_spec - -Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_slots - -Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_spec - -Modules/_testcapimodule.c - HeapCTypeWithWeakref_slots - -Modules/_testcapimodule.c - HeapCTypeWithWeakref_spec - -Modules/_testcapimodule.c - HeapCType_slots - -Modules/_testcapimodule.c - HeapCType_spec - -Modules/_testcapimodule.c - HeapDocCType_slots - -Modules/_testcapimodule.c - HeapDocCType_spec - -Modules/_testcapimodule.c - HeapGcCType_slots - -Modules/_testcapimodule.c - HeapGcCType_spec - -Modules/_testcapimodule.c - MethClass_Type - -Modules/_testcapimodule.c - MethInstance_Type - -Modules/_testcapimodule.c - MethStatic_Type - -Modules/_testcapimodule.c - MethodDescriptor2_Type - -Modules/_testcapimodule.c - MethodDescriptorBase_Type - -Modules/_testcapimodule.c - MethodDescriptorDerived_Type - -Modules/_testcapimodule.c - MethodDescriptorNopGet_Type - -Modules/_testcapimodule.c - MyList_Type - -Modules/_testcapimodule.c - PyRecursingInfinitelyError_Type - -Modules/_testcapimodule.c - TestError - -Modules/_testcapimodule.c - TestMethods - -Modules/_testcapimodule.c - _HashInheritanceTester_Type - -Modules/_testcapimodule.c - _testcapimodule - -Modules/_testcapimodule.c - awaitType - -Modules/_testcapimodule.c - awaitType_as_async - -Modules/_testcapimodule.c - capsule_context - -Modules/_testcapimodule.c - capsule_destructor_call_count - -Modules/_testcapimodule.c - capsule_error - -Modules/_testcapimodule.c - capsule_name - -Modules/_testcapimodule.c - capsule_pointer - -Modules/_testcapimodule.c - decimal_initialized - -Modules/_testcapimodule.c - generic_alias_methods - -Modules/_testcapimodule.c - generic_methods - -Modules/_testcapimodule.c - heapctype_members - -Modules/_testcapimodule.c - heapctypesetattr_members - -Modules/_testcapimodule.c - heapctypesubclass_members - -Modules/_testcapimodule.c - heapctypewithdict_getsetlist - -Modules/_testcapimodule.c - heapctypewithdict_members - -Modules/_testcapimodule.c - heapctypewithnegativedict_members - -Modules/_testcapimodule.c - heapctypewithweakref_members - -Modules/_testcapimodule.c - ipowType - -Modules/_testcapimodule.c - ipowType_as_number - -Modules/_testcapimodule.c - matmulType - -Modules/_testcapimodule.c - matmulType_as_number - -Modules/_testcapimodule.c - meth_class_methods - -Modules/_testcapimodule.c - meth_instance_methods - -Modules/_testcapimodule.c - meth_static_methods - -Modules/_testcapimodule.c - ml - -Modules/_testcapimodule.c - str1 - -Modules/_testcapimodule.c - str2 - -Modules/_testcapimodule.c - test_members - -Modules/_testcapimodule.c - test_run_counter - -Modules/_testcapimodule.c - test_structmembersType - -Modules/_testcapimodule.c - thread_done - -Modules/_testcapimodule.c - x - -Modules/_testcapimodule.c getargs_keyword_only keywords - -Modules/_testcapimodule.c getargs_keywords keywords - -Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - -Modules/_testcapimodule.c getargs_s_hash_int2 keywords static char*[] -Modules/_testcapimodule.c make_exception_with_doc kwlist - -Modules/_testcapimodule.c raise_SIGINT_then_send_None PyId_send - -Modules/_testcapimodule.c slot_tp_del PyId___tp_del__ - -Modules/_testcapimodule.c test_capsule buffer - -Modules/_testcapimodule.c test_empty_argparse kwlist - -Modules/_testcapimodule.c test_structmembers_new keywords - -Modules/_testcapimodule.c getargs_s_hash_int keywords - -Modules/_testcapimodule.c - g_dict_watch_events - -Modules/_testcapimodule.c - g_dict_watchers_installed - -Modules/_testcapimodule.c - g_type_modified_events - -Modules/_testcapimodule.c - g_type_watchers_installed - -Modules/_testimportmultiple.c - _barmodule - -Modules/_testimportmultiple.c - _foomodule - -Modules/_testimportmultiple.c - _testimportmultiple - -Modules/_testinternalcapi.c - TestMethods - -Modules/_testinternalcapi.c - _testcapimodule - -Modules/_testmultiphase.c - Example_Type_slots - -Modules/_testmultiphase.c - Example_Type_spec - -Modules/_testmultiphase.c - Example_methods - -Modules/_testmultiphase.c - StateAccessType_Type_slots - -Modules/_testmultiphase.c - StateAccessType_methods - -Modules/_testmultiphase.c - StateAccessType_spec - -Modules/_testmultiphase.c - Str_Type_slots - -Modules/_testmultiphase.c - Str_Type_spec - -Modules/_testmultiphase.c - def_bad_large - -Modules/_testmultiphase.c - def_bad_negative - -Modules/_testmultiphase.c - def_create_int_with_state - -Modules/_testmultiphase.c - def_create_null - -Modules/_testmultiphase.c - def_create_raise - -Modules/_testmultiphase.c - def_create_unreported_exception - -Modules/_testmultiphase.c - def_exec_err - -Modules/_testmultiphase.c - def_exec_raise - -Modules/_testmultiphase.c - def_exec_unreported_exception - -Modules/_testmultiphase.c - def_meth_state_access - -Modules/_testmultiphase.c - def_negative_size - -Modules/_testmultiphase.c - def_nonascii_kana - -Modules/_testmultiphase.c - def_nonascii_latin - -Modules/_testmultiphase.c - def_nonmodule - -Modules/_testmultiphase.c - def_nonmodule_with_exec_slots - -Modules/_testmultiphase.c - def_nonmodule_with_methods - -Modules/_testmultiphase.c - imp_dummy_def - -Modules/_testmultiphase.c - main_def - -Modules/_testmultiphase.c - main_slots - -Modules/_testmultiphase.c - meth_state_access_slots - -Modules/_testmultiphase.c - nonmodule_methods - -Modules/_testmultiphase.c - null_slots_def - -Modules/_testmultiphase.c - slots_bad_large - -Modules/_testmultiphase.c - slots_bad_negative - -Modules/_testmultiphase.c - slots_create_nonmodule - -Modules/_testmultiphase.c - slots_create_nonmodule - -Modules/_testmultiphase.c - slots_create_null - -Modules/_testmultiphase.c - slots_create_raise - -Modules/_testmultiphase.c - slots_create_unreported_exception - -Modules/_testmultiphase.c - slots_exec_err - -Modules/_testmultiphase.c - slots_exec_raise - -Modules/_testmultiphase.c - slots_exec_unreported_exception - -Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - -Modules/_testmultiphase.c - testexport_methods - -Modules/_testmultiphase.c - uninitialized_def - -Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - -Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - -Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - -Modules/_xxtestfuzz/fuzzer.c - ast_literal_eval_method - -Modules/_xxtestfuzz/fuzzer.c - compiled_patterns - -Modules/_xxtestfuzz/fuzzer.c - csv_error - -Modules/_xxtestfuzz/fuzzer.c - csv_module - -Modules/_xxtestfuzz/fuzzer.c - json_loads_method - -Modules/_xxtestfuzz/fuzzer.c - regex_patterns - -Modules/_xxtestfuzz/fuzzer.c - sre_compile_method - -Modules/_xxtestfuzz/fuzzer.c - sre_error_exception - -Modules/_xxtestfuzz/fuzzer.c - struct_error - -Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_MATCH_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput STRUCT_UNPACK_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED - - - -################################## -# should be const -# XXX Make them const. - -# These are all variables that we will be leaving global. - -# All module defs, type defs, etc. are handled in c-analyzr/cpython/_analyzer.py. -# All kwlist arrays are handled in c-analyzr/cpython/_analyzer.py. - -#----------------------- -# other vars that are actually constant +# other vars that are actually constant Include/internal/pycore_blocks_output_buffer.h - BUFFER_BLOCK_SIZE - Modules/_csv.c - quote_styles - @@ -646,3 +353,327 @@ Python/stdlib_module_names.h - _Py_stdlib_module_names - Python/sysmodule.c - _PySys_ImplCacheTag - Python/sysmodule.c - _PySys_ImplName - Python/sysmodule.c - whatstrings - + +##----------------------- +## test code + +Modules/_ctypes/_ctypes_test.c - _ctypes_test_slots - +Modules/_ctypes/_ctypes_test.c - _ctypes_testmodule - +Modules/_ctypes/_ctypes_test.c - _xxx_lib - +Modules/_ctypes/_ctypes_test.c - an_integer - +Modules/_ctypes/_ctypes_test.c - bottom - +Modules/_ctypes/_ctypes_test.c - last_tf_arg_s - +Modules/_ctypes/_ctypes_test.c - last_tf_arg_u - +Modules/_ctypes/_ctypes_test.c - last_tfrsuv_arg - +Modules/_ctypes/_ctypes_test.c - left - +Modules/_ctypes/_ctypes_test.c - module_methods - +Modules/_ctypes/_ctypes_test.c - my_eggs - +Modules/_ctypes/_ctypes_test.c - my_spams - +Modules/_ctypes/_ctypes_test.c - right - +Modules/_ctypes/_ctypes_test.c - top - +Modules/_testbuffer.c - NDArray_Type - +Modules/_testbuffer.c - StaticArray_Type - +Modules/_testbuffer.c - Struct - +Modules/_testbuffer.c - _testbuffer_functions - +Modules/_testbuffer.c - _testbuffermodule - +Modules/_testbuffer.c - calcsize - +Modules/_testbuffer.c - infobuf - +Modules/_testbuffer.c - ndarray_as_buffer - +Modules/_testbuffer.c - ndarray_as_mapping - +Modules/_testbuffer.c - ndarray_as_sequence - +Modules/_testbuffer.c - ndarray_getset - +Modules/_testbuffer.c - ndarray_methods - +Modules/_testbuffer.c - simple_fmt - +Modules/_testbuffer.c - simple_format - +Modules/_testbuffer.c - static_buffer - +Modules/_testbuffer.c - static_mem - +Modules/_testbuffer.c - static_shape - +Modules/_testbuffer.c - static_strides - +Modules/_testbuffer.c - staticarray_as_buffer - +Modules/_testbuffer.c - structmodule - +Modules/_testbuffer.c ndarray_init kwlist - +Modules/_testbuffer.c ndarray_memoryview_from_buffer format - +Modules/_testbuffer.c ndarray_memoryview_from_buffer info - +Modules/_testbuffer.c ndarray_memoryview_from_buffer shape - +Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - +Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - +Modules/_testbuffer.c ndarray_push kwlist - +Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/heaptype.c - _testcapimodule - +Modules/_testcapi/unicode.c - _testcapimodule - +Modules/_testcapimodule.c - ContainerNoGC_members - +Modules/_testcapimodule.c - ContainerNoGC_type - +Modules/_testcapimodule.c - FmData - +Modules/_testcapimodule.c - FmHook - +Modules/_testcapimodule.c - GenericAlias_Type - +Modules/_testcapimodule.c - Generic_Type - +Modules/_testcapimodule.c - HeapCTypeSetattr_slots - +Modules/_testcapimodule.c - HeapCTypeSetattr_spec - +Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_slots - +Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_spec - +Modules/_testcapimodule.c - HeapCTypeSubclass_slots - +Modules/_testcapimodule.c - HeapCTypeSubclass_spec - +Modules/_testcapimodule.c - HeapCTypeWithBuffer_slots - +Modules/_testcapimodule.c - HeapCTypeWithBuffer_spec - +Modules/_testcapimodule.c - HeapCTypeWithDict_slots - +Modules/_testcapimodule.c - HeapCTypeWithDict_spec - +Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_slots - +Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_spec - +Modules/_testcapimodule.c - HeapCTypeWithWeakref_slots - +Modules/_testcapimodule.c - HeapCTypeWithWeakref_spec - +Modules/_testcapimodule.c - HeapCType_slots - +Modules/_testcapimodule.c - HeapCType_spec - +Modules/_testcapimodule.c - HeapDocCType_slots - +Modules/_testcapimodule.c - HeapDocCType_spec - +Modules/_testcapimodule.c - HeapGcCType_slots - +Modules/_testcapimodule.c - HeapGcCType_spec - +Modules/_testcapimodule.c - MethClass_Type - +Modules/_testcapimodule.c - MethInstance_Type - +Modules/_testcapimodule.c - MethStatic_Type - +Modules/_testcapimodule.c - MethodDescriptor2_Type - +Modules/_testcapimodule.c - MethodDescriptorBase_Type - +Modules/_testcapimodule.c - MethodDescriptorDerived_Type - +Modules/_testcapimodule.c - MethodDescriptorNopGet_Type - +Modules/_testcapimodule.c - MyList_Type - +Modules/_testcapimodule.c - PyRecursingInfinitelyError_Type - +Modules/_testcapimodule.c - TestError - +Modules/_testcapimodule.c - TestMethods - +Modules/_testcapimodule.c - _HashInheritanceTester_Type - +Modules/_testcapimodule.c - _testcapimodule - +Modules/_testcapimodule.c - awaitType - +Modules/_testcapimodule.c - awaitType_as_async - +Modules/_testcapimodule.c - capsule_context - +Modules/_testcapimodule.c - capsule_destructor_call_count - +Modules/_testcapimodule.c - capsule_error - +Modules/_testcapimodule.c - capsule_name - +Modules/_testcapimodule.c - capsule_pointer - +Modules/_testcapimodule.c - decimal_initialized - +Modules/_testcapimodule.c - generic_alias_methods - +Modules/_testcapimodule.c - generic_methods - +Modules/_testcapimodule.c - heapctype_members - +Modules/_testcapimodule.c - heapctypesetattr_members - +Modules/_testcapimodule.c - heapctypesubclass_members - +Modules/_testcapimodule.c - heapctypewithdict_getsetlist - +Modules/_testcapimodule.c - heapctypewithdict_members - +Modules/_testcapimodule.c - heapctypewithnegativedict_members - +Modules/_testcapimodule.c - heapctypewithweakref_members - +Modules/_testcapimodule.c - ipowType - +Modules/_testcapimodule.c - ipowType_as_number - +Modules/_testcapimodule.c - matmulType - +Modules/_testcapimodule.c - matmulType_as_number - +Modules/_testcapimodule.c - meth_class_methods - +Modules/_testcapimodule.c - meth_instance_methods - +Modules/_testcapimodule.c - meth_static_methods - +Modules/_testcapimodule.c - ml - +Modules/_testcapimodule.c - str1 - +Modules/_testcapimodule.c - str2 - +Modules/_testcapimodule.c - test_members - +Modules/_testcapimodule.c - test_run_counter - +Modules/_testcapimodule.c - test_structmembersType - +Modules/_testcapimodule.c - thread_done - +Modules/_testcapimodule.c - x - +Modules/_testcapimodule.c getargs_keyword_only keywords - +Modules/_testcapimodule.c getargs_keywords keywords - +Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - +Modules/_testcapimodule.c getargs_s_hash_int2 keywords static char*[] +Modules/_testcapimodule.c make_exception_with_doc kwlist - +Modules/_testcapimodule.c raise_SIGINT_then_send_None PyId_send - +Modules/_testcapimodule.c slot_tp_del PyId___tp_del__ - +Modules/_testcapimodule.c test_capsule buffer - +Modules/_testcapimodule.c test_empty_argparse kwlist - +Modules/_testcapimodule.c test_structmembers_new keywords - +Modules/_testcapimodule.c getargs_s_hash_int keywords - +Modules/_testcapimodule.c - g_dict_watch_events - +Modules/_testcapimodule.c - g_dict_watchers_installed - +Modules/_testcapimodule.c - g_type_modified_events - +Modules/_testcapimodule.c - g_type_watchers_installed - +Modules/_testimportmultiple.c - _barmodule - +Modules/_testimportmultiple.c - _foomodule - +Modules/_testimportmultiple.c - _testimportmultiple - +Modules/_testinternalcapi.c - TestMethods - +Modules/_testinternalcapi.c - _testcapimodule - +Modules/_testmultiphase.c - Example_Type_slots - +Modules/_testmultiphase.c - Example_Type_spec - +Modules/_testmultiphase.c - Example_methods - +Modules/_testmultiphase.c - StateAccessType_Type_slots - +Modules/_testmultiphase.c - StateAccessType_methods - +Modules/_testmultiphase.c - StateAccessType_spec - +Modules/_testmultiphase.c - Str_Type_slots - +Modules/_testmultiphase.c - Str_Type_spec - +Modules/_testmultiphase.c - def_bad_large - +Modules/_testmultiphase.c - def_bad_negative - +Modules/_testmultiphase.c - def_create_int_with_state - +Modules/_testmultiphase.c - def_create_null - +Modules/_testmultiphase.c - def_create_raise - +Modules/_testmultiphase.c - def_create_unreported_exception - +Modules/_testmultiphase.c - def_exec_err - +Modules/_testmultiphase.c - def_exec_raise - +Modules/_testmultiphase.c - def_exec_unreported_exception - +Modules/_testmultiphase.c - def_meth_state_access - +Modules/_testmultiphase.c - def_negative_size - +Modules/_testmultiphase.c - def_nonascii_kana - +Modules/_testmultiphase.c - def_nonascii_latin - +Modules/_testmultiphase.c - def_nonmodule - +Modules/_testmultiphase.c - def_nonmodule_with_exec_slots - +Modules/_testmultiphase.c - def_nonmodule_with_methods - +Modules/_testmultiphase.c - imp_dummy_def - +Modules/_testmultiphase.c - main_def - +Modules/_testmultiphase.c - main_slots - +Modules/_testmultiphase.c - meth_state_access_slots - +Modules/_testmultiphase.c - nonmodule_methods - +Modules/_testmultiphase.c - null_slots_def - +Modules/_testmultiphase.c - slots_bad_large - +Modules/_testmultiphase.c - slots_bad_negative - +Modules/_testmultiphase.c - slots_create_nonmodule - +Modules/_testmultiphase.c - slots_create_nonmodule - +Modules/_testmultiphase.c - slots_create_null - +Modules/_testmultiphase.c - slots_create_raise - +Modules/_testmultiphase.c - slots_create_unreported_exception - +Modules/_testmultiphase.c - slots_exec_err - +Modules/_testmultiphase.c - slots_exec_raise - +Modules/_testmultiphase.c - slots_exec_unreported_exception - +Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - +Modules/_testmultiphase.c - testexport_methods - +Modules/_testmultiphase.c - uninitialized_def - +Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - +Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - +Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - +Modules/_xxtestfuzz/fuzzer.c - ast_literal_eval_method - +Modules/_xxtestfuzz/fuzzer.c - compiled_patterns - +Modules/_xxtestfuzz/fuzzer.c - csv_error - +Modules/_xxtestfuzz/fuzzer.c - csv_module - +Modules/_xxtestfuzz/fuzzer.c - json_loads_method - +Modules/_xxtestfuzz/fuzzer.c - regex_patterns - +Modules/_xxtestfuzz/fuzzer.c - sre_compile_method - +Modules/_xxtestfuzz/fuzzer.c - sre_error_exception - +Modules/_xxtestfuzz/fuzzer.c - struct_error - +Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_MATCH_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput STRUCT_UNPACK_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED - + +##----------------------- +## the analyzer should have ignored these +# XXX Fix the analyzer. + +## forward/extern references +Include/py_curses.h - PyCurses_API - +Include/pydecimal.h - _decimal_api - +Modules/_blake2/blake2module.c - blake2b_type_spec - +Modules/_blake2/blake2module.c - blake2s_type_spec - +Modules/_io/fileio.c - _Py_open_cloexec_works - +Modules/_io/_iomodule.h - PyIOBase_Type - +Modules/_io/_iomodule.h - PyRawIOBase_Type - +Modules/_io/_iomodule.h - PyBufferedIOBase_Type - +Modules/_io/_iomodule.h - PyTextIOBase_Type - +Modules/_io/_iomodule.h - PyFileIO_Type - +Modules/_io/_iomodule.h - PyBytesIO_Type - +Modules/_io/_iomodule.h - PyStringIO_Type - +Modules/_io/_iomodule.h - PyBufferedReader_Type - +Modules/_io/_iomodule.h - PyBufferedWriter_Type - +Modules/_io/_iomodule.h - PyBufferedRWPair_Type - +Modules/_io/_iomodule.h - PyBufferedRandom_Type - +Modules/_io/_iomodule.h - PyTextIOWrapper_Type - +Modules/_io/_iomodule.h - PyIncrementalNewlineDecoder_Type - +Modules/_io/_iomodule.h - _PyBytesIOBuffer_Type - +Modules/_io/_iomodule.h - _PyIO_Module - +Modules/_io/_iomodule.h - _PyIO_str_close - +Modules/_io/_iomodule.h - _PyIO_str_closed - +Modules/_io/_iomodule.h - _PyIO_str_decode - +Modules/_io/_iomodule.h - _PyIO_str_encode - +Modules/_io/_iomodule.h - _PyIO_str_fileno - +Modules/_io/_iomodule.h - _PyIO_str_flush - +Modules/_io/_iomodule.h - _PyIO_str_getstate - +Modules/_io/_iomodule.h - _PyIO_str_isatty - +Modules/_io/_iomodule.h - _PyIO_str_newlines - +Modules/_io/_iomodule.h - _PyIO_str_nl - +Modules/_io/_iomodule.h - _PyIO_str_peek - +Modules/_io/_iomodule.h - _PyIO_str_read - +Modules/_io/_iomodule.h - _PyIO_str_read1 - +Modules/_io/_iomodule.h - _PyIO_str_readable - +Modules/_io/_iomodule.h - _PyIO_str_readall - +Modules/_io/_iomodule.h - _PyIO_str_readinto - +Modules/_io/_iomodule.h - _PyIO_str_readline - +Modules/_io/_iomodule.h - _PyIO_str_reset - +Modules/_io/_iomodule.h - _PyIO_str_seek - +Modules/_io/_iomodule.h - _PyIO_str_seekable - +Modules/_io/_iomodule.h - _PyIO_str_setstate - +Modules/_io/_iomodule.h - _PyIO_str_tell - +Modules/_io/_iomodule.h - _PyIO_str_truncate - +Modules/_io/_iomodule.h - _PyIO_str_writable - +Modules/_io/_iomodule.h - _PyIO_str_write - +Modules/_io/_iomodule.h - _PyIO_empty_str - +Modules/_io/_iomodule.h - _PyIO_empty_bytes - +Modules/_multiprocessing/multiprocessing.h - _PyMp_SemLockType - +Modules/_sqlite/module.c - _pysqlite_converters - +Modules/_sqlite/module.c - _pysqlite_enable_callback_tracebacks - +Modules/_sqlite/module.c - pysqlite_BaseTypeAdapted - +Modules/_sqlite/module.h - pysqlite_global_state - +Modules/_testcapimodule.c - _PyBytesIOBuffer_Type - +Modules/posixmodule.c - _Py_open_cloexec_works - +Modules/posixmodule.c - environ - +Objects/object.c - _Py_GenericAliasIterType - +Objects/object.c - _PyMemoryIter_Type - +Objects/object.c - _PyLineIterator - +Objects/object.c - _PyPositionsIterator - +Python/perf_trampoline.c - _Py_trampoline_func_start - +Python/perf_trampoline.c - _Py_trampoline_func_end - +Python/importdl.h - _PyImport_DynLoadFiletab - +Modules/expat/xmlrole.c - prolog0 - +Modules/expat/xmlrole.c - prolog1 - +Modules/expat/xmlrole.c - prolog2 - +Modules/expat/xmlrole.c - doctype0 - +Modules/expat/xmlrole.c - doctype1 - +Modules/expat/xmlrole.c - doctype2 - +Modules/expat/xmlrole.c - doctype3 - +Modules/expat/xmlrole.c - doctype4 - +Modules/expat/xmlrole.c - doctype5 - +Modules/expat/xmlrole.c - internalSubset - +Modules/expat/xmlrole.c - entity0 - +Modules/expat/xmlrole.c - entity1 - +Modules/expat/xmlrole.c - entity2 - +Modules/expat/xmlrole.c - entity3 - +Modules/expat/xmlrole.c - entity4 - +Modules/expat/xmlrole.c - entity5 - +Modules/expat/xmlrole.c - entity6 - +Modules/expat/xmlrole.c - entity7 - +Modules/expat/xmlrole.c - entity8 - +Modules/expat/xmlrole.c - entity9 - +Modules/expat/xmlrole.c - entity10 - +Modules/expat/xmlrole.c - notation0 - +Modules/expat/xmlrole.c - notation1 - +Modules/expat/xmlrole.c - notation2 - +Modules/expat/xmlrole.c - notation3 - +Modules/expat/xmlrole.c - notation4 - +Modules/expat/xmlrole.c - attlist0 - +Modules/expat/xmlrole.c - attlist1 - +Modules/expat/xmlrole.c - attlist2 - +Modules/expat/xmlrole.c - attlist3 - +Modules/expat/xmlrole.c - attlist4 - +Modules/expat/xmlrole.c - attlist5 - +Modules/expat/xmlrole.c - attlist6 - +Modules/expat/xmlrole.c - attlist7 - +Modules/expat/xmlrole.c - attlist8 - +Modules/expat/xmlrole.c - attlist9 - +Modules/expat/xmlrole.c - element0 - +Modules/expat/xmlrole.c - element1 - +Modules/expat/xmlrole.c - element2 - +Modules/expat/xmlrole.c - element3 - +Modules/expat/xmlrole.c - element4 - +Modules/expat/xmlrole.c - element5 - +Modules/expat/xmlrole.c - element6 - +Modules/expat/xmlrole.c - element7 - +Modules/expat/xmlrole.c - externalSubset0 - +Modules/expat/xmlrole.c - externalSubset1 - +Modules/expat/xmlrole.c - condSect0 - +Modules/expat/xmlrole.c - condSect1 - +Modules/expat/xmlrole.c - condSect2 - +Modules/expat/xmlrole.c - declClose - +Modules/expat/xmlrole.c - error - + +## other +Modules/_io/_iomodule.c - _PyIO_Module - +Modules/_sqlite/module.c - _sqlite3module - diff --git a/Tools/c-analyzer/table-file.py b/Tools/c-analyzer/table-file.py index 3cc05cc9de77..d36f814415c8 100644 --- a/Tools/c-analyzer/table-file.py +++ b/Tools/c-analyzer/table-file.py @@ -1,43 +1,59 @@ +KINDS = [ + 'section-major', + 'section-minor', + 'section-group', + 'row', +] + + def iter_clean_lines(lines): lines = iter(lines) - for line in lines: - line = line.strip() - if line.startswith('# XXX'): + for rawline in lines: + line = rawline.strip() + if line.startswith('#') and not rawline.startswith('##'): continue - yield line + yield line, rawline def parse_table_lines(lines): lines = iter_clean_lines(lines) - for line in lines: - if line.startswith(('####', '#----')): - kind = 0 if line[1] == '#' else 1 - try: - line = next(lines).strip() - except StopIteration: - line = '' - if not line.startswith('# '): - raise NotImplementedError(line) - yield kind, line[2:].lstrip() - continue - - maybe = None - while line.startswith('#'): - if line != '#' and line[1] == ' ': - maybe = line[2:].lstrip() - try: - line = next(lines).strip() - except StopIteration: - return - if not line: - break - else: - if line: - if maybe: - yield 2, maybe - yield 'row', line + group = None + prev = '' + for line, rawline in lines: + if line.startswith('## '): + assert not rawline.startswith(' '), (line, rawline) + if group: + assert prev, (line, rawline) + kind, after, _ = group + assert kind and kind != 'section-group', (group, line, rawline) + assert after is not None, (group, line, rawline) + else: + assert not prev, (prev, line, rawline) + kind, after = group = ('section-group', None) + title = line[3:].lstrip() + assert title, (line, rawline) + if after is not None: + try: + line, rawline = next(lines) + except StopIteration: + line = None + if line != after: + raise NotImplementedError((group, line, rawline)) + yield kind, title + group = None + elif group: + raise NotImplementedError((group, line, rawline)) + elif line.startswith('##---'): + assert line.rstrip('-') == '##', (line, rawline) + group = ('section-minor', '', line) + elif line.startswith('#####'): + assert not line.strip('#'), (line, rawline) + group = ('section-major', '', line) + elif line: + yield 'row', line + prev = line def iter_sections(lines): @@ -49,12 +65,13 @@ def iter_sections(lines): if header is None: header = value continue - raise NotImplementedError(value) + raise NotImplementedError(repr(value)) yield tuple(section), value else: if header is None: header = False - section[kind:] = [value] + start = KINDS.index(kind) + section[start:] = [value] def collect_sections(lines): From webhook-mailer at python.org Wed Dec 7 17:57:03 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Wed, 07 Dec 2022 22:57:03 -0000 Subject: [Python-checkins] gh-81057: Move More Globals to _PyRuntimeState (gh-100092) Message-ID: <mailman.2848.1670453824.3313.python-checkins@python.org> https://github.com/python/cpython/commit/91a8e002c21a5388c5152c5a4871b9a2d85f0fc1 commit: 91a8e002c21a5388c5152c5a4871b9a2d85f0fc1 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-07T15:56:31-07:00 summary: gh-81057: Move More Globals to _PyRuntimeState (gh-100092) https://github.com/python/cpython/issues/81057 files: M Include/internal/pycore_dtoa.h M Include/internal/pycore_obmalloc.h M Include/internal/pycore_obmalloc_init.h M Include/internal/pycore_parser.h M Include/internal/pycore_runtime.h M Include/internal/pycore_unicodeobject.h M Modules/_io/bufferedio.c M Objects/obmalloc.c M Objects/unicodeobject.c M Parser/pegen.c M Python/dtoa.c M Python/initconfig.c M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index fdc6e74ecd25..67189cf0ade6 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -25,7 +25,7 @@ Bigint { #ifdef Py_USING_MEMORY_DEBUGGER struct _dtoa_runtime_state { - int _not_used; + int _not_used; }; #define _dtoa_runtime_state_INIT {0} @@ -41,9 +41,12 @@ struct _dtoa_runtime_state { ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) struct _dtoa_runtime_state { - struct Bigint *freelist[Bigint_Kmax+1]; - double preallocated[Bigint_PREALLOC_SIZE]; - double *preallocated_next; + /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + // XXX This should be freed during runtime fini. + struct Bigint *p5s; + struct Bigint *freelist[Bigint_Kmax+1]; + double preallocated[Bigint_PREALLOC_SIZE]; + double *preallocated_next; }; #define _dtoa_runtime_state_INIT(runtime) \ { \ diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 93349d89c6ab..a5c7f4528f91 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -658,6 +658,7 @@ struct _obmalloc_usage { struct _obmalloc_state { + int dump_debug_stats; struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; struct _obmalloc_usage usage; diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index c0fb057d0665..c9f197e72de9 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -56,6 +56,7 @@ extern "C" { #define _obmalloc_state_INIT(obmalloc) \ { \ + .dump_debug_stats = -1, \ .pools = { \ .used = _obmalloc_pools_INIT(obmalloc.pools), \ }, \ diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index e2de24e2ca97..2d2b56bd824c 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -8,12 +8,31 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif + +#include "pycore_pyarena.h" // PyArena + + +#ifdef Py_DEBUG +#define _PYPEGEN_NSTATISTICS 2000 +#endif + +struct _parser_runtime_state { +#ifdef Py_DEBUG + long memo_statistics[_PYPEGEN_NSTATISTICS]; +#else + int _not_used; +#endif +}; + + + extern struct _mod* _PyParser_ASTFromString( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena); + extern struct _mod* _PyParser_ASTFromFile( FILE *fp, PyObject *filename_ob, @@ -25,6 +44,7 @@ extern struct _mod* _PyParser_ASTFromFile( int *errcode, PyArena *arena); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c1829cb1bdad..0720e2ed4422 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -17,6 +17,7 @@ extern "C" { #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState +#include "pycore_parser.h" // struct _parser_runtime_state #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state @@ -129,6 +130,10 @@ typedef struct pyruntimestate { unsigned long main_thread; + PyWideStringList orig_argv; + + struct _parser_runtime_state parser; + #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index b315ca1ae5b6..19faceebf1d8 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_fileutils.h" // _Py_error_handler +#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI void _PyUnicode_ExactDealloc(PyObject *op); @@ -52,6 +53,8 @@ struct _Py_unicode_ids { struct _Py_unicode_state { struct _Py_unicode_fs_codec fs_codec; + _PyUnicode_Name_CAPI *ucnhash_capi; + // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId() struct _Py_unicode_ids ids; }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 6df55b5b8303..ba8969f0bcd1 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -746,26 +746,26 @@ _buffered_init(buffered *self) int _PyIO_trap_eintr(void) { - static PyObject *eintr_int = NULL; PyObject *typ, *val, *tb; PyOSErrorObject *env_err; - - if (eintr_int == NULL) { - eintr_int = PyLong_FromLong(EINTR); - assert(eintr_int != NULL); - } if (!PyErr_ExceptionMatches(PyExc_OSError)) return 0; PyErr_Fetch(&typ, &val, &tb); PyErr_NormalizeException(&typ, &val, &tb); env_err = (PyOSErrorObject *) val; assert(env_err != NULL); - if (env_err->myerrno != NULL && - PyObject_RichCompareBool(env_err->myerrno, eintr_int, Py_EQ) > 0) { - Py_DECREF(typ); - Py_DECREF(val); - Py_XDECREF(tb); - return 1; + if (env_err->myerrno != NULL) { + assert(EINTR > 0 && EINTR < INT_MAX); + assert(PyLong_CheckExact(env_err->myerrno)); + int overflow; + int myerrno = PyLong_AsLongAndOverflow(env_err->myerrno, &overflow); + PyErr_Clear(); + if (myerrno == EINTR) { + Py_DECREF(typ); + Py_DECREF(val); + Py_XDECREF(tb); + return 1; + } } /* This silences any error set by PyObject_RichCompareBool() */ PyErr_Restore(typ, val, tb); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 4c08bc214cd2..276c5a276c06 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -908,11 +908,12 @@ new_arena(void) struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ void *address; - static int debug_stats = -1; + int debug_stats = _PyRuntime.obmalloc.dump_debug_stats; if (debug_stats == -1) { const char *opt = Py_GETENV("PYTHONMALLOCSTATS"); debug_stats = (opt != NULL && *opt != '\0'); + _PyRuntime.obmalloc.dump_debug_stats = debug_stats; } if (debug_stats) { _PyObject_DebugMallocStats(stderr); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index deeca35714b7..b721ccd805ed 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5697,8 +5697,6 @@ PyUnicode_AsUTF16String(PyObject *unicode) /* --- Unicode Escape Codec ----------------------------------------------- */ -static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; - PyObject * _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, Py_ssize_t size, @@ -5711,6 +5709,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, const char *end; PyObject *errorHandler = NULL; PyObject *exc = NULL; + _PyUnicode_Name_CAPI *ucnhash_capi; + PyInterpreterState *interp = _PyInterpreterState_Get(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; @@ -5858,6 +5858,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, /* \N{name} */ case 'N': + ucnhash_capi = interp->unicode.ucnhash_capi; if (ucnhash_capi == NULL) { /* load the unicode data module */ ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( @@ -5869,6 +5870,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, ); goto onError; } + interp->unicode.ucnhash_capi = ucnhash_capi; } message = "malformed \\N character escape"; @@ -15128,10 +15130,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) assert(get_interned_dict() == NULL); // bpo-47182: force a unicodedata CAPI capsule re-import on // subsequent initialization of main interpreter. - ucnhash_capi = NULL; } _PyUnicode_FiniEncodings(&state->fs_codec); + interp->unicode.ucnhash_capi = NULL; unicode_clear_identifiers(state); } diff --git a/Parser/pegen.c b/Parser/pegen.c index d34a86e9c883..d84e06861ede 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -246,8 +246,8 @@ _PyPegen_fill_token(Parser *p) // The array counts the number of tokens skipped by memoization, // indexed by type. -#define NSTATISTICS 2000 -static long memo_statistics[NSTATISTICS]; +#define NSTATISTICS _PYPEGEN_NSTATISTICS +#define memo_statistics _PyRuntime.parser.memo_statistics void _PyPegen_clear_memo_statistics() diff --git a/Python/dtoa.c b/Python/dtoa.c index 1b47d83bf77a..cff5f1b0658e 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -673,10 +673,6 @@ mult(Bigint *a, Bigint *b) #ifndef Py_USING_MEMORY_DEBUGGER -/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ - -static Bigint *p5s; - /* multiply the Bigint b by 5**k. Returns a pointer to the result, or NULL on failure; if the returned pointer is distinct from b then the original Bigint b will have been Bfree'd. Ignores the sign of b. */ @@ -696,7 +692,7 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; - p5 = p5s; + p5 = _PyRuntime.dtoa.p5s; if (!p5) { /* first time */ p5 = i2b(625); @@ -704,7 +700,7 @@ pow5mult(Bigint *b, int k) Bfree(b); return NULL; } - p5s = p5; + _PyRuntime.dtoa.p5s = p5; p5->next = 0; } for(;;) { diff --git a/Python/initconfig.c b/Python/initconfig.c index 67f6777d3b1d..64ae987b3f34 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -595,17 +595,13 @@ _Py_ClearStandardStreamEncoding(void) /* --- Py_GetArgcArgv() ------------------------------------------- */ -/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */ -static PyWideStringList orig_argv = {.length = 0, .items = NULL}; - - void _Py_ClearArgcArgv(void) { PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyWideStringList_Clear(&orig_argv); + _PyWideStringList_Clear(&_PyRuntime.orig_argv); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -620,7 +616,9 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - res = _PyWideStringList_Copy(&orig_argv, &argv_list); + // XXX _PyRuntime.orig_argv only gets cleared by Py_Main(), + // so it it currently leaks for embedders. + res = _PyWideStringList_Copy(&_PyRuntime.orig_argv, &argv_list); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return res; @@ -631,8 +629,8 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) void Py_GetArgcArgv(int *argc, wchar_t ***argv) { - *argc = (int)orig_argv.length; - *argv = orig_argv.items; + *argc = (int)_PyRuntime.orig_argv.length; + *argv = _PyRuntime.orig_argv.items; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index af5b7eb8eaf0..5c8164517f13 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -305,20 +305,12 @@ Objects/sliceobject.c - _Py_EllipsisObject - ##----------------------- ## effectively-const but initialized lazily -## idempotent -Python/dtoa.c - p5s - -Objects/obmalloc.c new_arena debug_stats - - ## others Python/perf_trampoline.c - perf_map_file - -Objects/unicodeobject.c - ucnhash_capi - ##----------------------- ## state -## local buffer -Python/suggestions.c levenshtein_distance buffer - - ## other Objects/object.c - _Py_RefTotal - Python/perf_trampoline.c - perf_status - @@ -398,7 +390,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/_io/bufferedio.c _PyIO_trap_eintr eintr_int - Modules/posixmodule.c os_dup2_impl dup3_works - Modules/posixmodule.c - structseq_new - Modules/posixmodule.c - ticks_per_second - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 128336a997eb..257823574fa8 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -87,8 +87,6 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - -# XXX This only gets cleared by Py_Main(). -Python/initconfig.c - orig_argv - ##----------------------- ## public C-API @@ -143,15 +141,6 @@ Python/pylifecycle.c - runtime_initialized - Modules/syslogmodule.c - S_ident_o - Modules/syslogmodule.c - S_log_open - -##----------------------- -## *not* tied to init/fini cycle - -# These do not ge reset with each init/fini cycle. -# XXX These should probably be tied to init/fini. Move to _PyRuntimeState? - -# special-use diagnistic state -Parser/pegen.c - memo_statistics - - ##----------------------- ## one-off temporary state From webhook-mailer at python.org Wed Dec 7 19:38:57 2022 From: webhook-mailer at python.org (vstinner) Date: Thu, 08 Dec 2022 00:38:57 -0000 Subject: [Python-checkins] gh-100086: Add build info to test.libregrtest (#100093) Message-ID: <mailman.2849.1670459938.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3c892022472eb975360fb3f0caa6f6fcc6fbf220 commit: 3c892022472eb975360fb3f0caa6f6fcc6fbf220 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-08T01:38:47+01:00 summary: gh-100086: Add build info to test.libregrtest (#100093) The Python test runner (libregrtest) now logs Python build information like "debug" vs "release" build, or LTO and PGO optimizations. files: A Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst M Lib/test/libregrtest/main.py M Lib/test/libregrtest/utils.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3eeef029b22d..19ccf2db5e7f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -17,7 +17,8 @@ ChildError, DidNotRun) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests -from test.libregrtest.utils import removepy, count, format_duration, printlist +from test.libregrtest.utils import (removepy, count, format_duration, + printlist, get_build_info) from test import support from test.support import os_helper from test.support import threading_helper @@ -491,6 +492,7 @@ def display_header(self): print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), "%s-endian" % sys.byteorder) + print("== Python build:", ' '.join(get_build_info())) print("== cwd:", os.getcwd()) cpu_count = os.cpu_count() if cpu_count: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index e6909170334e..fb13fa0e243b 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,6 +1,7 @@ import math import os.path import sys +import sysconfig import textwrap from test import support @@ -208,3 +209,87 @@ def clear_caches(): pass else: fractions._hash_algorithm.cache_clear() + + +def get_build_info(): + # Get most important configure and build options as a list of strings. + # Example: ['debug', 'ASAN+MSAN'] or ['release', 'LTO+PGO']. + + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + cflags = sysconfig.get_config_var('PY_CFLAGS') or '' + cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' + + build = [] + if hasattr(sys, 'gettotalrefcount'): + # --with-pydebug + build.append('debug') + + if '-DNDEBUG' in (cflags + cflags_nodist): + build.append('without_assert') + else: + build.append('release') + + if '--with-assertions' in config_args: + build.append('with_assert') + elif '-DNDEBUG' not in (cflags + cflags_nodist): + build.append('with_assert') + + # --enable-framework=name + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + if framework: + build.append(f'framework={framework}') + + # --enable-shared + shared = int(sysconfig.get_config_var('PY_ENABLE_SHARED') or '0') + if shared: + build.append('shared') + + # --with-lto + optimizations = [] + if '-flto=thin' in ldflags_nodist: + optimizations.append('ThinLTO') + elif '-flto' in ldflags_nodist: + optimizations.append('LTO') + + # --enable-optimizations + pgo_options = ( + # GCC + '-fprofile-use', + # clang: -fprofile-instr-use=code.profclangd + '-fprofile-instr-use', + # ICC + "-prof-use", + ) + if any(option in cflags_nodist for option in pgo_options): + optimizations.append('PGO') + if optimizations: + build.append('+'.join(optimizations)) + + # --with-address-sanitizer + sanitizers = [] + if support.check_sanitizer(address=True): + sanitizers.append("ASAN") + # --with-memory-sanitizer + if support.check_sanitizer(memory=True): + sanitizers.append("MSAN") + # --with-undefined-behavior-sanitizer + if support.check_sanitizer(ub=True): + sanitizers.append("UBSAN") + if sanitizers: + build.append('+'.join(sanitizers)) + + # --with-trace-refs + if hasattr(sys, 'getobjects'): + build.append("TraceRefs") + # --enable-pystats + if hasattr(sys, '_stats_on'): + build.append("pystats") + # --with-valgrind + if sysconfig.get_config_var('WITH_VALGRIND'): + build.append("valgrind") + # --with-dtrace + if sysconfig.get_config_var('WITH_DTRACE'): + build.append("dtrace") + + return build diff --git a/Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst b/Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst new file mode 100644 index 000000000000..a5f1bb9f5a5e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst @@ -0,0 +1,3 @@ +The Python test runner (libregrtest) now logs Python build information like +"debug" vs "release" build, or LTO and PGO optimizations. Patch by Victor +Stinner. From webhook-mailer at python.org Wed Dec 7 21:21:03 2022 From: webhook-mailer at python.org (corona10) Date: Thu, 08 Dec 2022 02:21:03 -0000 Subject: [Python-checkins] gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) Message-ID: <mailman.2850.1670466064.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dc8a86893df37e137cfe992e95e7d66cd68e9eaf commit: dc8a86893df37e137cfe992e95e7d66cd68e9eaf branch: main author: Dong-hee Na <donghee.na at python.org> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-08T11:20:34+09:00 summary: gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) files: A Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst M Lib/test/test_urllib2.py M Lib/urllib/error.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 28f88412fdca..498c0382d213 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1824,6 +1824,10 @@ def test_HTTPError_interface(self): expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + def test_parse_proxy(self): parse_proxy_test_cases = [ ('proxy.example.com', diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8e..feec0e7f848e 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.StringIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst new file mode 100644 index 000000000000..b1c170dff3ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst @@ -0,0 +1,2 @@ +Update :exc:`~urllib.error.HTTPError` to be initialized properly, even if +the ``fp`` is ``None``. Patch by Dong-hee Na. From webhook-mailer at python.org Wed Dec 7 21:24:58 2022 From: webhook-mailer at python.org (ethanfurman) Date: Thu, 08 Dec 2022 02:24:58 -0000 Subject: [Python-checkins] gh-92120: The docstring of enum.Enum is invalid in reST (GH-92122) Message-ID: <mailman.2851.1670466298.3313.python-checkins@python.org> https://github.com/python/cpython/commit/90d5c9b195a8133688c2d7b6ad9e0ca8b76df1df commit: 90d5c9b195a8133688c2d7b6ad9e0ca8b76df1df branch: main author: Takeshi KOMIYA <i.tkomiya at gmail.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-07T18:24:52-08:00 summary: gh-92120: The docstring of enum.Enum is invalid in reST (GH-92122) Closes #92120 files: A Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst M Lib/enum.py diff --git a/Lib/enum.py b/Lib/enum.py index f07b821f1a60..b2f7b9f8d4ad 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1054,20 +1054,20 @@ class Enum(metaclass=EnumType): Access them by: - - attribute access:: + - attribute access: - >>> Color.RED - <Color.RED: 1> + >>> Color.RED + <Color.RED: 1> - value lookup: - >>> Color(1) - <Color.RED: 1> + >>> Color(1) + <Color.RED: 1> - name lookup: - >>> Color['RED'] - <Color.RED: 1> + >>> Color['RED'] + <Color.RED: 1> Enumerations can be iterated over, and know how many members they have: diff --git a/Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst b/Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst new file mode 100644 index 000000000000..d585535ee38d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst @@ -0,0 +1 @@ +Fix reStructuredText syntax errors in docstrings in the :mod:`enum` module. From webhook-mailer at python.org Wed Dec 7 23:18:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 04:18:02 -0000 Subject: [Python-checkins] gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) Message-ID: <mailman.2852.1670473083.3313.python-checkins@python.org> https://github.com/python/cpython/commit/846898e5ab0fb1bf740263573f01669be186645f commit: 846898e5ab0fb1bf740263573f01669be186645f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T20:17:23-08:00 summary: gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) (cherry picked from commit dc8a86893df37e137cfe992e95e7d66cd68e9eaf) Co-authored-by: Dong-hee Na <donghee.na at python.org> files: A Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst M Lib/test/test_urllib2.py M Lib/urllib/error.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 46a0ab1d83e6..db92ba56c34f 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1825,6 +1825,10 @@ def test_HTTPError_interface(self): expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + def test_parse_proxy(self): parse_proxy_test_cases = [ ('proxy.example.com', diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8e..feec0e7f848e 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.StringIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst new file mode 100644 index 000000000000..b1c170dff3ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst @@ -0,0 +1,2 @@ +Update :exc:`~urllib.error.HTTPError` to be initialized properly, even if +the ``fp`` is ``None``. Patch by Dong-hee Na. From webhook-mailer at python.org Wed Dec 7 23:18:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 04:18:02 -0000 Subject: [Python-checkins] gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) Message-ID: <mailman.2853.1670473084.3313.python-checkins@python.org> https://github.com/python/cpython/commit/85be25827bb94112037c391eca66a0d796468825 commit: 85be25827bb94112037c391eca66a0d796468825 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T20:17:35-08:00 summary: gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) (cherry picked from commit dc8a86893df37e137cfe992e95e7d66cd68e9eaf) Co-authored-by: Dong-hee Na <donghee.na at python.org> files: A Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst M Lib/test/test_urllib2.py M Lib/urllib/error.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 399c94213a63..4ef391c4540c 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1823,6 +1823,10 @@ def test_HTTPError_interface(self): expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + def test_parse_proxy(self): parse_proxy_test_cases = [ ('proxy.example.com', diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8e..feec0e7f848e 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.StringIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst new file mode 100644 index 000000000000..b1c170dff3ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst @@ -0,0 +1,2 @@ +Update :exc:`~urllib.error.HTTPError` to be initialized properly, even if +the ``fp`` is ``None``. Patch by Dong-hee Na. From webhook-mailer at python.org Wed Dec 7 23:59:43 2022 From: webhook-mailer at python.org (gpshead) Date: Thu, 08 Dec 2022 04:59:43 -0000 Subject: [Python-checkins] gh-98030: socket: add missing TCP socket options (#98031) Message-ID: <mailman.2854.1670475583.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cce836296016463032495c6ca739ab469ed13d3c commit: cce836296016463032495c6ca739ab469ed13d3c branch: main author: Matthieu Baerts <matttbe at gmail.com> committer: gpshead <greg at krypto.org> date: 2022-12-07T20:59:37-08:00 summary: gh-98030: socket: add missing TCP socket options (#98031) A few TCP socket options have been added to the Linux kernel these last few years. This commit adds all the ones available in Linux 6.0: https://elixir.bootlin.com/linux/v6.0/source/include/uapi/linux/tcp.h#L91 While at it, the TCP_FASTOPEN option has been moved lower in the list just to keep the same order as in tcp.h to ease future synchronisations. Signed-off-by: Matthieu Baerts <matthieu.baerts at tessares.net> files: A Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst M Doc/library/socket.rst M Modules/socketmodule.c diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c946407ea1d8..de2e1aa3868b 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -428,7 +428,14 @@ Constants .. versionchanged:: 3.12 Added ``SO_RTABLE`` and ``SO_USER_COOKIE``. On OpenBSD and FreeBSD respectively those constants can be used in the same way that - ``SO_MARK`` is used on Linux. + ``SO_MARK`` is used on Linux. Also added missing TCP socket options from + Linux: ``TCP_MD5SIG``, ``TCP_THIN_LINEAR_TIMEOUTS``, ``TCP_THIN_DUPACK``, + ``TCP_REPAIR``, ``TCP_REPAIR_QUEUE``, ``TCP_QUEUE_SEQ``, + ``TCP_REPAIR_OPTIONS``, ``TCP_TIMESTAMP``, ``TCP_CC_INFO``, + ``TCP_SAVE_SYN``, ``TCP_SAVED_SYN``, ``TCP_REPAIR_WINDOW``, + ``TCP_FASTOPEN_CONNECT``, ``TCP_ULP``, ``TCP_MD5SIG_EXT``, + ``TCP_FASTOPEN_KEY``, ``TCP_FASTOPEN_NO_COOKIE``, + ``TCP_ZEROCOPY_RECEIVE``, ``TCP_INQ``, ``TCP_TX_DELAY``. .. data:: AF_CAN PF_CAN diff --git a/Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst b/Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst new file mode 100644 index 000000000000..7768ed0817e8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst @@ -0,0 +1,7 @@ +Add missing TCP socket options from Linux: ``TCP_MD5SIG``, +``TCP_THIN_LINEAR_TIMEOUTS``, ``TCP_THIN_DUPACK``, ``TCP_REPAIR``, +``TCP_REPAIR_QUEUE``, ``TCP_QUEUE_SEQ``, ``TCP_REPAIR_OPTIONS``, +``TCP_TIMESTAMP``, ``TCP_CC_INFO``, ``TCP_SAVE_SYN``, ``TCP_SAVED_SYN``, +``TCP_REPAIR_WINDOW``, ``TCP_FASTOPEN_CONNECT``, ``TCP_ULP``, +``TCP_MD5SIG_EXT``, ``TCP_FASTOPEN_KEY``, ``TCP_FASTOPEN_NO_COOKIE``, +``TCP_ZEROCOPY_RECEIVE``, ``TCP_INQ``, ``TCP_TX_DELAY``. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7d438448ef36..2c59c2f2c89b 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8484,18 +8484,78 @@ PyInit__socket(void) #ifdef TCP_QUICKACK PyModule_AddIntMacro(m, TCP_QUICKACK); #endif -#ifdef TCP_FASTOPEN - PyModule_AddIntMacro(m, TCP_FASTOPEN); -#endif #ifdef TCP_CONGESTION PyModule_AddIntMacro(m, TCP_CONGESTION); #endif +#ifdef TCP_MD5SIG + PyModule_AddIntMacro(m, TCP_MD5SIG); +#endif +#ifdef TCP_THIN_LINEAR_TIMEOUTS + PyModule_AddIntMacro(m, TCP_THIN_LINEAR_TIMEOUTS); +#endif +#ifdef TCP_THIN_DUPACK + PyModule_AddIntMacro(m, TCP_THIN_DUPACK); +#endif #ifdef TCP_USER_TIMEOUT PyModule_AddIntMacro(m, TCP_USER_TIMEOUT); #endif +#ifdef TCP_REPAIR + PyModule_AddIntMacro(m, TCP_REPAIR); +#endif +#ifdef TCP_REPAIR_QUEUE + PyModule_AddIntMacro(m, TCP_REPAIR_QUEUE); +#endif +#ifdef TCP_QUEUE_SEQ + PyModule_AddIntMacro(m, TCP_QUEUE_SEQ); +#endif +#ifdef TCP_REPAIR_OPTIONS + PyModule_AddIntMacro(m, TCP_REPAIR_OPTIONS); +#endif +#ifdef TCP_FASTOPEN + PyModule_AddIntMacro(m, TCP_FASTOPEN); +#endif +#ifdef TCP_TIMESTAMP + PyModule_AddIntMacro(m, TCP_TIMESTAMP); +#endif #ifdef TCP_NOTSENT_LOWAT PyModule_AddIntMacro(m, TCP_NOTSENT_LOWAT); #endif +#ifdef TCP_CC_INFO + PyModule_AddIntMacro(m, TCP_CC_INFO); +#endif +#ifdef TCP_SAVE_SYN + PyModule_AddIntMacro(m, TCP_SAVE_SYN); +#endif +#ifdef TCP_SAVED_SYN + PyModule_AddIntMacro(m, TCP_SAVED_SYN); +#endif +#ifdef TCP_REPAIR_WINDOW + PyModule_AddIntMacro(m, TCP_REPAIR_WINDOW); +#endif +#ifdef TCP_FASTOPEN_CONNECT + PyModule_AddIntMacro(m, TCP_FASTOPEN_CONNECT); +#endif +#ifdef TCP_ULP + PyModule_AddIntMacro(m, TCP_ULP); +#endif +#ifdef TCP_MD5SIG_EXT + PyModule_AddIntMacro(m, TCP_MD5SIG_EXT); +#endif +#ifdef TCP_FASTOPEN_KEY + PyModule_AddIntMacro(m, TCP_FASTOPEN_KEY); +#endif +#ifdef TCP_FASTOPEN_NO_COOKIE + PyModule_AddIntMacro(m, TCP_FASTOPEN_NO_COOKIE); +#endif +#ifdef TCP_ZEROCOPY_RECEIVE + PyModule_AddIntMacro(m, TCP_ZEROCOPY_RECEIVE); +#endif +#ifdef TCP_INQ + PyModule_AddIntMacro(m, TCP_INQ); +#endif +#ifdef TCP_TX_DELAY + PyModule_AddIntMacro(m, TCP_TX_DELAY); +#endif /* IPX options */ #ifdef IPX_TYPE From webhook-mailer at python.org Thu Dec 8 01:58:15 2022 From: webhook-mailer at python.org (ethanfurman) Date: Thu, 08 Dec 2022 06:58:15 -0000 Subject: [Python-checkins] gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) Message-ID: <mailman.2855.1670482696.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ded02ca54d7bfa32c8eab0871d56e4547cd356eb commit: ded02ca54d7bfa32c8eab0871d56e4547cd356eb branch: main author: Ethan Furman <ethan at stoneleaf.us> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-07T22:58:08-08:00 summary: gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) When checking for auto() instances, only top-level usage is supported, which means either alone or as part of a regular tuple. Other containers, such as lists, dicts, or namedtuples, will not have auto() transformed into a value. files: A Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst M Lib/enum.py M Lib/test/test_enum.py diff --git a/Lib/enum.py b/Lib/enum.py index b2f7b9f8d4ad..a0cad066dc23 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -436,7 +436,9 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if isinstance(value, tuple): + if type(value) is tuple and any(isinstance(v, auto) for v in value): + # insist on an actual tuple, no subclasses, in keeping with only supporting + # top-level auto() usage (not contained in any other data structure) auto_valued = [] for v in value: if isinstance(v, auto): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 376347e4c111..d876c8e5fb77 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2841,6 +2841,19 @@ class MyIntFlag(IntFlag): self.assertEqual(deep, flags) self.assertEqual(copied.value, 1 | 2 | 8) + def test_namedtuple_as_value(self): + from collections import namedtuple + TTuple = namedtuple('TTuple', 'id a blist') + class NTEnum(Enum): + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTEnum.NONE), "<NTEnum.NONE: TTuple(id=0, a=0, blist=[])>") + self.assertEqual(NTEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual( + [x.value for x in NTEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" diff --git a/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst new file mode 100644 index 000000000000..256f2bcd39f8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst @@ -0,0 +1 @@ +Fix ``tuple`` subclasses being cast to ``tuple`` when used as enum values. From webhook-mailer at python.org Thu Dec 8 02:21:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 07:21:55 -0000 Subject: [Python-checkins] gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) Message-ID: <mailman.2856.1670484116.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d4426c829565e3ef922c091ee9bd48bc556f2550 commit: d4426c829565e3ef922c091ee9bd48bc556f2550 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T23:21:29-08:00 summary: gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) When checking for auto() instances, only top-level usage is supported, which means either alone or as part of a regular tuple. Other containers, such as lists, dicts, or namedtuples, will not have auto() transformed into a value. (cherry picked from commit ded02ca54d7bfa32c8eab0871d56e4547cd356eb) Co-authored-by: Ethan Furman <ethan at stoneleaf.us> files: A Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst M Lib/enum.py M Lib/test/test_enum.py diff --git a/Lib/enum.py b/Lib/enum.py index 1efddfa1a0ac..7ad599bb1e62 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -430,7 +430,9 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if isinstance(value, tuple): + if type(value) is tuple and any(isinstance(v, auto) for v in value): + # insist on an actual tuple, no subclasses, in keeping with only supporting + # top-level auto() usage (not contained in any other data structure) auto_valued = [] for v in value: if isinstance(v, auto): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index a9b80ad0d3db..0e2da1d64c19 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2727,6 +2727,19 @@ class MyIntFlag(IntFlag): self.assertEqual(deep, flags) self.assertEqual(copied.value, 1 | 2 | 8) + def test_namedtuple_as_value(self): + from collections import namedtuple + TTuple = namedtuple('TTuple', 'id a blist') + class NTEnum(Enum): + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTEnum.NONE), "<NTEnum.NONE: TTuple(id=0, a=0, blist=[])>") + self.assertEqual(NTEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual( + [x.value for x in NTEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" diff --git a/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst new file mode 100644 index 000000000000..256f2bcd39f8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst @@ -0,0 +1 @@ +Fix ``tuple`` subclasses being cast to ``tuple`` when used as enum values. From webhook-mailer at python.org Thu Dec 8 03:21:20 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 08 Dec 2022 08:21:20 -0000 Subject: [Python-checkins] Fix `test_run_until_complete_baseexception` test to check for `KeyboardInterrupt` in asyncio (#24477) Message-ID: <mailman.2857.1670487681.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e8fff515f056737d6d055ea5438b2135863548b1 commit: e8fff515f056737d6d055ea5438b2135863548b1 branch: main author: Fantix King <fantix.king at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-08T13:51:04+05:30 summary: Fix `test_run_until_complete_baseexception` test to check for `KeyboardInterrupt` in asyncio (#24477) files: M Lib/test/test_asyncio/test_base_events.py diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 65dd4d42708b..532a7e5385ae 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -861,20 +861,15 @@ async def raise_keyboard_interrupt(): self.loop._process_events = mock.Mock() - try: + with self.assertRaises(KeyboardInterrupt): self.loop.run_until_complete(raise_keyboard_interrupt()) - except KeyboardInterrupt: - pass def func(): self.loop.stop() func.called = True func.called = False - try: - self.loop.call_soon(func) - self.loop.run_forever() - except KeyboardInterrupt: - pass + self.loop.call_later(0.01, func) + self.loop.run_forever() self.assertTrue(func.called) def test_single_selecter_event_callback_after_stopping(self): From webhook-mailer at python.org Thu Dec 8 03:26:44 2022 From: webhook-mailer at python.org (vstinner) Date: Thu, 08 Dec 2022 08:26:44 -0000 Subject: [Python-checkins] test_ast uses infinite_recursion() to prevent crash (#100104) Message-ID: <mailman.2858.1670488005.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cd67c1bb30eccd0c6fd1386405df225aed4c91a9 commit: cd67c1bb30eccd0c6fd1386405df225aed4c91a9 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-08T09:26:38+01:00 summary: test_ast uses infinite_recursion() to prevent crash (#100104) test.test_ast_recursion_limit() now uses infinite_recursion() of test.support to prevent crashes on debug builds. Before this change, the test crashed on ARM64 Windows 3.x buildbot worker which builds Python in debug mode. files: M Lib/test/test_ast.py diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 773fba87632b..ab6a63faa590 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -837,7 +837,8 @@ def check_limit(prefix, repeated): details = "Compiling ({!r} + {!r} * {})".format( prefix, repeated, depth) with self.assertRaises(RecursionError, msg=details): - ast.parse(broken) + with support.infinite_recursion(): + ast.parse(broken) check_limit("a", "()") check_limit("a", ".b") From webhook-mailer at python.org Thu Dec 8 03:52:08 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 08:52:08 -0000 Subject: [Python-checkins] test_ast uses infinite_recursion() to prevent crash (GH-100104) Message-ID: <mailman.2859.1670489529.3313.python-checkins@python.org> https://github.com/python/cpython/commit/42fde2d164eb6f9c31a01b290114026d857676ad commit: 42fde2d164eb6f9c31a01b290114026d857676ad branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T00:52:02-08:00 summary: test_ast uses infinite_recursion() to prevent crash (GH-100104) test.test_ast_recursion_limit() now uses infinite_recursion() of test.support to prevent crashes on debug builds. Before this change, the test crashed on ARM64 Windows 3.x buildbot worker which builds Python in debug mode. (cherry picked from commit cd67c1bb30eccd0c6fd1386405df225aed4c91a9) Co-authored-by: Victor Stinner <vstinner at python.org> files: M Lib/test/test_ast.py diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 7581adc8fc27..c33c0ae22379 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -831,7 +831,8 @@ def check_limit(prefix, repeated): details = "Compiling ({!r} + {!r} * {})".format( prefix, repeated, depth) with self.assertRaises(RecursionError, msg=details): - ast.parse(broken) + with support.infinite_recursion(): + ast.parse(broken) check_limit("a", "()") check_limit("a", ".b") From webhook-mailer at python.org Thu Dec 8 04:02:12 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 09:02:12 -0000 Subject: [Python-checkins] [3.11] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100022) Message-ID: <mailman.2860.1670490134.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f43b3d51a9488a7f2334fffde1c0209ebf054a58 commit: f43b3d51a9488a7f2334fffde1c0209ebf054a58 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T01:02:06-08:00 summary: [3.11] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100022) (cherry picked from commit 124ecd657646f808d1d3282c37ee19aae6bcb47f) Co-authored-by: Michael F?rderer <michael.foerderer at gmx.de> files: A Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst M Lib/ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 73b1bd12ddca..10c6799300f6 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -639,12 +639,15 @@ def _getfinalpathname_nonstrict(path): # 21: ERROR_NOT_READY (implies drive with no media) # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file) # 50: ERROR_NOT_SUPPORTED + # 53: ERROR_BAD_NETPATH + # 65: ERROR_NETWORK_ACCESS_DENIED # 67: ERROR_BAD_NET_NAME (implies remote server unavailable) # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME + # 161: ERROR_BAD_PATHNAME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst new file mode 100644 index 000000000000..79f8c506b54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst @@ -0,0 +1,2 @@ +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) +and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). From webhook-mailer at python.org Thu Dec 8 04:02:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 09:02:34 -0000 Subject: [Python-checkins] [3.10] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100023) Message-ID: <mailman.2861.1670490154.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0e2c7839bd297ad284fd07bf3736f722b87175df commit: 0e2c7839bd297ad284fd07bf3736f722b87175df branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T01:02:28-08:00 summary: [3.10] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100023) (cherry picked from commit 124ecd657646f808d1d3282c37ee19aae6bcb47f) Co-authored-by: Michael F?rderer <michael.foerderer at gmx.de> files: A Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst M Lib/ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 97edfa52aaaf..c14e5c7ceca5 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -625,12 +625,15 @@ def _getfinalpathname_nonstrict(path): # 21: ERROR_NOT_READY (implies drive with no media) # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file) # 50: ERROR_NOT_SUPPORTED + # 53: ERROR_BAD_NETPATH + # 65: ERROR_NETWORK_ACCESS_DENIED # 67: ERROR_BAD_NET_NAME (implies remote server unavailable) # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME + # 161: ERROR_BAD_PATHNAME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst new file mode 100644 index 000000000000..79f8c506b54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst @@ -0,0 +1,2 @@ +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) +and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). From webhook-mailer at python.org Thu Dec 8 12:35:05 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 17:35:05 -0000 Subject: [Python-checkins] Use sphinxext-opengraph to generate OpenGraph metadata (GH-99931) Message-ID: <mailman.2862.1670520907.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0274a3bc3b1a479c766915a562db6fb19ef97daa commit: 0274a3bc3b1a479c766915a562db6fb19ef97daa branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T09:34:52-08:00 summary: Use sphinxext-opengraph to generate OpenGraph metadata (GH-99931) (cherry picked from commit f49c735e525cf031ddbfc19161aafac4fb18837b) Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: A Doc/_static/og-image.png A Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst M Doc/conf.py M Doc/requirements.txt diff --git a/Doc/_static/og-image.png b/Doc/_static/og-image.png new file mode 100644 index 000000000000..0e80751e7403 Binary files /dev/null and b/Doc/_static/og-image.png differ diff --git a/Doc/conf.py b/Doc/conf.py index fd4ee2d5eee8..98b076c0ee57 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -13,9 +13,25 @@ # General configuration # --------------------- -extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', - 'pyspecific', 'c_annotations', 'escape4chm', - 'asdl_highlight', 'peg_highlight', 'glossary_search'] +extensions = [ + 'asdl_highlight', + 'c_annotations', + 'escape4chm', + 'glossary_search', + 'peg_highlight', + 'pyspecific', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', +] + +# Skip if downstream redistributors haven't installed it +try: + import sphinxext.opengraph +except ImportError: + pass +else: + extensions.append('sphinxext.opengraph') + doctest_global_setup = ''' try: @@ -109,7 +125,7 @@ html_use_opensearch = 'https://docs.python.org/' + version # Additional static files. -html_static_path = ['tools/static'] +html_static_path = ['_static', 'tools/static'] # Output file base name for HTML help builder. htmlhelp_basename = 'python' + release.replace('.', '') @@ -234,3 +250,13 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' + +# sphinxext-opengraph config +ogp_site_url = 'https://docs.python.org/3/' +ogp_site_name = 'Python documentation' +ogp_image = '_static/og-image.png' +ogp_custom_meta_tags = [ + '<meta property="og:image:width" content="200">', + '<meta property="og:image:height" content="200">', + '<meta name="theme-color" content="#3776ab">', +] diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 958665db69e2..134f39d6d7b3 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -8,6 +8,7 @@ sphinx==4.5.0 blurb sphinx-lint==0.6.7 +sphinxext-opengraph>=0.7.1 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst new file mode 100644 index 000000000000..0c01a2cb2cfa --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst @@ -0,0 +1,2 @@ +Use `sphinxext-opengraph <https://sphinxext-opengraph.readthedocs.io/>`__ +to generate `OpenGraph metadata <https://ogp.me/>`__. From webhook-mailer at python.org Thu Dec 8 15:37:27 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 08 Dec 2022 20:37:27 -0000 Subject: [Python-checkins] gh-96250: Improve sqlite3 injection attack example (#99270) Message-ID: <mailman.2863.1670531848.3313.python-checkins@python.org> https://github.com/python/cpython/commit/41d4ac9da348ca33056e271d71588b2dc3a6d48d commit: 41d4ac9da348ca33056e271d71588b2dc3a6d48d branch: main author: Jia Junjie <62194633+jiajunjie at users.noreply.github.com> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-08T21:37:08+01:00 summary: gh-96250: Improve sqlite3 injection attack example (#99270) Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 960f2966afe1..2b6387cbfb52 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1929,12 +1929,16 @@ How to use placeholders to bind values in SQL queries SQL operations usually need to use values from Python variables. However, beware of using Python's string operations to assemble queries, as they -are vulnerable to `SQL injection attacks`_ (see the `xkcd webcomic -<https://xkcd.com/327/>`_ for a humorous example of what can go wrong):: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) +are vulnerable to `SQL injection attacks`_. For example, an attacker can simply +close the single quote and inject ``OR TRUE`` to select all rows:: + + >>> # Never do this -- insecure! + >>> symbol = input() + ' OR TRUE; -- + >>> sql = "SELECT * FROM stocks WHERE symbol = '%s'" % symbol + >>> print(sql) + SELECT * FROM stocks WHERE symbol = '' OR TRUE; --' + >>> cur.execute(sql) Instead, use the DB-API's parameter substitution. To insert a variable into a query string, use a placeholder in the string, and substitute the actual values From webhook-mailer at python.org Thu Dec 8 15:45:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 20:45:59 -0000 Subject: [Python-checkins] gh-96250: Improve sqlite3 injection attack example (GH-99270) Message-ID: <mailman.2864.1670532361.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8ef604518745989a3991a37f58368d2ce3c27d84 commit: 8ef604518745989a3991a37f58368d2ce3c27d84 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T12:45:32-08:00 summary: gh-96250: Improve sqlite3 injection attack example (GH-99270) (cherry picked from commit 41d4ac9da348ca33056e271d71588b2dc3a6d48d) Co-authored-by: Jia Junjie <62194633+jiajunjie at users.noreply.github.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9775f80624a0..65fa1b613153 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1427,12 +1427,16 @@ How to use placeholders to bind values in SQL queries SQL operations usually need to use values from Python variables. However, beware of using Python's string operations to assemble queries, as they -are vulnerable to `SQL injection attacks`_ (see the `xkcd webcomic -<https://xkcd.com/327/>`_ for a humorous example of what can go wrong):: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) +are vulnerable to `SQL injection attacks`_. For example, an attacker can simply +close the single quote and inject ``OR TRUE`` to select all rows:: + + >>> # Never do this -- insecure! + >>> symbol = input() + ' OR TRUE; -- + >>> sql = "SELECT * FROM stocks WHERE symbol = '%s'" % symbol + >>> print(sql) + SELECT * FROM stocks WHERE symbol = '' OR TRUE; --' + >>> cur.execute(sql) Instead, use the DB-API's parameter substitution. To insert a variable into a query string, use a placeholder in the string, and substitute the actual values From webhook-mailer at python.org Thu Dec 8 15:45:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 20:45:59 -0000 Subject: [Python-checkins] gh-96250: Improve sqlite3 injection attack example (GH-99270) Message-ID: <mailman.2865.1670532361.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ecb16d5d63b2299c37c321ea4d8cf6b51c626ac1 commit: ecb16d5d63b2299c37c321ea4d8cf6b51c626ac1 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T12:45:40-08:00 summary: gh-96250: Improve sqlite3 injection attack example (GH-99270) (cherry picked from commit 41d4ac9da348ca33056e271d71588b2dc3a6d48d) Co-authored-by: Jia Junjie <62194633+jiajunjie at users.noreply.github.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index b720ac995c44..e8db6c55c938 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1813,12 +1813,16 @@ How to use placeholders to bind values in SQL queries SQL operations usually need to use values from Python variables. However, beware of using Python's string operations to assemble queries, as they -are vulnerable to `SQL injection attacks`_ (see the `xkcd webcomic -<https://xkcd.com/327/>`_ for a humorous example of what can go wrong):: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) +are vulnerable to `SQL injection attacks`_. For example, an attacker can simply +close the single quote and inject ``OR TRUE`` to select all rows:: + + >>> # Never do this -- insecure! + >>> symbol = input() + ' OR TRUE; -- + >>> sql = "SELECT * FROM stocks WHERE symbol = '%s'" % symbol + >>> print(sql) + SELECT * FROM stocks WHERE symbol = '' OR TRUE; --' + >>> cur.execute(sql) Instead, use the DB-API's parameter substitution. To insert a variable into a query string, use a placeholder in the string, and substitute the actual values From webhook-mailer at python.org Thu Dec 8 16:08:25 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 08 Dec 2022 21:08:25 -0000 Subject: [Python-checkins] GH-98363: Have batched() return tuples (GH-100118) Message-ID: <mailman.2866.1670533705.3313.python-checkins@python.org> https://github.com/python/cpython/commit/35cc0ea736a323119157117d93e5d68d8247e89f commit: 35cc0ea736a323119157117d93e5d68d8247e89f branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-08T15:08:16-06:00 summary: GH-98363: Have batched() return tuples (GH-100118) files: M Doc/library/itertools.rst M Lib/test/test_itertools.py M Modules/clinic/itertoolsmodule.c.h M Modules/itertoolsmodule.c diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0b5978505a96..624d2430ac20 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -52,7 +52,7 @@ Iterator Arguments Results Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`batched` p, n [p0, p1, ..., p_n-1], ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` @@ -166,11 +166,11 @@ loops that truncate the stream. .. function:: batched(iterable, n) - Batch data from the *iterable* into lists of length *n*. The last + Batch data from the *iterable* into tuples of length *n*. The last batch may be shorter than *n*. - Loops over the input iterable and accumulates data into lists up to - size *n*. The input is consumed lazily, just enough to fill a list. + Loops over the input iterable and accumulates data into tuples up to + size *n*. The input is consumed lazily, just enough to fill a batch. The result is yielded as soon as the batch is full or when the input iterable is exhausted: @@ -179,14 +179,14 @@ loops that truncate the stream. >>> flattened_data = ['roses', 'red', 'violets', 'blue', 'sugar', 'sweet'] >>> unflattened = list(batched(flattened_data, 2)) >>> unflattened - [['roses', 'red'], ['violets', 'blue'], ['sugar', 'sweet']] + [('roses', 'red'), ('violets', 'blue'), ('sugar', 'sweet')] >>> for batch in batched('ABCDEFG', 3): ... print(batch) ... - ['A', 'B', 'C'] - ['D', 'E', 'F'] - ['G'] + ('A', 'B', 'C') + ('D', 'E', 'F') + ('G',) Roughly equivalent to:: @@ -195,7 +195,7 @@ loops that truncate the stream. if n < 1: raise ValueError('n must be at least one') it = iter(iterable) - while (batch := list(islice(it, n))): + while (batch := tuple(islice(it, n))): yield batch .. versionadded:: 3.12 diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 5f5bcbc7cfb8..b447b6cbab9c 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -161,11 +161,11 @@ def test_accumulate(self): def test_batched(self): self.assertEqual(list(batched('ABCDEFG', 3)), - [['A', 'B', 'C'], ['D', 'E', 'F'], ['G']]) + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 2)), - [['A', 'B'], ['C', 'D'], ['E', 'F'], ['G']]) + [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 1)), - [['A'], ['B'], ['C'], ['D'], ['E'], ['F'], ['G']]) + [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)]) with self.assertRaises(TypeError): # Too few arguments list(batched('ABCDEFG')) @@ -188,8 +188,8 @@ def test_batched(self): with self.subTest(s=s, n=n, batches=batches): # Order is preserved and no data is lost self.assertEqual(''.join(chain(*batches)), s) - # Each batch is an exact list - self.assertTrue(all(type(batch) is list for batch in batches)) + # Each batch is an exact tuple + self.assertTrue(all(type(batch) is tuple for batch in batches)) # All but the last batch is of size n if batches: last_batch = batches.pop() @@ -1809,12 +1809,12 @@ class TestPurePythonRoughEquivalents(unittest.TestCase): def test_batched_recipe(self): def batched_recipe(iterable, n): - "Batch data into lists of length n. The last batch may be shorter." + "Batch data into tuples of length n. The last batch may be shorter." # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: raise ValueError('n must be at least one') it = iter(iterable) - while (batch := list(islice(it, n))): + while (batch := tuple(islice(it, n))): yield batch for iterable, n in product( @@ -2087,7 +2087,7 @@ def test_accumulate(self): def test_batched(self): s = 'abcde' - r = [['a', 'b'], ['c', 'd'], ['e']] + r = [('a', 'b'), ('c', 'd'), ('e',)] n = 2 for g in (G, I, Ig, L, R): with self.subTest(g=g): diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 17f9ebb24939..287de524e913 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -12,19 +12,19 @@ PyDoc_STRVAR(batched_new__doc__, "batched(iterable, n)\n" "--\n" "\n" -"Batch data into lists of length n. The last batch may be shorter than n.\n" +"Batch data into tuples of length n. The last batch may be shorter than n.\n" "\n" -"Loops over the input iterable and accumulates data into lists\n" +"Loops over the input iterable and accumulates data into tuples\n" "up to size n. The input is consumed lazily, just enough to\n" -"fill a list. The result is yielded as soon as a batch is full\n" +"fill a batch. The result is yielded as soon as a batch is full\n" "or when the input iterable is exhausted.\n" "\n" " >>> for batch in batched(\'ABCDEFG\', 3):\n" " ... print(batch)\n" " ...\n" -" [\'A\', \'B\', \'C\']\n" -" [\'D\', \'E\', \'F\']\n" -" [\'G\']"); +" (\'A\', \'B\', \'C\')\n" +" (\'D\', \'E\', \'F\')\n" +" (\'G\',)"); static PyObject * batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n); @@ -913,4 +913,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=efea8cd1e647bd17 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0229ebd72962f130 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d45000788c53..60ec11c32d01 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -56,11 +56,13 @@ static PyTypeObject pairwise_type; /* batched object ************************************************************/ /* Note: The built-in zip() function includes a "strict" argument - that is needed because that function can silently truncate data - and there is no easy way for a user to detect that condition. - The same reasoning does not apply to batched() which never drops - data. Instead, it produces a shorter list which can be handled - as the user sees fit. + that was needed because that function would silently truncate data, + and there was no easy way for a user to detect the data loss. + The same reasoning does not apply to batched() which never drops data. + Instead, batched() produces a shorter tuple which can be handled + as the user sees fit. If requested, it would be reasonable to add + "fillvalue" support which had demonstrated value in zip_longest(). + For now, the API is kept simple and clean. */ typedef struct { @@ -74,25 +76,25 @@ typedef struct { itertools.batched.__new__ as batched_new iterable: object n: Py_ssize_t -Batch data into lists of length n. The last batch may be shorter than n. +Batch data into tuples of length n. The last batch may be shorter than n. -Loops over the input iterable and accumulates data into lists +Loops over the input iterable and accumulates data into tuples up to size n. The input is consumed lazily, just enough to -fill a list. The result is yielded as soon as a batch is full +fill a batch. The result is yielded as soon as a batch is full or when the input iterable is exhausted. >>> for batch in batched('ABCDEFG', 3): ... print(batch) ... - ['A', 'B', 'C'] - ['D', 'E', 'F'] - ['G'] + ('A', 'B', 'C') + ('D', 'E', 'F') + ('G',) [clinic start generated code]*/ static PyObject * batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) -/*[clinic end generated code: output=7ebc954d655371b6 input=f28fd12cb52365f0]*/ +/*[clinic end generated code: output=7ebc954d655371b6 input=ffd70726927c5129]*/ { PyObject *it; batchedobject *bo; @@ -150,12 +152,12 @@ batched_next(batchedobject *bo) if (it == NULL) { return NULL; } - result = PyList_New(n); + result = PyTuple_New(n); if (result == NULL) { return NULL; } iternextfunc iternext = *Py_TYPE(it)->tp_iternext; - PyObject **items = _PyList_ITEMS(result); + PyObject **items = _PyTuple_ITEMS(result); for (i=0 ; i < n ; i++) { item = iternext(it); if (item == NULL) { From webhook-mailer at python.org Thu Dec 8 16:31:33 2022 From: webhook-mailer at python.org (gvanrossum) Date: Thu, 08 Dec 2022 21:31:33 -0000 Subject: [Python-checkins] GH-98831: Typed stack effects, and more instructions converted (#99764) Message-ID: <mailman.2867.1670535095.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c85be734d1823f02a3600d7cd9195cecbf51afe8 commit: c85be734d1823f02a3600d7cd9195cecbf51afe8 branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-08T13:31:27-08:00 summary: GH-98831: Typed stack effects, and more instructions converted (#99764) Stack effects can now have a type, e.g. `inst(X, (left, right -- jump/uint64_t)) { ... }`. Instructions converted to the non-legacy format: * COMPARE_OP * COMPARE_OP_FLOAT_JUMP * COMPARE_OP_INT_JUMP * COMPARE_OP_STR_JUMP * STORE_ATTR * DELETE_ATTR * STORE_GLOBAL * STORE_ATTR_INSTANCE_VALUE * STORE_ATTR_WITH_HINT * STORE_ATTR_SLOT, and complete the store_attr family * Complete the store_subscr family: STORE_SUBSCR{,DICT,LIST_INT} (STORE_SUBSCR was alread half converted, but wasn't using cache effects yet.) * DELETE_SUBSCR * PRINT_EXPR * INTERPRETER_EXIT (a bit weird, ends in return) * RETURN_VALUE * GET_AITER (had to restructure it some) The original had mysterious `SET_TOP(NULL)` before `goto error`. I assume those just account for `obj` having been decref'ed, so I got rid of them in favor of the cleanup implied by `ERROR_IF()`. * LIST_APPEND (a bit unhappy with it) * SET_ADD (also a bit unhappy with it) Various other improvements/refactorings as well. files: M Python/bytecodes.c M Python/generated_cases.c.h M Tools/cases_generator/generate_cases.py M Tools/cases_generator/parser.py diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d0480ac01eb6..7a1bfdcc9e26 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -81,8 +81,17 @@ do { \ // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; static PyObject *container, *start, *stop, *v, *lhs, *rhs; -static PyObject *list, *tuple, *dict; -static PyObject *exit_func, *lasti, *val; +static PyObject *list, *tuple, *dict, *owner; +static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter; +static size_t jump; +// Dummy variables for cache effects +static _Py_CODEUNIT when_to_jump_mask, invert, counter, index, hint; +static uint32_t type_version; +// Dummy opcode names for 'op' opcodes +#define _COMPARE_OP_FLOAT 1003 +#define _COMPARE_OP_INT 1004 +#define _COMPARE_OP_STR 1005 +#define _JUMP_IF 1006 static PyObject * dummy_func( @@ -205,7 +214,7 @@ dummy_func( }; - inst(BINARY_OP_MULTIPLY_INT, (left, right, unused/1 -- prod)) { + inst(BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- prod)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); @@ -216,7 +225,7 @@ dummy_func( ERROR_IF(prod == NULL, error); } - inst(BINARY_OP_MULTIPLY_FLOAT, (left, right, unused/1 -- prod)) { + inst(BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- prod)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); @@ -229,7 +238,7 @@ dummy_func( ERROR_IF(prod == NULL, error); } - inst(BINARY_OP_SUBTRACT_INT, (left, right, unused/1 -- sub)) { + inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); @@ -240,7 +249,7 @@ dummy_func( ERROR_IF(sub == NULL, error); } - inst(BINARY_OP_SUBTRACT_FLOAT, (left, right, unused/1 -- sub)) { + inst(BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- sub)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); @@ -252,7 +261,7 @@ dummy_func( ERROR_IF(sub == NULL, error); } - inst(BINARY_OP_ADD_UNICODE, (left, right, unused/1 -- res)) { + inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -299,7 +308,7 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); } - inst(BINARY_OP_ADD_FLOAT, (left, right, unused/1 -- sum)) { + inst(BINARY_OP_ADD_FLOAT, (unused/1, left, right -- sum)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -312,7 +321,7 @@ dummy_func( ERROR_IF(sum == NULL, error); } - inst(BINARY_OP_ADD_INT, (left, right, unused/1 -- sum)) { + inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -331,7 +340,7 @@ dummy_func( BINARY_SUBSCR_TUPLE_INT, }; - inst(BINARY_SUBSCR, (container, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR, (unused/4, container, sub -- res)) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -377,7 +386,7 @@ dummy_func( ERROR_IF(err, error); } - inst(BINARY_SUBSCR_LIST_INT, (list, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR_LIST_INT, (unused/4, list, sub -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -396,7 +405,7 @@ dummy_func( Py_DECREF(list); } - inst(BINARY_SUBSCR_TUPLE_INT, (tuple, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR_TUPLE_INT, (unused/4, tuple, sub -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -415,7 +424,7 @@ dummy_func( Py_DECREF(tuple); } - inst(BINARY_SUBSCR_DICT, (dict, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR_DICT, (unused/4, dict, sub -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -426,14 +435,14 @@ dummy_func( } Py_DECREF(dict); Py_DECREF(sub); - ERROR_IF(1, error); + ERROR_IF(true, error); } Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); Py_DECREF(sub); } - inst(BINARY_SUBSCR_GETITEM, (container, sub, unused/1, type_version/2, func_version/1 -- unused)) { + inst(BINARY_SUBSCR_GETITEM, (unused/1, type_version/2, func_version/1, container, sub -- unused)) { PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -457,52 +466,48 @@ dummy_func( DISPATCH_INLINED(new_frame); } - // stack effect: (__0 -- ) - inst(LIST_APPEND) { - PyObject *v = POP(); - PyObject *list = PEEK(oparg); - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) - goto error; + // Alternative: (list, unused[oparg], v -- list, unused[oparg]) + inst(LIST_APPEND, (v --)) { + PyObject *list = PEEK(oparg + 1); // +1 to account for v staying on stack + ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error); PREDICT(JUMP_BACKWARD); } - // stack effect: (__0 -- ) - inst(SET_ADD) { - PyObject *v = POP(); - PyObject *set = PEEK(oparg); - int err; - err = PySet_Add(set, v); + // Alternative: (set, unused[oparg], v -- set, unused[oparg]) + inst(SET_ADD, (v --)) { + PyObject *set = PEEK(oparg + 1); // +1 to account for v staying on stack + int err = PySet_Add(set, v); Py_DECREF(v); - if (err != 0) - goto error; + ERROR_IF(err, error); PREDICT(JUMP_BACKWARD); } - inst(STORE_SUBSCR, (v, container, sub -- )) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + family(store_subscr) = { + STORE_SUBSCR, + STORE_SUBSCR_DICT, + STORE_SUBSCR_LIST_INT, + }; + + inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - ERROR_IF(err != 0, error); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + ERROR_IF(err, error); } - // stack effect: (__0, __1, __2 -- ) - inst(STORE_SUBSCR_LIST_INT) { + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *list = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -515,60 +520,42 @@ dummy_func( PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); - STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); } - // stack effect: (__0, __1, __2 -- ) - inst(STORE_SUBSCR_DICT) { + inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *dict = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STACK_SHRINK(3); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + ERROR_IF(err, error); } - // stack effect: (__0, __1 -- ) - inst(DELETE_SUBSCR) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - int err; - STACK_SHRINK(2); + inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ - err = PyObject_DelItem(container, sub); + int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) - goto error; + ERROR_IF(err, error); } - // stack effect: (__0 -- ) - inst(PRINT_EXPR) { - PyObject *value = POP(); + inst(PRINT_EXPR, (value --)) { PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook)); PyObject *res; + // Can't use ERROR_IF here. if (hook == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); Py_DECREF(value); - goto error; + ERROR_IF(true, error); } res = PyObject_CallOneArg(hook, value); Py_DECREF(value); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); Py_DECREF(res); } @@ -595,11 +582,10 @@ dummy_func( goto error; } - // stack effect: (__0 -- ) - inst(INTERPRETER_EXIT) { + inst(INTERPRETER_EXIT, (retval --)) { assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - PyObject *retval = POP(); + STACK_SHRINK(1); // Since we're not going to DISPATCH() assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; @@ -610,9 +596,8 @@ dummy_func( return retval; } - // stack effect: (__0 -- ) - inst(RETURN_VALUE) { - PyObject *retval = POP(); + inst(RETURN_VALUE, (retval --)) { + STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); @@ -627,48 +612,37 @@ dummy_func( goto resume_frame; } - // stack effect: ( -- ) - inst(GET_AITER) { + inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; - PyObject *iter = NULL; - PyObject *obj = TOP(); PyTypeObject *type = Py_TYPE(obj); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter != NULL) { - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) { - SET_TOP(NULL); - goto error; - } - } - else { - SET_TOP(NULL); + if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - goto error; + ERROR_IF(true, error); } + iter = (*getter)(obj); + Py_DECREF(obj); + ERROR_IF(iter == NULL, error); + if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - SET_TOP(NULL); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - goto error; + ERROR_IF(true, error); } - - SET_TOP(iter); } // stack effect: ( -- __0) @@ -1119,53 +1093,43 @@ dummy_func( Py_DECREF(seq); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + family(store_attr) = { + STORE_ATTR, + STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR_SLOT, + STORE_ATTR_WITH_HINT, + }; + + inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); + int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + ERROR_IF(err, error); } - // stack effect: (__0 -- ) - inst(DELETE_ATTR) { + inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(names, oparg); - PyObject *owner = POP(); - int err; - err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); - if (err != 0) - goto error; + ERROR_IF(err, error); } - // stack effect: (__0 -- ) - inst(STORE_GLOBAL) { + inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); - int err; - err = PyDict_SetItem(GLOBALS(), name, v); + int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err != 0) - goto error; + ERROR_IF(err, error); } inst(DELETE_GLOBAL, (--)) { @@ -1954,22 +1918,15 @@ dummy_func( DISPATCH_INLINED(new_frame); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR_INSTANCE_VALUE) { + inst(STORE_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, value, owner --)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); STAT_INC(STORE_ATTR, hit); - Py_ssize_t index = cache->index; - STACK_SHRINK(1); - PyObject *value = POP(); PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; values->values[index] = value; @@ -1980,16 +1937,11 @@ dummy_func( Py_DECREF(old_value); } Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR_WITH_HINT) { + inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -1999,17 +1951,14 @@ dummy_func( DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg); - uint16_t hint = cache->index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *value, *old_value; + PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2018,8 +1967,6 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2032,36 +1979,32 @@ dummy_func( /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR_SLOT) { + inst(STORE_ATTR_SLOT, (unused/1, type_version/2, index/1, value, owner --)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - char *addr = (char *)owner + cache->index; + char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); - STACK_SHRINK(1); - PyObject *value = POP(); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } - // stack effect: (__0 -- ) - inst(COMPARE_OP) { + family(compare_op) = { + COMPARE_OP, + _COMPARE_OP_FLOAT, + _COMPARE_OP_INT, + _COMPARE_OP_STR, + }; + + inst(COMPARE_OP, (unused/2, left, right -- res)) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *right = TOP(); - PyObject *left = SECOND(); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2069,57 +2012,43 @@ dummy_func( STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); + res = PyObject_RichCompare(left, right, oparg); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + ERROR_IF(res == NULL, error); } - // stack effect: (__0 -- ) - inst(COMPARE_OP_FLOAT_JUMP) { + // The result is an int disguised as an object pointer. + op(_COMPARE_OP_FLOAT, (unused/1, when_to_jump_mask/1, left, right -- jump: size_t)) { assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - int sign = (dleft > dright) - (dleft < dright); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); DEOPT_IF(isnan(dleft), COMPARE_OP); DEOPT_IF(isnan(dright), COMPARE_OP); STAT_INC(COMPARE_OP, hit); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + jump = sign_ish & when_to_jump_mask; + } + // The input is an int disguised as an object pointer! + op(_JUMP_IF, (jump: size_t --)) { assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); + if (jump) { + JUMPBY(oparg); } } + // We're praying that the compiler optimizes the flags manipuations. + super(COMPARE_OP_FLOAT_JUMP) = _COMPARE_OP_FLOAT + _JUMP_IF; - // stack effect: (__0 -- ) - inst(COMPARE_OP_INT_JUMP) { + // Similar to COMPARE_OP_FLOAT + op(_COMPARE_OP_INT, (unused/1, when_to_jump_mask/1, left, right -- jump: size_t)) { assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); @@ -2128,51 +2057,30 @@ dummy_func( assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - int sign = (ileft > iright) - (ileft < iright); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } + jump = sign_ish & when_to_jump_mask; } + super(COMPARE_OP_INT_JUMP) = _COMPARE_OP_INT + _JUMP_IF; - // stack effect: (__0 -- ) - inst(COMPARE_OP_STR_JUMP) { + // Similar to COMPARE_OP_FLOAT, but for ==, != only + op(_COMPARE_OP_STR, (unused/1, invert/1, left, right -- jump: size_t)) { assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int invert = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); int res = _PyUnicode_Equal(left, right); assert(oparg == Py_EQ || oparg == Py_NE); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - STACK_SHRINK(2); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(res == 0 || res == 1); assert(invert == 0 || invert == 1); - int jump = res ^ invert; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } + jump = res ^ invert; } + super(COMPARE_OP_STR_JUMP) = _COMPARE_OP_STR + _JUMP_IF; // stack effect: (__0 -- ) inst(IS_OP) { @@ -3633,7 +3541,7 @@ dummy_func( PUSH(Py_NewRef(peek)); } - inst(BINARY_OP, (lhs, rhs, unused/1 -- res)) { + inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -3691,9 +3599,6 @@ dummy_func( // Future families go below this point // -family(binary_subscr) = { - BINARY_SUBSCR, BINARY_SUBSCR_DICT, - BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, BINARY_SUBSCR_TUPLE_INT }; family(call) = { CALL, CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS, @@ -3702,9 +3607,6 @@ family(call) = { CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, CALL_NO_KW_TYPE_1 }; -family(compare_op) = { - COMPARE_OP, COMPARE_OP_FLOAT_JUMP, - COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP }; family(for_iter) = { FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; @@ -3719,13 +3621,7 @@ family(load_fast) = { LOAD_FAST, LOAD_FAST__LOAD_CONST, LOAD_FAST__LOAD_FAST }; family(load_global) = { LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN, LOAD_GLOBAL_MODULE }; -family(store_attr) = { - STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, - STORE_ATTR_SLOT, STORE_ATTR_WITH_HINT }; family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST }; -family(store_subscr) = { - STORE_SUBSCR, STORE_SUBSCR_DICT, - STORE_SUBSCR_LIST_INT }; family(unpack_sequence) = { UNPACK_SEQUENCE, UNPACK_SEQUENCE_LIST, UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_TWO_TUPLE }; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0805386866b3..510b6c4a75b8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -139,7 +139,7 @@ if (prod == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, prod); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -159,7 +159,7 @@ if (prod == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, prod); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -177,7 +177,7 @@ if (sub == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sub); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -196,7 +196,7 @@ if (sub == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sub); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -214,7 +214,7 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, res); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -268,7 +268,7 @@ if (sum == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sum); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -286,7 +286,7 @@ if (sum == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sum); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -311,7 +311,7 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -380,7 +380,7 @@ Py_DECREF(list); STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -406,7 +406,7 @@ Py_DECREF(tuple); STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -424,14 +424,14 @@ } Py_DECREF(dict); Py_DECREF(sub); - if (1) goto pop_2_error; + if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -464,22 +464,21 @@ } TARGET(LIST_APPEND) { - PyObject *v = POP(); - PyObject *list = PEEK(oparg); - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) - goto error; + PyObject *v = PEEK(1); + PyObject *list = PEEK(oparg + 1); // +1 to account for v staying on stack + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); } TARGET(SET_ADD) { - PyObject *v = POP(); - PyObject *set = PEEK(oparg); - int err; - err = PySet_Add(set, v); + PyObject *v = PEEK(1); + PyObject *set = PEEK(oparg + 1); // +1 to account for v staying on stack + int err = PySet_Add(set, v); Py_DECREF(v); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); } @@ -489,31 +488,32 @@ PyObject *sub = PEEK(1); PyObject *container = PEEK(2); PyObject *v = PEEK(3); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + uint16_t counter = read_u16(next_instr + 0); + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) goto pop_3_error; - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (err) goto pop_3_error; STACK_SHRINK(3); + JUMPBY(1); DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { + PyObject *sub = PEEK(1); + PyObject *list = PEEK(2); + PyObject *value = PEEK(3); assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *list = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -526,61 +526,58 @@ PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); - STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + STACK_SHRINK(3); + JUMPBY(1); DISPATCH(); } TARGET(STORE_SUBSCR_DICT) { + PyObject *sub = PEEK(1); + PyObject *dict = PEEK(2); + PyObject *value = PEEK(3); assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *dict = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STACK_SHRINK(3); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (err) goto pop_3_error; + STACK_SHRINK(3); + JUMPBY(1); DISPATCH(); } TARGET(DELETE_SUBSCR) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - int err; - STACK_SHRINK(2); + PyObject *sub = PEEK(1); + PyObject *container = PEEK(2); /* del container[sub] */ - err = PyObject_DelItem(container, sub); + int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) - goto error; + if (err) goto pop_2_error; + STACK_SHRINK(2); DISPATCH(); } TARGET(PRINT_EXPR) { - PyObject *value = POP(); + PyObject *value = PEEK(1); PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook)); PyObject *res; + // Can't use ERROR_IF here. if (hook == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); Py_DECREF(value); - goto error; + if (true) goto pop_1_error; } res = PyObject_CallOneArg(hook, value); Py_DECREF(value); - if (res == NULL) - goto error; + if (res == NULL) goto pop_1_error; Py_DECREF(res); + STACK_SHRINK(1); DISPATCH(); } @@ -607,9 +604,10 @@ } TARGET(INTERPRETER_EXIT) { + PyObject *retval = PEEK(1); assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - PyObject *retval = POP(); + STACK_SHRINK(1); // Since we're not going to DISPATCH() assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; @@ -621,7 +619,8 @@ } TARGET(RETURN_VALUE) { - PyObject *retval = POP(); + PyObject *retval = PEEK(1); + STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); @@ -637,46 +636,39 @@ } TARGET(GET_AITER) { + PyObject *obj = PEEK(1); + PyObject *iter; unaryfunc getter = NULL; - PyObject *iter = NULL; - PyObject *obj = TOP(); PyTypeObject *type = Py_TYPE(obj); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter != NULL) { - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) { - SET_TOP(NULL); - goto error; - } - } - else { - SET_TOP(NULL); + if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - goto error; + if (true) goto pop_1_error; } + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; + if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - SET_TOP(NULL); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - goto error; + if (true) goto pop_1_error; } - - SET_TOP(iter); + POKE(1, iter); DISPATCH(); } @@ -1131,51 +1123,46 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + PyObject *owner = PEEK(1); + PyObject *v = PEEK(2); + uint16_t counter = read_u16(next_instr + 0); + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); + int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + if (err) goto pop_2_error; + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(DELETE_ATTR) { + PyObject *owner = PEEK(1); PyObject *name = GETITEM(names, oparg); - PyObject *owner = POP(); - int err; - err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { + PyObject *v = PEEK(1); PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); - int err; - err = PyDict_SetItem(GLOBALS(), name, v); + int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } @@ -1970,20 +1957,18 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { + PyObject *owner = PEEK(1); + PyObject *value = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t index = read_u16(next_instr + 3); assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); STAT_INC(STORE_ATTR, hit); - Py_ssize_t index = cache->index; - STACK_SHRINK(1); - PyObject *value = POP(); PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; values->values[index] = value; @@ -1994,16 +1979,18 @@ Py_DECREF(old_value); } Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(STORE_ATTR_WITH_HINT) { + PyObject *owner = PEEK(1); + PyObject *value = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t hint = read_u16(next_instr + 3); assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -2013,17 +2000,14 @@ DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg); - uint16_t hint = cache->index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *value, *old_value; + PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2032,8 +2016,6 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2046,37 +2028,39 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(STORE_ATTR_SLOT) { + PyObject *owner = PEEK(1); + PyObject *value = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t index = read_u16(next_instr + 3); assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - char *addr = (char *)owner + cache->index; + char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); - STACK_SHRINK(1); - PyObject *value = POP(); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(COMPARE_OP) { PREDICTED(COMPARE_OP); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *res; _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *right = TOP(); - PyObject *left = SECOND(); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2084,109 +2068,13 @@ STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); + res = PyObject_RichCompare(left, right, oparg); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - DISPATCH(); - } - - TARGET(COMPARE_OP_FLOAT_JUMP) { - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - int sign = (dleft > dright) - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } - DISPATCH(); - } - - TARGET(COMPARE_OP_INT_JUMP) { - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); - Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; - Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - int sign = (ileft > iright) - (ileft < iright); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } - DISPATCH(); - } - - TARGET(COMPARE_OP_STR_JUMP) { - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int invert = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int res = _PyUnicode_Equal(left, right); - assert(oparg == Py_EQ || oparg == Py_NE); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(res == 0 || res == 1); - assert(invert == 0 || invert == 1); - int jump = res ^ invert; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(2); DISPATCH(); } @@ -3681,7 +3569,7 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, res); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -3714,20 +3602,20 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_1 = value; + _tmp_2 = value; } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_2 = value; + _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_2); - POKE(2, _tmp_1); + POKE(1, _tmp_1); + POKE(2, _tmp_2); DISPATCH(); } @@ -3739,19 +3627,19 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_1 = value; + _tmp_2 = value; } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETITEM(consts, oparg); Py_INCREF(value); - _tmp_2 = value; + _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_2); - POKE(2, _tmp_1); + POKE(1, _tmp_1); + POKE(2, _tmp_2); DISPATCH(); } @@ -3762,7 +3650,7 @@ SETLOCAL(oparg, value); } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETLOCAL(oparg); @@ -3775,16 +3663,16 @@ } TARGET(STORE_FAST__STORE_FAST) { - PyObject *_tmp_1 = PEEK(2); - PyObject *_tmp_2 = PEEK(1); + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); { - PyObject *value = _tmp_2; + PyObject *value = _tmp_1; SETLOCAL(oparg, value); } NEXTOPARG(); - next_instr++; + JUMPBY(1); { - PyObject *value = _tmp_1; + PyObject *value = _tmp_2; SETLOCAL(oparg, value); } STACK_SHRINK(2); @@ -3798,32 +3686,145 @@ PyObject *value; value = GETITEM(consts, oparg); Py_INCREF(value); - _tmp_1 = value; + _tmp_2 = value; } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_2 = value; + _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_2); - POKE(2, _tmp_1); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + + TARGET(COMPARE_OP_FLOAT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); + DEOPT_IF(isnan(dleft), COMPARE_OP); + DEOPT_IF(isnan(dright), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_INT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); + Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; + Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_STR_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t invert = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int res = _PyUnicode_Equal(left, right); + assert(oparg == Py_EQ || oparg == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(res == 0 || res == 1); + assert(invert == 0 || invert == 1); + jump = res ^ invert; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); DISPATCH(); } TARGET(END_FOR) { - PyObject *_tmp_1 = PEEK(2); - PyObject *_tmp_2 = PEEK(1); + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); { - PyObject *value = _tmp_2; + PyObject *value = _tmp_1; Py_DECREF(value); } { - PyObject *value = _tmp_1; + PyObject *value = _tmp_2; Py_DECREF(value); } STACK_SHRINK(2); diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 2952634a3cda..844272291245 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -13,6 +13,7 @@ import typing import parser +from parser import StackEffect DEFAULT_INPUT = os.path.relpath( os.path.join(os.path.dirname(__file__), "../../Python/bytecodes.c") @@ -22,7 +23,7 @@ ) BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" -RE_PREDICTED = r"(?s)(?:PREDICT$|GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)$;" +RE_PREDICTED = r"^\s*(?:PREDICT$|GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)$;\s*$" UNUSED = "unused" BITS_PER_CODE_UNIT = 16 @@ -73,6 +74,34 @@ def block(self, head: str): yield self.emit("}") + def stack_adjust(self, diff: int): + if diff > 0: + self.emit(f"STACK_GROW({diff});") + elif diff < 0: + self.emit(f"STACK_SHRINK({-diff});") + + def declare(self, dst: StackEffect, src: StackEffect | None): + if dst.name == UNUSED: + return + typ = f"{dst.type} " if dst.type else "PyObject *" + init = "" + if src: + cast = self.cast(dst, src) + init = f" = {cast}{src.name}" + self.emit(f"{typ}{dst.name}{init};") + + def assign(self, dst: StackEffect, src: StackEffect): + if src.name == UNUSED: + return + cast = self.cast(dst, src) + if m := re.match(r"^PEEK$(\d+)$$", dst.name): + self.emit(f"POKE({m.group(1)}, {cast}{src.name});") + else: + self.emit(f"{dst.name} = {cast}{src.name};") + + def cast(self, dst: StackEffect, src: StackEffect) -> str: + return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" + @dataclasses.dataclass class Instruction: @@ -83,13 +112,15 @@ class Instruction: kind: typing.Literal["inst", "op"] name: str block: parser.Block + block_text: list[str] # Block.text, less curlies, less PREDICT() calls + predictions: list[str] # Prediction targets (instruction names) # Computed by constructor always_exits: bool cache_offset: int cache_effects: list[parser.CacheEffect] - input_effects: list[parser.StackEffect] - output_effects: list[parser.StackEffect] + input_effects: list[StackEffect] + output_effects: list[StackEffect] # Set later family: parser.Family | None = None @@ -100,13 +131,14 @@ def __init__(self, inst: parser.InstDef): self.kind = inst.kind self.name = inst.name self.block = inst.block - self.always_exits = always_exits(self.block) + self.block_text, self.predictions = extract_block_text(self.block) + self.always_exits = always_exits(self.block_text) self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parser.CacheEffect) ] self.cache_offset = sum(c.size for c in self.cache_effects) self.input_effects = [ - effect for effect in inst.inputs if isinstance(effect, parser.StackEffect) + effect for effect in inst.inputs if isinstance(effect, StackEffect) ] self.output_effects = inst.outputs # For consistency/completeness @@ -122,42 +154,39 @@ def write(self, out: Formatter) -> None: ) # Write input stack effect variable declarations and initializations - for i, seffect in enumerate(reversed(self.input_effects), 1): - if seffect.name != UNUSED: - out.emit(f"PyObject *{seffect.name} = PEEK({i});") + for i, ieffect in enumerate(reversed(self.input_effects), 1): + src = StackEffect(f"PEEK({i})", "") + out.declare(ieffect, src) # Write output stack effect variable declarations - input_names = {seffect.name for seffect in self.input_effects} - input_names.add(UNUSED) - for seffect in self.output_effects: - if seffect.name not in input_names: - out.emit(f"PyObject *{seffect.name};") + input_names = {ieffect.name for ieffect in self.input_effects} + for oeffect in self.output_effects: + if oeffect.name not in input_names: + out.declare(oeffect, None) self.write_body(out, 0) # Skip the rest if the block always exits - if always_exits(self.block): + if self.always_exits: return # Write net stack growth/shrinkage diff = len(self.output_effects) - len(self.input_effects) - if diff > 0: - out.emit(f"STACK_GROW({diff});") - elif diff < 0: - out.emit(f"STACK_SHRINK({-diff});") + out.stack_adjust(diff) # Write output stack effect assignments - unmoved_names = {UNUSED} + unmoved_names: set[str] = set() for ieffect, oeffect in zip(self.input_effects, self.output_effects): if ieffect.name == oeffect.name: unmoved_names.add(ieffect.name) - for i, seffect in enumerate(reversed(self.output_effects)): - if seffect.name not in unmoved_names: - out.emit(f"POKE({i+1}, {seffect.name});") + for i, oeffect in enumerate(reversed(self.output_effects), 1): + if oeffect.name not in unmoved_names: + dst = StackEffect(f"PEEK({i})", "") + out.assign(dst, oeffect) # Write cache effect if self.cache_offset: - out.emit(f"next_instr += {self.cache_offset};") + out.emit(f"JUMPBY({self.cache_offset});") def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None: """Write the instruction body.""" @@ -171,36 +200,19 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None # is always an object pointer. # If this becomes false, we need a way to specify # syntactically what type the cache data is. - type = "PyObject *" + typ = "PyObject *" func = "read_obj" else: - type = f"uint{bits}_t " + typ = f"uint{bits}_t " func = f"read_u{bits}" - out.emit(f"{type}{ceffect.name} = {func}(next_instr + {cache_offset});") + out.emit(f"{typ}{ceffect.name} = {func}(next_instr + {cache_offset});") cache_offset += ceffect.size assert cache_offset == self.cache_offset + cache_adjust - # Get lines of text with proper dedent - blocklines = self.block.to_text(dedent=dedent).splitlines(True) - - # Remove blank lines from both ends - while blocklines and not blocklines[0].strip(): - blocklines.pop(0) - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Remove leading and trailing braces - assert blocklines and blocklines[0].strip() == "{" - assert blocklines and blocklines[-1].strip() == "}" - blocklines.pop() - blocklines.pop(0) - - # Remove trailing blank lines - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - # Write the body, substituting a goto for ERROR_IF() - for line in blocklines: + assert dedent <= 0 + extra = " " * -dedent + for line in self.block_text: if m := re.match(r"(\s*)ERROR_IF$(.+), (\w+)$;\s*$", line): space, cond, label = m.groups() # ERROR_IF() must pop the inputs from the stack. @@ -215,34 +227,36 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None else: break if ninputs: - out.write_raw(f"{space}if ({cond}) goto pop_{ninputs}_{label};\n") + out.write_raw( + f"{extra}{space}if ({cond}) goto pop_{ninputs}_{label};\n" + ) else: - out.write_raw(f"{space}if ({cond}) goto {label};\n") + out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") else: - out.write_raw(line) + out.write_raw(extra + line) InstructionOrCacheEffect = Instruction | parser.CacheEffect +StackEffectMapping = list[tuple[StackEffect, StackEffect]] @dataclasses.dataclass class Component: instr: Instruction - input_mapping: dict[str, parser.StackEffect] - output_mapping: dict[str, parser.StackEffect] + input_mapping: StackEffectMapping + output_mapping: StackEffectMapping def write_body(self, out: Formatter, cache_adjust: int) -> None: with out.block(""): - for var, ieffect in self.input_mapping.items(): - out.emit(f"PyObject *{ieffect.name} = {var};") - for oeffect in self.output_mapping.values(): - out.emit(f"PyObject *{oeffect.name};") - self.instr.write_body(out, dedent=-4, cache_adjust=cache_adjust) - for var, oeffect in self.output_mapping.items(): - out.emit(f"{var} = {oeffect.name};") + for var, ieffect in self.input_mapping: + out.declare(ieffect, var) + for _, oeffect in self.output_mapping: + out.declare(oeffect, None) + self.instr.write_body(out, dedent=-4, cache_adjust=cache_adjust) -# TODO: Use a common base class for {Super,Macro}Instruction + for var, oeffect in self.output_mapping: + out.assign(var, oeffect) @dataclasses.dataclass @@ -250,7 +264,7 @@ class SuperOrMacroInstruction: """Common fields for super- and macro instructions.""" name: str - stack: list[str] + stack: list[StackEffect] initial_sp: int final_sp: int @@ -369,7 +383,11 @@ def analyze(self) -> None: def find_predictions(self) -> None: """Find the instructions that need PREDICTED() labels.""" for instr in self.instrs.values(): - for target in re.findall(RE_PREDICTED, instr.block.text): + targets = set(instr.predictions) + for line in instr.block_text: + if m := re.match(RE_PREDICTED, line): + targets.add(m.group(1)) + for target in targets: if target_instr := self.instrs.get(target): target_instr.predicted = True else: @@ -440,24 +458,9 @@ def analyze_super(self, super: parser.Super) -> SuperInstruction: stack, initial_sp = self.stack_analysis(components) sp = initial_sp parts: list[Component] = [] - for component in components: - match component: - case parser.CacheEffect() as ceffect: - parts.append(ceffect) - case Instruction() as instr: - input_mapping = {} - for ieffect in reversed(instr.input_effects): - sp -= 1 - if ieffect.name != UNUSED: - input_mapping[stack[sp]] = ieffect - output_mapping = {} - for oeffect in instr.output_effects: - if oeffect.name != UNUSED: - output_mapping[stack[sp]] = oeffect - sp += 1 - parts.append(Component(instr, input_mapping, output_mapping)) - case _: - typing.assert_never(component) + for instr in components: + part, sp = self.analyze_instruction(instr, stack, sp) + parts.append(part) final_sp = sp return SuperInstruction(super.name, stack, initial_sp, final_sp, super, parts) @@ -471,22 +474,26 @@ def analyze_macro(self, macro: parser.Macro) -> MacroInstruction: case parser.CacheEffect() as ceffect: parts.append(ceffect) case Instruction() as instr: - input_mapping = {} - for ieffect in reversed(instr.input_effects): - sp -= 1 - if ieffect.name != UNUSED: - input_mapping[stack[sp]] = ieffect - output_mapping = {} - for oeffect in instr.output_effects: - if oeffect.name != UNUSED: - output_mapping[stack[sp]] = oeffect - sp += 1 - parts.append(Component(instr, input_mapping, output_mapping)) + part, sp = self.analyze_instruction(instr, stack, sp) + parts.append(part) case _: typing.assert_never(component) final_sp = sp return MacroInstruction(macro.name, stack, initial_sp, final_sp, macro, parts) + def analyze_instruction( + self, instr: Instruction, stack: list[StackEffect], sp: int + ) -> tuple[Component, int]: + input_mapping: StackEffectMapping = [] + for ieffect in reversed(instr.input_effects): + sp -= 1 + input_mapping.append((stack[sp], ieffect)) + output_mapping: StackEffectMapping = [] + for oeffect in instr.output_effects: + output_mapping.append((stack[sp], oeffect)) + sp += 1 + return Component(instr, input_mapping, output_mapping), sp + def check_super_components(self, super: parser.Super) -> list[Instruction]: components: list[Instruction] = [] for op in super.ops: @@ -514,7 +521,7 @@ def check_macro_components( def stack_analysis( self, components: typing.Iterable[InstructionOrCacheEffect] - ) -> tuple[list[str], int]: + ) -> tuple[list[StackEffect], int]: """Analyze a super-instruction or macro. Print an error if there's a cache effect (which we don't support yet). @@ -536,7 +543,10 @@ def stack_analysis( # At this point, 'current' is the net stack effect, # and 'lowest' and 'highest' are the extremes. # Note that 'lowest' may be negative. - stack = [f"_tmp_{i+1}" for i in range(highest - lowest)] + # TODO: Reverse the numbering. + stack = [ + StackEffect(f"_tmp_{i+1}", "") for i in reversed(range(highest - lowest)) + ] return stack, -lowest def write_instructions(self) -> None: @@ -561,7 +571,9 @@ def write_instructions(self) -> None: if instr.predicted: self.out.emit(f"PREDICTED({name});") instr.write(self.out) - if not always_exits(instr.block): + if not instr.always_exits: + for prediction in instr.predictions: + self.out.emit(f"PREDICT({prediction});") self.out.emit(f"DISPATCH();") # Write and count super-instructions @@ -589,11 +601,11 @@ def write_super(self, sup: SuperInstruction) -> None: for comp in sup.parts: if not first: self.out.emit("NEXTOPARG();") - self.out.emit("next_instr++;") + self.out.emit("JUMPBY(1);") first = False comp.write_body(self.out, 0) if comp.instr.cache_offset: - self.out.emit(f"next_instr += {comp.instr.cache_offset};") + self.out.emit(f"JUMPBY({comp.instr.cache_offset});") def write_macro(self, mac: MacroInstruction) -> None: """Write code for a macro instruction.""" @@ -608,43 +620,68 @@ def write_macro(self, mac: MacroInstruction) -> None: cache_adjust += comp.instr.cache_offset if cache_adjust: - self.out.emit(f"next_instr += {cache_adjust};") + self.out.emit(f"JUMPBY({cache_adjust});") @contextlib.contextmanager def wrap_super_or_macro(self, up: SuperOrMacroInstruction): """Shared boilerplate for super- and macro instructions.""" + # TODO: Somewhere (where?) make it so that if one instruction + # has an output that is input to another, and the variable names + # and types match and don't conflict with other instructions, + # that variable is declared with the right name and type in the + # outer block, rather than trusting the compiler to optimize it. self.out.emit("") with self.out.block(f"TARGET({up.name})"): - for i, var in enumerate(up.stack): + for i, var in reversed(list(enumerate(up.stack))): + src = None if i < up.initial_sp: - self.out.emit(f"PyObject *{var} = PEEK({up.initial_sp - i});") - else: - self.out.emit(f"PyObject *{var};") + src = StackEffect(f"PEEK({up.initial_sp - i})", "") + self.out.declare(var, src) yield - if up.final_sp > up.initial_sp: - self.out.emit(f"STACK_GROW({up.final_sp - up.initial_sp});") - elif up.final_sp < up.initial_sp: - self.out.emit(f"STACK_SHRINK({up.initial_sp - up.final_sp});") + self.out.stack_adjust(up.final_sp - up.initial_sp) for i, var in enumerate(reversed(up.stack[: up.final_sp]), 1): - self.out.emit(f"POKE({i}, {var});") + dst = StackEffect(f"PEEK({i})", "") + self.out.assign(dst, var) self.out.emit(f"DISPATCH();") -def always_exits(block: parser.Block) -> bool: +def extract_block_text(block: parser.Block) -> tuple[list[str], list[str]]: + # Get lines of text with proper dedent + blocklines = block.text.splitlines(True) + + # Remove blank lines from both ends + while blocklines and not blocklines[0].strip(): + blocklines.pop(0) + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + + # Remove leading and trailing braces + assert blocklines and blocklines[0].strip() == "{" + assert blocklines and blocklines[-1].strip() == "}" + blocklines.pop() + blocklines.pop(0) + + # Remove trailing blank lines + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + + # Separate PREDICT(...) macros from end + predictions: list[str] = [] + while blocklines and (m := re.match(r"^\s*PREDICT$(\w+)$;\s*$", blocklines[-1])): + predictions.insert(0, m.group(1)) + blocklines.pop() + + return blocklines, predictions + + +def always_exits(lines: list[str]) -> bool: """Determine whether a block always ends in a return/goto/etc.""" - text = block.text - lines = text.splitlines() - while lines and not lines[-1].strip(): - lines.pop() - if not lines or lines[-1].strip() != "}": - return False - lines.pop() if not lines: return False - line = lines.pop().rstrip() + line = lines[-1].rstrip() # Indent must match exactly (TODO: Do something better) if line[:12] != " " * 12: return False diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 02a7834d2215..d802c733dfd1 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -62,7 +62,8 @@ class Block(Node): @dataclass class StackEffect(Node): name: str - # TODO: type, condition + type: str = "" + # TODO: array, condition @dataclass @@ -147,7 +148,7 @@ def inst_header(self) -> InstHeader | None: if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)): name = tkn.text if self.expect(lx.COMMA): - inp, outp = self.stack_effect() + inp, outp = self.io_effect() if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: return InstHeader(kind, name, inp, outp) @@ -156,7 +157,7 @@ def inst_header(self) -> InstHeader | None: return InstHeader(kind, name, [], []) return None - def stack_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: + def io_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: # '(' [inputs] '--' [outputs] ')' if self.expect(lx.LPAREN): inputs = self.inputs() or [] @@ -181,23 +182,7 @@ def inputs(self) -> list[InputEffect] | None: @contextual def input(self) -> InputEffect | None: - # IDENTIFIER '/' INTEGER (CacheEffect) - # IDENTIFIER (StackEffect) - if tkn := self.expect(lx.IDENTIFIER): - if self.expect(lx.DIVIDE): - if num := self.expect(lx.NUMBER): - try: - size = int(num.text) - except ValueError: - raise self.make_syntax_error( - f"Expected integer, got {num.text!r}" - ) - else: - return CacheEffect(tkn.text, size) - raise self.make_syntax_error("Expected integer") - else: - # TODO: Arrays, conditions - return StackEffect(tkn.text) + return self.cache_effect() or self.stack_effect() def outputs(self) -> list[OutputEffect] | None: # output (, output)* @@ -214,8 +199,30 @@ def outputs(self) -> list[OutputEffect] | None: @contextual def output(self) -> OutputEffect | None: + return self.stack_effect() + + @contextual + def cache_effect(self) -> CacheEffect | None: + # IDENTIFIER '/' NUMBER + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.DIVIDE): + num = self.require(lx.NUMBER).text + try: + size = int(num) + except ValueError: + raise self.make_syntax_error(f"Expected integer, got {num!r}") + else: + return CacheEffect(tkn.text, size) + + @contextual + def stack_effect(self) -> StackEffect | None: + # IDENTIFIER [':' IDENTIFIER] + # TODO: Arrays, conditions if tkn := self.expect(lx.IDENTIFIER): - return StackEffect(tkn.text) + type = "" + if self.expect(lx.COLON): + type = self.require(lx.IDENTIFIER).text + return StackEffect(tkn.text, type) @contextual def super_def(self) -> Super | None: From webhook-mailer at python.org Thu Dec 8 17:38:12 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Thu, 08 Dec 2022 22:38:12 -0000 Subject: [Python-checkins] gh-81057: Move OS-Related Globals to _PyRuntimeState (gh-100082) Message-ID: <mailman.2868.1670539093.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cda9f0236fd7800c6db5eec207668c4bfec4a45d commit: cda9f0236fd7800c6db5eec207668c4bfec4a45d branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T15:38:06-07:00 summary: gh-81057: Move OS-Related Globals to _PyRuntimeState (gh-100082) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_os.h M Include/internal/pycore_global_objects_fini_generated.h M Include/internal/pycore_global_strings.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Include/internal/pycore_runtime_init_generated.h M Include/internal/pycore_unicodeobject_generated.h M Lib/test/test_os.py M Makefile.pre.in M Modules/posixmodule.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9951fa9951e6..6aba2f19ebde 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1051,6 +1051,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(node_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ns)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nstype)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(null)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(number)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(obj)); @@ -1089,6 +1090,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(posix)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 12144b02f455..acb9a4fbb92d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -537,6 +537,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(node_offset) STRUCT_FOR_ID(ns) STRUCT_FOR_ID(nstype) + STRUCT_FOR_ID(nt) STRUCT_FOR_ID(null) STRUCT_FOR_ID(number) STRUCT_FOR_ID(obj) @@ -575,6 +576,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(pos) STRUCT_FOR_ID(pos1) STRUCT_FOR_ID(pos2) + STRUCT_FOR_ID(posix) STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) diff --git a/Include/internal/pycore_os.h b/Include/internal/pycore_os.h new file mode 100644 index 000000000000..e4899bd5032d --- /dev/null +++ b/Include/internal/pycore_os.h @@ -0,0 +1,38 @@ +#ifndef Py_INTERNAL_OS_H +#define Py_INTERNAL_OS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#ifndef MS_WINDOWS +#define _OS_NEED_TICKS_PER_SECOND +# define need_ticks_per_second_STATE \ + long ticks_per_second; +# define need_ticks_per_second_INIT \ + .ticks_per_second = -1, +#else +# define need_ticks_per_second_STATE +# define need_ticks_per_second_INIT +#endif /* MS_WINDOWS */ + + +struct _os_runtime_state { + int _not_used; + need_ticks_per_second_STATE +}; +# define _OS_RUNTIME_INIT \ + { \ + ._not_used = 0, \ + need_ticks_per_second_INIT \ + } + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OS_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0720e2ed4422..c6b48cae7760 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_os.h" // struct _os_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -103,6 +104,7 @@ typedef struct pyruntimestate { * KeyboardInterrupt exception, suggesting the user pressed ^C. */ int unhandled_keyboard_interrupt; } signals; + struct _os_runtime_state os; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ab53876e355f..70263892e570 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -26,6 +26,7 @@ extern "C" { }, \ .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ .pyhash_state = pyhash_state_INIT, \ + .os = _OS_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 87b0f2ed8dfa..6d1b8702c776 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1043,6 +1043,7 @@ extern "C" { INIT_ID(node_offset), \ INIT_ID(ns), \ INIT_ID(nstype), \ + INIT_ID(nt), \ INIT_ID(null), \ INIT_ID(number), \ INIT_ID(obj), \ @@ -1081,6 +1082,7 @@ extern "C" { INIT_ID(pos), \ INIT_ID(pos1), \ INIT_ID(pos2), \ + INIT_ID(posix), \ INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 80be342b5b3b..7f407c0141b8 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -980,6 +980,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(nstype); PyUnicode_InternInPlace(&string); + string = &_Py_ID(nt); + PyUnicode_InternInPlace(&string); string = &_Py_ID(null); PyUnicode_InternInPlace(&string); string = &_Py_ID(number); @@ -1056,6 +1058,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(pos2); PyUnicode_InternInPlace(&string); + string = &_Py_ID(posix); + PyUnicode_InternInPlace(&string); string = &_Py_ID(print_file_and_line); PyUnicode_InternInPlace(&string); string = &_Py_ID(priority); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 94db8bb7737a..e0577916428a 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -606,12 +606,13 @@ def test_stat_attributes_bytes(self): def test_stat_result_pickle(self): result = os.stat(self.fname) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - p = pickle.dumps(result, proto) - self.assertIn(b'stat_result', p) - if proto < 4: - self.assertIn(b'cos\nstat_result\n', p) - unpickled = pickle.loads(p) - self.assertEqual(result, unpickled) + with self.subTest(f'protocol {proto}'): + p = pickle.dumps(result, proto) + self.assertIn(b'stat_result', p) + if proto < 4: + self.assertIn(b'cos\nstat_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index f6df7a620dea..80144e528875 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1654,6 +1654,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ + $(srcdir)/Include/internal/pycore_os.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7fc8aef9b303..cbf4d5b3fcec 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -495,9 +495,11 @@ extern char *ctermid_r(char *); #ifdef MS_WINDOWS # define INITFUNC PyInit_nt # define MODNAME "nt" +# define MODNAME_OBJ &_Py_ID(nt) #else # define INITFUNC PyInit_posix # define MODNAME "posix" +# define MODNAME_OBJ &_Py_ID(posix) #endif #if defined(__sun) @@ -974,6 +976,7 @@ typedef struct { #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) PyObject *SchedParamType; #endif + newfunc statresult_new_orig; PyObject *StatResultType; PyObject *StatVFSResultType; PyObject *TerminalSizeType; @@ -2225,7 +2228,6 @@ static PyStructSequence_Desc waitid_result_desc = { 5 }; #endif -static newfunc structseq_new; static PyObject * statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -2233,6 +2235,18 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyStructSequence *result; int i; + // ht_module doesn't get set in PyStructSequence_NewType(), + // so we can't use PyType_GetModule(). + PyObject *mod = PyImport_GetModule(MODNAME_OBJ); + if (mod == NULL) { + return NULL; + } + _posixstate *state = get_posix_state(mod); + if (state == NULL) { + return NULL; + } +#define structseq_new state->statresult_new_orig + result = (PyStructSequence*)structseq_new(type, args, kwds); if (!result) return NULL; @@ -9051,10 +9065,23 @@ build_times_result(PyObject *module, double user, double system, } -#ifndef MS_WINDOWS -#define NEED_TICKS_PER_SECOND -static long ticks_per_second = -1; -#endif /* MS_WINDOWS */ +#ifdef _OS_NEED_TICKS_PER_SECOND +#define ticks_per_second _PyRuntime.os.ticks_per_second +static void +ticks_per_second_init(void) +{ + if (ticks_per_second != -1) { + return; + } +# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) + ticks_per_second = sysconf(_SC_CLK_TCK); +# elif defined(HZ) + ticks_per_second = HZ; +# else + ticks_per_second = 60; /* magic fallback value; may be bogus */ +# endif +} +#endif /*[clinic input] os.times @@ -9089,10 +9116,10 @@ os_times_impl(PyObject *module) (double)0, (double)0); } +#elif !defined(_OS_NEED_TICKS_PER_SECOND) +# error "missing ticks_per_second" #else /* MS_WINDOWS */ { - - struct tms t; clock_t c; errno = 0; @@ -15912,7 +15939,7 @@ posixmodule_exec(PyObject *m) } PyModule_AddObject(m, "stat_result", Py_NewRef(StatResultType)); state->StatResultType = StatResultType; - structseq_new = ((PyTypeObject *)StatResultType)->tp_new; + state->statresult_new_orig = ((PyTypeObject *)StatResultType)->tp_new; ((PyTypeObject *)StatResultType)->tp_new = statresult_new; statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */ @@ -15922,14 +15949,9 @@ posixmodule_exec(PyObject *m) } PyModule_AddObject(m, "statvfs_result", Py_NewRef(StatVFSResultType)); state->StatVFSResultType = StatVFSResultType; -#ifdef NEED_TICKS_PER_SECOND -# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); -# elif defined(HZ) - ticks_per_second = HZ; -# else - ticks_per_second = 60; /* magic fallback value; may be bogus */ -# endif + +#ifdef _OS_NEED_TICKS_PER_SECOND + ticks_per_second_init(); #endif #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f62434370cfd..fa3924968a6b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -236,6 +236,7 @@ <ClInclude Include="..\Include\internal\pycore_object.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h" /> + <ClInclude Include="..\Include\internal\pycore_os.h" /> <ClInclude Include="..\Include\internal\pycore_pathconfig.h" /> <ClInclude Include="..\Include\internal\pycore_pyarena.h" /> <ClInclude Include="..\Include\internal\pycore_pyerrors.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f44a1ad8550a..e29c6b2330d1 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -612,6 +612,9 @@ <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_os.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_pathconfig.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 5c8164517f13..8e05bc3e4f23 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -390,9 +390,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/posixmodule.c os_dup2_impl dup3_works - -Modules/posixmodule.c - structseq_new - -Modules/posixmodule.c - ticks_per_second - Modules/timemodule.c _PyTime_GetClockWithInfo initialized - Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 257823574fa8..814c55b75c09 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -11,14 +11,18 @@ filename funcname name reason # These are effectively const. ##----------------------- -## process-global resources - set during first init +## process-global resources ## indicators for resource availability/capability +# (set during first init) Python/bootstrap_hash.c py_getrandom getrandom_works - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c set_inheritable ioctl_works - +# (set lazily, *after* first init) +# XXX Is this thread-safe? +Modules/posixmodule.c os_dup2_impl dup3_works - -## resource init +## resource init - set during first init Python/thread.c - initialized - Python/thread_pthread.h - condattr_monotonic - # safe static buffer used during one-time initialization From webhook-mailer at python.org Thu Dec 8 18:46:14 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Thu, 08 Dec 2022 23:46:14 -0000 Subject: [Python-checkins] gh-81057: Move time Globals to _PyRuntimeState (gh-100122) Message-ID: <mailman.2869.1670543176.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8a3f06c54b52e5e7490a131b261384baac9d6090 commit: 8a3f06c54b52e5e7490a131b261384baac9d6090 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T16:46:09-07:00 summary: gh-81057: Move time Globals to _PyRuntimeState (gh-100122) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_time.h D Include/internal/pycore_os.h M Include/internal/pycore_pylifecycle.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M Modules/posixmodule.c M Modules/timemodule.c M PCbuild/_freeze_module.vcxproj M PCbuild/_freeze_module.vcxproj.filters M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/pylifecycle.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_os.h b/Include/internal/pycore_os.h deleted file mode 100644 index e4899bd5032d..000000000000 --- a/Include/internal/pycore_os.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef Py_INTERNAL_OS_H -#define Py_INTERNAL_OS_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - - -#ifndef MS_WINDOWS -#define _OS_NEED_TICKS_PER_SECOND -# define need_ticks_per_second_STATE \ - long ticks_per_second; -# define need_ticks_per_second_INIT \ - .ticks_per_second = -1, -#else -# define need_ticks_per_second_STATE -# define need_ticks_per_second_INIT -#endif /* MS_WINDOWS */ - - -struct _os_runtime_state { - int _not_used; - need_ticks_per_second_STATE -}; -# define _OS_RUNTIME_INIT \ - { \ - ._not_used = 0, \ - need_ticks_per_second_INIT \ - } - - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_OS_H */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 4c0ffa7a9b1a..370e4cbd59f9 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -44,6 +44,7 @@ extern void _PySys_Fini(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); +extern PyStatus _PyTime_Init(void); extern PyStatus _PyImportZip_Init(PyThreadState *tstate); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c6b48cae7760..458493eac70f 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -21,7 +21,7 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state -#include "pycore_os.h" // struct _os_runtime_state +#include "pycore_time.h" // struct _time_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -104,7 +104,7 @@ typedef struct pyruntimestate { * KeyboardInterrupt exception, suggesting the user pressed ^C. */ int unhandled_keyboard_interrupt; } signals; - struct _os_runtime_state os; + struct _time_runtime_state time; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 70263892e570..ab53876e355f 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -26,7 +26,6 @@ extern "C" { }, \ .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ .pyhash_state = pyhash_state_INIT, \ - .os = _OS_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h new file mode 100644 index 000000000000..949170c44937 --- /dev/null +++ b/Include/internal/pycore_time.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_TIME_H +#define Py_INTERNAL_TIME_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +struct _time_runtime_state { +#ifdef HAVE_TIMES + int ticks_per_second_initialized; + long ticks_per_second; +#else + int _not_used; +#endif +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TIME_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 80144e528875..c21826fbb94f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1654,7 +1654,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_os.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ @@ -1673,6 +1672,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tuple.h \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cbf4d5b3fcec..f5175350e12a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9065,24 +9065,6 @@ build_times_result(PyObject *module, double user, double system, } -#ifdef _OS_NEED_TICKS_PER_SECOND -#define ticks_per_second _PyRuntime.os.ticks_per_second -static void -ticks_per_second_init(void) -{ - if (ticks_per_second != -1) { - return; - } -# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); -# elif defined(HZ) - ticks_per_second = HZ; -# else - ticks_per_second = 60; /* magic fallback value; may be bogus */ -# endif -} -#endif - /*[clinic input] os.times @@ -9116,22 +9098,24 @@ os_times_impl(PyObject *module) (double)0, (double)0); } -#elif !defined(_OS_NEED_TICKS_PER_SECOND) -# error "missing ticks_per_second" #else /* MS_WINDOWS */ { struct tms t; clock_t c; errno = 0; c = times(&t); - if (c == (clock_t) -1) + if (c == (clock_t) -1) { return posix_error(); + } + assert(_PyRuntime.time.ticks_per_second_initialized); +#define ticks_per_second _PyRuntime.time.ticks_per_second return build_times_result(module, (double)t.tms_utime / ticks_per_second, (double)t.tms_stime / ticks_per_second, (double)t.tms_cutime / ticks_per_second, (double)t.tms_cstime / ticks_per_second, (double)c / ticks_per_second); +#undef ticks_per_second } #endif /* MS_WINDOWS */ #endif /* HAVE_TIMES */ @@ -15950,10 +15934,6 @@ posixmodule_exec(PyObject *m) PyModule_AddObject(m, "statvfs_result", Py_NewRef(StatVFSResultType)); state->StatVFSResultType = StatVFSResultType; -#ifdef _OS_NEED_TICKS_PER_SECOND - ticks_per_second_init(); -#endif - #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) sched_param_desc.name = MODNAME ".sched_param"; PyObject *SchedParamType = (PyObject *)PyStructSequence_NewType(&sched_param_desc); diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 11c888af03e8..ba4128c0fdf5 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -62,6 +62,54 @@ #define SEC_TO_NS (1000 * 1000 * 1000) +#ifdef HAVE_TIMES + +static int +check_ticks_per_second(long tps, const char *context) +{ + /* Effectively, check that _PyTime_MulDiv(t, SEC_TO_NS, ticks_per_second) + cannot overflow. */ + if (tps >= 0 && (_PyTime_t)tps > _PyTime_MAX / SEC_TO_NS) { + PyErr_Format(PyExc_OverflowError, "%s is too large", context); + return -1; + } + return 0; +} + +# define ticks_per_second _PyRuntime.time.ticks_per_second + +static void +ensure_ticks_per_second(void) +{ + if (_PyRuntime.time.ticks_per_second_initialized) { + return; + } + _PyRuntime.time.ticks_per_second_initialized = 1; +# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) + ticks_per_second = sysconf(_SC_CLK_TCK); + if (ticks_per_second < 1) { + ticks_per_second = -1; + } +# elif defined(HZ) + ticks_per_second = HZ; +# else + ticks_per_second = 60; /* magic fallback value; may be bogus */ +# endif +} + +#endif /* HAVE_TIMES */ + + +PyStatus +_PyTime_Init(void) +{ +#ifdef HAVE_TIMES + ensure_ticks_per_second(); +#endif + return PyStatus_Ok(); +} + + /* Forward declarations */ static int pysleep(_PyTime_t timeout); @@ -140,18 +188,8 @@ Return the current time in nanoseconds since the Epoch."); static int _PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) { - static int initialized = 0; - - if (!initialized) { - initialized = 1; - - /* Make sure that _PyTime_MulDiv(ticks, SEC_TO_NS, CLOCKS_PER_SEC) - above cannot overflow */ - if ((_PyTime_t)CLOCKS_PER_SEC > _PyTime_MAX / SEC_TO_NS) { - PyErr_SetString(PyExc_OverflowError, - "CLOCKS_PER_SEC is too large"); - return -1; - } + if (check_ticks_per_second(CLOCKS_PER_SEC, "CLOCKS_PER_SEC") < 0) { + return -1; } if (info) { @@ -1308,36 +1346,10 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) struct tms t; if (times(&t) != (clock_t)-1) { - static long ticks_per_second = -1; - - if (ticks_per_second == -1) { - long freq; -#if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - freq = sysconf(_SC_CLK_TCK); - if (freq < 1) { - freq = -1; - } -#elif defined(HZ) - freq = HZ; -#else - freq = 60; /* magic fallback value; may be bogus */ -#endif - - if (freq != -1) { - /* check that _PyTime_MulDiv(t, SEC_TO_NS, ticks_per_second) - cannot overflow below */ -#if LONG_MAX > _PyTime_MAX / SEC_TO_NS - if ((_PyTime_t)freq > _PyTime_MAX / SEC_TO_NS) { - PyErr_SetString(PyExc_OverflowError, - "_SC_CLK_TCK is too large"); - return -1; - } -#endif - - ticks_per_second = freq; - } + assert(_PyRuntime.time.ticks_per_second_initialized); + if (check_ticks_per_second(ticks_per_second, "_SC_CLK_TCK") < 0) { + return -1; } - if (ticks_per_second != -1) { if (info) { info->implementation = "times()"; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 8454bd67b1db..fce1f6705100 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -110,6 +110,7 @@ <ClCompile Include="..\Modules\getpath_noop.c" /> <ClCompile Include="..\Modules\posixmodule.c" /> <ClCompile Include="..\Modules\signalmodule.c" /> + <ClCompile Include="..\Modules\timemodule.c" /> <ClCompile Include="..\Modules\_tracemalloc.c" /> <ClCompile Include="..\Modules\_io\_iomodule.c" /> <ClCompile Include="..\Modules\_io\bufferedio.c" /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 6e8498dceb1c..dce6278987c5 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -367,6 +367,9 @@ <ClCompile Include="..\Python\thread.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\timemodule.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Parser\token.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index fa3924968a6b..f9f0c191b5eb 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -236,7 +236,6 @@ <ClInclude Include="..\Include\internal\pycore_object.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h" /> - <ClInclude Include="..\Include\internal\pycore_os.h" /> <ClInclude Include="..\Include\internal\pycore_pathconfig.h" /> <ClInclude Include="..\Include\internal\pycore_pyarena.h" /> <ClInclude Include="..\Include\internal\pycore_pyerrors.h" /> @@ -255,6 +254,7 @@ <ClInclude Include="..\Include\internal\pycore_structseq.h" /> <ClInclude Include="..\Include\internal\pycore_sysmodule.h" /> <ClInclude Include="..\Include\internal\pycore_symtable.h" /> + <ClInclude Include="..\Include\internal\pycore_time.h" /> <ClInclude Include="..\Include\internal\pycore_token.h" /> <ClInclude Include="..\Include\internal\pycore_traceback.h" /> <ClInclude Include="..\Include\internal\pycore_tuple.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index e29c6b2330d1..e683650ad7da 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -612,9 +612,6 @@ <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h"> <Filter>Include\internal</Filter> </ClInclude> - <ClInclude Include="..\Include\internal\pycore_os.h"> - <Filter>Include\internal</Filter> - </ClInclude> <ClInclude Include="..\Include\internal\pycore_pathconfig.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -666,6 +663,9 @@ <ClInclude Include="..\Include\internal\pycore_symtable.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_time.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_token.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8209132ebc6c..e13660f51138 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -606,6 +606,11 @@ pycore_init_runtime(_PyRuntimeState *runtime, return status; } + status = _PyTime_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _PyImport_Init(); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 8e05bc3e4f23..3f6e1c740535 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -387,12 +387,6 @@ Modules/faulthandler.c - old_stack - ################################## ## global non-objects to fix in builtin modules -##----------------------- -## initialized once - -Modules/timemodule.c _PyTime_GetClockWithInfo initialized - -Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - - ##----------------------- ## state From webhook-mailer at python.org Thu Dec 8 18:54:14 2022 From: webhook-mailer at python.org (gvanrossum) Date: Thu, 08 Dec 2022 23:54:14 -0000 Subject: [Python-checkins] GH-98831: Generate things in the input order (#100123) Message-ID: <mailman.2870.1670543655.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1cfa704f64193701e400a77d2287f3526ff026f8 commit: 1cfa704f64193701e400a77d2287f3526ff026f8 branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-08T15:54:07-08:00 summary: GH-98831: Generate things in the input order (#100123) This makes it easier to see what changed in the generated code when converting an instruction to super or macro. files: M Python/generated_cases.c.h M Tools/cases_generator/generate_cases.py diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 510b6c4a75b8..76eb6085ec57 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -63,6 +63,115 @@ DISPATCH(); } + TARGET(LOAD_FAST__LOAD_FAST) { + PyObject *_tmp_1; + PyObject *_tmp_2; + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_2 = value; + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_1 = value; + } + STACK_GROW(2); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + + TARGET(LOAD_FAST__LOAD_CONST) { + PyObject *_tmp_1; + PyObject *_tmp_2; + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_2 = value; + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETITEM(consts, oparg); + Py_INCREF(value); + _tmp_1 = value; + } + STACK_GROW(2); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + + TARGET(STORE_FAST__LOAD_FAST) { + PyObject *_tmp_1 = PEEK(1); + { + PyObject *value = _tmp_1; + SETLOCAL(oparg, value); + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_1 = value; + } + POKE(1, _tmp_1); + DISPATCH(); + } + + TARGET(STORE_FAST__STORE_FAST) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *value = _tmp_1; + SETLOCAL(oparg, value); + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value = _tmp_2; + SETLOCAL(oparg, value); + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(LOAD_CONST__LOAD_FAST) { + PyObject *_tmp_1; + PyObject *_tmp_2; + { + PyObject *value; + value = GETITEM(consts, oparg); + Py_INCREF(value); + _tmp_2 = value; + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_1 = value; + } + STACK_GROW(2); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + TARGET(POP_TOP) { PyObject *value = PEEK(1); Py_DECREF(value); @@ -78,6 +187,21 @@ DISPATCH(); } + TARGET(END_FOR) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *value = _tmp_1; + Py_DECREF(value); + } + { + PyObject *value = _tmp_2; + Py_DECREF(value); + } + STACK_SHRINK(2); + DISPATCH(); + } + TARGET(UNARY_POSITIVE) { PyObject *value = PEEK(1); PyObject *res; @@ -2078,6 +2202,119 @@ DISPATCH(); } + TARGET(COMPARE_OP_FLOAT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); + DEOPT_IF(isnan(dleft), COMPARE_OP); + DEOPT_IF(isnan(dright), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_INT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); + Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; + Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_STR_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t invert = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int res = _PyUnicode_Equal(left, right); + assert(oparg == Py_EQ || oparg == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(res == 0 || res == 1); + assert(invert == 0 || invert == 1); + jump = res ^ invert; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + TARGET(IS_OP) { PyObject *right = POP(); PyObject *left = TOP(); @@ -3593,240 +3830,3 @@ TARGET(CACHE) { Py_UNREACHABLE(); } - - TARGET(LOAD_FAST__LOAD_FAST) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_2 = value; - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_1 = value; - } - STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); - DISPATCH(); - } - - TARGET(LOAD_FAST__LOAD_CONST) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_2 = value; - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETITEM(consts, oparg); - Py_INCREF(value); - _tmp_1 = value; - } - STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); - DISPATCH(); - } - - TARGET(STORE_FAST__LOAD_FAST) { - PyObject *_tmp_1 = PEEK(1); - { - PyObject *value = _tmp_1; - SETLOCAL(oparg, value); - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_1 = value; - } - POKE(1, _tmp_1); - DISPATCH(); - } - - TARGET(STORE_FAST__STORE_FAST) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *value = _tmp_1; - SETLOCAL(oparg, value); - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value = _tmp_2; - SETLOCAL(oparg, value); - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(LOAD_CONST__LOAD_FAST) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - PyObject *value; - value = GETITEM(consts, oparg); - Py_INCREF(value); - _tmp_2 = value; - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_1 = value; - } - STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); - DISPATCH(); - } - - TARGET(COMPARE_OP_FLOAT_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - jump = sign_ish & when_to_jump_mask; - _tmp_2 = (PyObject *)jump; - } - JUMPBY(2); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(COMPARE_OP_INT_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); - Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; - Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - jump = sign_ish & when_to_jump_mask; - _tmp_2 = (PyObject *)jump; - } - JUMPBY(2); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(COMPARE_OP_STR_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - uint16_t invert = read_u16(next_instr + 1); - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int res = _PyUnicode_Equal(left, right); - assert(oparg == Py_EQ || oparg == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(res == 0 || res == 1); - assert(invert == 0 || invert == 1); - jump = res ^ invert; - _tmp_2 = (PyObject *)jump; - } - JUMPBY(2); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(END_FOR) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *value = _tmp_1; - Py_DECREF(value); - } - { - PyObject *value = _tmp_2; - Py_DECREF(value); - } - STACK_SHRINK(2); - DISPATCH(); - } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 844272291245..5930c797b8a4 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -311,6 +311,7 @@ def error(self, msg: str, node: parser.Node) -> None: print(f"{self.filename}:{lineno}: {msg}", file=sys.stderr) self.errors += 1 + everything: list[parser.InstDef | parser.Super | parser.Macro] instrs: dict[str, Instruction] # Includes ops supers: dict[str, parser.Super] super_instrs: dict[str, SuperInstruction] @@ -344,6 +345,7 @@ def parse(self) -> None: # Parse from start psr.setpos(start) + self.everything = [] self.instrs = {} self.supers = {} self.macros = {} @@ -352,10 +354,13 @@ def parse(self) -> None: match thing: case parser.InstDef(name=name): self.instrs[name] = Instruction(thing) + self.everything.append(thing) case parser.Super(name): self.supers[name] = thing + self.everything.append(thing) case parser.Macro(name): self.macros[name] = thing + self.everything.append(thing) case parser.Family(name): self.families[name] = thing case _: @@ -560,33 +565,24 @@ def write_instructions(self) -> None: # Create formatter; the rest of the code uses this. self.out = Formatter(f, 8) - # Write and count regular instructions + # Write and count instructions of all kinds n_instrs = 0 - for name, instr in self.instrs.items(): - if instr.kind != "inst": - continue # ops are not real instructions - n_instrs += 1 - self.out.emit("") - with self.out.block(f"TARGET({name})"): - if instr.predicted: - self.out.emit(f"PREDICTED({name});") - instr.write(self.out) - if not instr.always_exits: - for prediction in instr.predictions: - self.out.emit(f"PREDICT({prediction});") - self.out.emit(f"DISPATCH();") - - # Write and count super-instructions n_supers = 0 - for sup in self.super_instrs.values(): - n_supers += 1 - self.write_super(sup) - - # Write and count macro instructions n_macros = 0 - for macro in self.macro_instrs.values(): - n_macros += 1 - self.write_macro(macro) + for thing in self.everything: + match thing: + case parser.InstDef(): + if thing.kind == "inst": + n_instrs += 1 + self.write_instr(self.instrs[thing.name]) + case parser.Super(): + n_supers += 1 + self.write_super(self.super_instrs[thing.name]) + case parser.Macro(): + n_macros += 1 + self.write_macro(self.macro_instrs[thing.name]) + case _: + typing.assert_never(thing) print( f"Wrote {n_instrs} instructions, {n_supers} supers, " @@ -594,6 +590,18 @@ def write_instructions(self) -> None: file=sys.stderr, ) + def write_instr(self, instr: Instruction) -> None: + name = instr.name + self.out.emit("") + with self.out.block(f"TARGET({name})"): + if instr.predicted: + self.out.emit(f"PREDICTED({name});") + instr.write(self.out) + if not instr.always_exits: + for prediction in instr.predictions: + self.out.emit(f"PREDICT({prediction});") + self.out.emit(f"DISPATCH();") + def write_super(self, sup: SuperInstruction) -> None: """Write code for a super-instruction.""" with self.wrap_super_or_macro(sup): From webhook-mailer at python.org Thu Dec 8 19:17:26 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 00:17:26 -0000 Subject: [Python-checkins] gh-81057: Move Ceval Trampoline Globals to _PyRuntimeState (gh-100083) Message-ID: <mailman.2871.1670545046.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bc8cdf8c3d58f5d28c9e70c72eaae52c6d13f961 commit: bc8cdf8c3d58f5d28c9e70c72eaae52c6d13f961 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T17:17:20-07:00 summary: gh-81057: Move Ceval Trampoline Globals to _PyRuntimeState (gh-100083) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_ceval_state.h M Include/internal/pycore_interp.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/perf_trampoline.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h new file mode 100644 index 000000000000..9ba42eb03b26 --- /dev/null +++ b/Include/internal/pycore_ceval_state.h @@ -0,0 +1,100 @@ +#ifndef Py_INTERNAL_CEVAL_STATE_H +#define Py_INTERNAL_CEVAL_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_gil.h" // struct _gil_runtime_state + + +typedef enum { + PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state + PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized + PERF_STATUS_OK = 1, // Perf trampoline is ready to be executed +} perf_status_t; + + +#ifdef PY_HAVE_PERF_TRAMPOLINE +struct code_arena_st; + +struct trampoline_api_st { + void* (*init_state)(void); + void (*write_state)(void* state, const void *code_addr, + unsigned int code_size, PyCodeObject* code); + int (*free_state)(void* state); + void *state; +}; +#endif + +struct _ceval_runtime_state { + struct { +#ifdef PY_HAVE_PERF_TRAMPOLINE + perf_status_t status; + Py_ssize_t extra_code_index; + struct code_arena_st *code_arena; + struct trampoline_api_st trampoline_api; + FILE *map_file; +#else + int _not_used; +#endif + } perf; + /* Request for checking signals. It is shared by all interpreters (see + bpo-40513). Any thread of any interpreter can receive a signal, but only + the main thread of the main interpreter can handle signals: see + _Py_ThreadCanHandleSignals(). */ + _Py_atomic_int signals_pending; + struct _gil_runtime_state gil; +}; + +#ifdef PY_HAVE_PERF_TRAMPOLINE +# define _PyEval_RUNTIME_PERF_INIT \ + { \ + .status = PERF_STATUS_NO_INIT, \ + .extra_code_index = -1, \ + } +#else +# define _PyEval_RUNTIME_PERF_INIT {0} +#endif + + +struct _pending_calls { + int busy; + PyThread_type_lock lock; + /* Request for running pending calls. */ + _Py_atomic_int calls_to_do; + /* Request for looking at the `async_exc` field of the current + thread state. + Guarded by the GIL. */ + int async_exc; +#define NPENDINGCALLS 32 + struct { + int (*func)(void *); + void *arg; + } calls[NPENDINGCALLS]; + int first; + int last; +}; + +struct _ceval_state { + int recursion_limit; + /* This single variable consolidates all requests to break out of + the fast path in the eval loop. */ + _Py_atomic_int eval_breaker; + /* Request for dropping the GIL */ + _Py_atomic_int gil_drop_request; + /* The GC is ready to be executed */ + _Py_atomic_int gc_scheduled; + struct _pending_calls pending; +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CEVAL_STATE_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b5c46773c90f..ffda1351952d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -12,6 +12,7 @@ extern "C" { #include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ast_state.h" // struct ast_state +#include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache #include "pycore_context.h" // struct _Py_context_state #include "pycore_dict_state.h" // struct _Py_dict_state @@ -28,37 +29,6 @@ extern "C" { #include "pycore_warnings.h" // struct _warnings_runtime_state -struct _pending_calls { - int busy; - PyThread_type_lock lock; - /* Request for running pending calls. */ - _Py_atomic_int calls_to_do; - /* Request for looking at the `async_exc` field of the current - thread state. - Guarded by the GIL. */ - int async_exc; -#define NPENDINGCALLS 32 - struct { - int (*func)(void *); - void *arg; - } calls[NPENDINGCALLS]; - int first; - int last; -}; - -struct _ceval_state { - int recursion_limit; - /* This single variable consolidates all requests to break out of - the fast path in the eval loop. */ - _Py_atomic_int eval_breaker; - /* Request for dropping the GIL */ - _Py_atomic_int gil_drop_request; - /* The GC is ready to be executed */ - _Py_atomic_int gc_scheduled; - struct _pending_calls pending; -}; - - // atexit state typedef struct { PyObject *func; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 458493eac70f..ac89c7d378e2 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,11 +9,11 @@ extern "C" { #endif #include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_ceval_state.h" // struct _ceval_runtime_state #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_dtoa.h" // struct _dtoa_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_function.h" // struct _func_runtime_state -#include "pycore_gil.h" // struct _gil_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState @@ -31,15 +31,6 @@ struct _getargs_runtime_state { /* ceval state */ -struct _ceval_runtime_state { - /* Request for checking signals. It is shared by all interpreters (see - bpo-40513). Any thread of any interpreter can receive a signal, but only - the main thread of the main interpreter can handle signals: see - _Py_ThreadCanHandleSignals(). */ - _Py_atomic_int signals_pending; - struct _gil_runtime_state gil; -}; - /* GIL state */ struct _gilstate_runtime_state { diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ab53876e355f..b569e5833f1d 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -41,6 +41,9 @@ extern "C" { .header = 1, \ }, \ }, \ + .ceval = { \ + .perf = _PyEval_RUNTIME_PERF_INIT, \ + }, \ .gilstate = { \ .check_enabled = 1, \ /* A TSS key must be initialized with Py_tss_NEEDS_INIT \ diff --git a/Makefile.pre.in b/Makefile.pre.in index c21826fbb94f..312031999df0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1623,6 +1623,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_bytesobject.h \ $(srcdir)/Include/internal/pycore_call.h \ $(srcdir)/Include/internal/pycore_ceval.h \ + $(srcdir)/Include/internal/pycore_ceval_state.h \ $(srcdir)/Include/internal/pycore_code.h \ $(srcdir)/Include/internal/pycore_compile.h \ $(srcdir)/Include/internal/pycore_condvar.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f9f0c191b5eb..58ea6ae7b4aa 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -204,6 +204,7 @@ <ClInclude Include="..\Include\internal\pycore_bytesobject.h" /> <ClInclude Include="..\Include\internal\pycore_call.h" /> <ClInclude Include="..\Include\internal\pycore_ceval.h" /> + <ClInclude Include="..\Include\internal\pycore_ceval_state.h" /> <ClInclude Include="..\Include\internal\pycore_code.h" /> <ClInclude Include="..\Include\internal\pycore_compile.h" /> <ClInclude Include="..\Include\internal\pycore_condvar.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index e683650ad7da..96c3bd6c30e9 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -519,6 +519,9 @@ <ClInclude Include="..\Include\internal\pycore_ceval.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_ceval_state.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_code.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 161e0ef74cf1..1957ab82c339 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -134,11 +134,6 @@ any DWARF information available for them). #include "pycore_frame.h" #include "pycore_interp.h" -typedef enum { - PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state - PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized - PERF_STATUS_OK = 1, // Perf trampoline is ready to be executed -} perf_status_t; #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -190,24 +185,13 @@ struct code_arena_st { }; typedef struct code_arena_st code_arena_t; - -struct trampoline_api_st { - void* (*init_state)(void); - void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); - int (*free_state)(void* state); - void *state; -}; - typedef struct trampoline_api_st trampoline_api_t; - -static perf_status_t perf_status = PERF_STATUS_NO_INIT; -static Py_ssize_t extra_code_index = -1; -static code_arena_t *code_arena; -static trampoline_api_t trampoline_api; - -static FILE *perf_map_file; +#define perf_status _PyRuntime.ceval.perf.status +#define extra_code_index _PyRuntime.ceval.perf.extra_code_index +#define perf_code_arena _PyRuntime.ceval.perf.code_arena +#define trampoline_api _PyRuntime.ceval.perf.trampoline_api +#define perf_map_file _PyRuntime.ceval.perf.map_file static void * perf_map_get_file(void) @@ -344,17 +328,17 @@ new_code_arena(void) new_arena->size = mem_size; new_arena->size_left = mem_size; new_arena->code_size = code_size; - new_arena->prev = code_arena; - code_arena = new_arena; + new_arena->prev = perf_code_arena; + perf_code_arena = new_arena; return 0; } static void free_code_arenas(void) { - code_arena_t *cur = code_arena; + code_arena_t *cur = perf_code_arena; code_arena_t *prev; - code_arena = NULL; // invalid static pointer + perf_code_arena = NULL; // invalid static pointer while (cur) { munmap(cur->start_addr, cur->size); prev = cur->prev; @@ -375,14 +359,14 @@ code_arena_new_code(code_arena_t *code_arena) static inline py_trampoline compile_trampoline(void) { - if ((code_arena == NULL) || - (code_arena->size_left <= code_arena->code_size)) { + if ((perf_code_arena == NULL) || + (perf_code_arena->size_left <= perf_code_arena->code_size)) { if (new_code_arena() < 0) { return NULL; } } - assert(code_arena->size_left <= code_arena->size); - return code_arena_new_code(code_arena); + assert(perf_code_arena->size_left <= perf_code_arena->size); + return code_arena_new_code(perf_code_arena); } static PyObject * @@ -405,7 +389,7 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, goto default_eval; } trampoline_api.write_state(trampoline_api.state, new_trampoline, - code_arena->code_size, co); + perf_code_arena->code_size, co); _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); f = new_trampoline; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 3f6e1c740535..f774cdeab386 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -302,21 +302,10 @@ Objects/sliceobject.c - _Py_EllipsisObject - ################################## ## global non-objects to fix in core code -##----------------------- -## effectively-const but initialized lazily - -## others -Python/perf_trampoline.c - perf_map_file - - ##----------------------- ## state -## other Objects/object.c - _Py_RefTotal - -Python/perf_trampoline.c - perf_status - -Python/perf_trampoline.c - extra_code_index - -Python/perf_trampoline.c - code_arena - -Python/perf_trampoline.c - trampoline_api - Python/thread_pthread_stubs.h - py_tls_entries - @@ -374,9 +363,8 @@ Modules/itertoolsmodule.c - teedataobject_type - Modules/itertoolsmodule.c - ziplongest_type - ##----------------------- -## other - ## state + Modules/faulthandler.c - fatal_error - Modules/faulthandler.c - thread - Modules/faulthandler.c - user_signals - From webhook-mailer at python.org Thu Dec 8 19:51:03 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 00:51:03 -0000 Subject: [Python-checkins] gh-81057: Move Threading-Related Globals to _PyRuntimeState (#100084) Message-ID: <mailman.2872.1670547065.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1160001b34615066b1188d5fb457131b1ebb928d commit: 1160001b34615066b1188d5fb457131b1ebb928d branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T17:50:58-07:00 summary: gh-81057: Move Threading-Related Globals to _PyRuntimeState (#100084) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_pythread.h M Include/internal/pycore_runtime.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/thread.c M Python/thread_nt.h M Python/thread_pthread.h M Python/thread_pthread_stubs.h M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h new file mode 100644 index 000000000000..4aeb285b89a8 --- /dev/null +++ b/Include/internal/pycore_pythread.h @@ -0,0 +1,81 @@ +#ifndef Py_INTERNAL_PYTHREAD_H +#define Py_INTERNAL_PYTHREAD_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#ifndef _POSIX_THREADS +/* This means pthreads are not implemented in libc headers, hence the macro + not present in unistd.h. But they still can be implemented as an external + library (e.g. gnu pth in pthread emulation) */ +# ifdef HAVE_PTHREAD_H +# include <pthread.h> /* _POSIX_THREADS */ +# endif +# ifndef _POSIX_THREADS +/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then + enough of the Posix threads package is implemented to support python + threads. + + This is valid for HP-UX 11.23 running on an ia64 system. If needed, add + a check of __ia64 to verify that we're running on an ia64 system instead + of a pa-risc system. +*/ +# ifdef __hpux +# ifdef _SC_THREADS +# define _POSIX_THREADS +# endif +# endif +# endif /* _POSIX_THREADS */ +#endif /* _POSIX_THREADS */ + +#if defined(_POSIX_THREADS) && !defined(HAVE_PTHREAD_STUBS) +# define _USE_PTHREADS +#endif + +#if defined(_USE_PTHREADS) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) +// monotonic is supported statically. It doesn't mean it works on runtime. +# define CONDATTR_MONOTONIC +#endif + + +#if defined(HAVE_PTHREAD_STUBS) +// pthread_key +struct py_stub_tls_entry { + bool in_use; + void *value; +}; +#endif + +struct _pythread_runtime_state { + int initialized; + +#ifdef _USE_PTHREADS + // This matches when thread_pthread.h is used. + struct { + /* NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported. */ + pthread_condattr_t *ptr; +# ifdef CONDATTR_MONOTONIC + /* The value to which condattr_monotonic is set. */ + pthread_condattr_t val; +# endif + } _condattr_monotonic; + +#endif // USE_PTHREADS + +#if defined(HAVE_PTHREAD_STUBS) + struct { + struct py_stub_tls_entry tls_entries[PTHREAD_KEYS_MAX]; + } stubs; +#endif +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYTHREAD_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index ac89c7d378e2..fe2de5feb47a 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -20,6 +20,7 @@ extern "C" { #include "pycore_parser.h" // struct _parser_runtime_state #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state +#include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state #include "pycore_time.h" // struct _time_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids @@ -96,6 +97,7 @@ typedef struct pyruntimestate { int unhandled_keyboard_interrupt; } signals; struct _time_runtime_state time; + struct _pythread_runtime_state threads; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Makefile.pre.in b/Makefile.pre.in index 312031999df0..de42d684f166 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1663,6 +1663,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pymem.h \ $(srcdir)/Include/internal/pycore_pymem_init.h \ $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pythread.h \ $(srcdir)/Include/internal/pycore_range.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 58ea6ae7b4aa..35fbff320f46 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -245,6 +245,7 @@ <ClInclude Include="..\Include\internal\pycore_pymem.h" /> <ClInclude Include="..\Include\internal\pycore_pymem_init.h" /> <ClInclude Include="..\Include\internal\pycore_pystate.h" /> + <ClInclude Include="..\Include\internal\pycore_pythread.h" /> <ClInclude Include="..\Include\internal\pycore_range.h" /> <ClInclude Include="..\Include\internal\pycore_runtime.h" /> <ClInclude Include="..\Include\internal\pycore_runtime_init.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 96c3bd6c30e9..19cb5cf1c807 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -639,6 +639,9 @@ <ClInclude Include="..\Include\internal\pycore_pystate.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_pythread.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_range.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Python/thread.c b/Python/thread.c index 3c1e78ed1bca..4581f1af043a 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -8,15 +8,7 @@ #include "Python.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniType() - -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include <pthread.h> /* _POSIX_THREADS */ -# endif -#endif +#include "pycore_pythread.h" #ifndef DONT_HAVE_STDIO_H #include <stdio.h> @@ -24,33 +16,17 @@ #include <stdlib.h> -#ifndef _POSIX_THREADS - -/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then - enough of the Posix threads package is implemented to support python - threads. - - This is valid for HP-UX 11.23 running on an ia64 system. If needed, add - a check of __ia64 to verify that we're running on an ia64 system instead - of a pa-risc system. -*/ -#ifdef __hpux -#ifdef _SC_THREADS -#define _POSIX_THREADS -#endif -#endif - -#endif /* _POSIX_THREADS */ - -static int initialized; static void PyThread__init_thread(void); /* Forward */ +#define initialized _PyRuntime.threads.initialized + void PyThread_init_thread(void) { - if (initialized) + if (initialized) { return; + } initialized = 1; PyThread__init_thread(); } @@ -58,7 +34,7 @@ PyThread_init_thread(void) #if defined(HAVE_PTHREAD_STUBS) # define PYTHREAD_NAME "pthread-stubs" # include "thread_pthread_stubs.h" -#elif defined(_POSIX_THREADS) +#elif defined(_USE_PTHREADS) /* AKA _PTHREADS */ # if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__) # define PYTHREAD_NAME "pthread-stubs" # else diff --git a/Python/thread_nt.h b/Python/thread_nt.h index d1f1323948a6..26f441bd6d3c 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -152,11 +152,12 @@ unsigned long PyThread_get_thread_native_id(void); #endif /* - * Initialization of the C package, should not be needed. + * Initialization for the current runtime. */ static void PyThread__init_thread(void) { + // Initialization of the C package should not be needed. } /* diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 1c5b320813af..ae312e987bd6 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -119,20 +119,16 @@ * pthread_cond support */ -#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) -// monotonic is supported statically. It doesn't mean it works on runtime. -#define CONDATTR_MONOTONIC -#endif - -// NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported. -static pthread_condattr_t *condattr_monotonic = NULL; +#define condattr_monotonic _PyRuntime.threads._condattr_monotonic.ptr static void init_condattr(void) { #ifdef CONDATTR_MONOTONIC - static pthread_condattr_t ca; +# define ca _PyRuntime.threads._condattr_monotonic.val + // XXX We need to check the return code? pthread_condattr_init(&ca); + // XXX We need to run pthread_condattr_destroy() during runtime fini. if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) { condattr_monotonic = &ca; // Use monotonic clock } @@ -192,15 +188,21 @@ typedef struct { "%s: %s\n", name, strerror(status)); error = 1; } /* - * Initialization. + * Initialization for the current runtime. */ static void PyThread__init_thread(void) { + // The library is only initialized once in the process, + // regardless of how many times the Python runtime is initialized. + static int lib_initialized = 0; + if (!lib_initialized) { + lib_initialized = 1; #if defined(_AIX) && defined(__GNUC__) - extern void pthread_init(void); - pthread_init(); + extern void pthread_init(void); + pthread_init(); #endif + } init_condattr(); } diff --git a/Python/thread_pthread_stubs.h b/Python/thread_pthread_stubs.h index 8b80c0f87e25..56e5b6141924 100644 --- a/Python/thread_pthread_stubs.h +++ b/Python/thread_pthread_stubs.h @@ -124,13 +124,10 @@ pthread_attr_destroy(pthread_attr_t *attr) return 0; } -// pthread_key -typedef struct { - bool in_use; - void *value; -} py_tls_entry; -static py_tls_entry py_tls_entries[PTHREAD_KEYS_MAX] = {0}; +typedef struct py_stub_tls_entry py_tls_entry; + +#define py_tls_entries (_PyRuntime.threads.stubs.tls_entries) int pthread_key_create(pthread_key_t *key, void (*destr_function)(void *)) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index f774cdeab386..94e9831db1fd 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -306,7 +306,6 @@ Objects/sliceobject.c - _Py_EllipsisObject - ## state Objects/object.c - _Py_RefTotal - -Python/thread_pthread_stubs.h - py_tls_entries - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 814c55b75c09..7d6ff0ba6d64 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -22,11 +22,8 @@ Python/fileutils.c set_inheritable ioctl_works - # XXX Is this thread-safe? Modules/posixmodule.c os_dup2_impl dup3_works - -## resource init - set during first init -Python/thread.c - initialized - -Python/thread_pthread.h - condattr_monotonic - -# safe static buffer used during one-time initialization -Python/thread_pthread.h init_condattr ca - +## guards around resource init +Python/thread_pthread.h PyThread__init_thread lib_initialized - ##----------------------- ## other values (not Python-specific) From webhook-mailer at python.org Thu Dec 8 20:16:44 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 01:16:44 -0000 Subject: [Python-checkins] gh-81057: Fix an ifdef in the time module (#100125) Message-ID: <mailman.2873.1670548605.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3e06b5030b18ca9d9d507423b582d13f38d393f2 commit: 3e06b5030b18ca9d9d507423b582d13f38d393f2 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T18:16:37-07:00 summary: gh-81057: Fix an ifdef in the time module (#100125) An earlier commit only defined check_ticks_per_second() when HAVE_TIMES is defined. However, we also need it when HAVE_CLOCK is defined. This primarily affects Windows. https://github.com/python/cpython/issues/81057 files: M Modules/timemodule.c diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ba4128c0fdf5..c2bacaae0c03 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -62,8 +62,7 @@ #define SEC_TO_NS (1000 * 1000 * 1000) -#ifdef HAVE_TIMES - +#if defined(HAVE_TIMES) || defined(HAVE_CLOCK) static int check_ticks_per_second(long tps, const char *context) { @@ -75,6 +74,9 @@ check_ticks_per_second(long tps, const char *context) } return 0; } +#endif /* HAVE_TIMES || HAVE_CLOCK */ + +#ifdef HAVE_TIMES # define ticks_per_second _PyRuntime.time.ticks_per_second From webhook-mailer at python.org Thu Dec 8 22:31:25 2022 From: webhook-mailer at python.org (ethanfurman) Date: Fri, 09 Dec 2022 03:31:25 -0000 Subject: [Python-checkins] gh-99087: Add missing newline for prompts in docs (GH-98993) Message-ID: <mailman.2874.1670556687.3313.python-checkins@python.org> https://github.com/python/cpython/commit/286e3c76a9cb8f1adc2a915f0d246a1e2e408733 commit: 286e3c76a9cb8f1adc2a915f0d246a1e2e408733 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-08T19:31:19-08:00 summary: gh-99087: Add missing newline for prompts in docs (GH-98993) Add newline for prompts so copying to REPL does not cause errors. files: M Doc/howto/enum.rst M Doc/library/argparse.rst M Doc/library/bz2.rst M Doc/library/collections.rst M Doc/library/datetime.rst M Doc/library/decimal.rst M Doc/library/doctest.rst M Doc/library/email.policy.rst M Doc/library/enum.rst M Doc/library/functions.rst M Doc/library/hashlib.rst M Doc/library/inspect.rst M Doc/library/re.rst M Doc/library/sqlite3.rst M Doc/library/statistics.rst M Doc/library/stdtypes.rst M Doc/library/unittest.mock.rst M Doc/library/xml.etree.elementtree.rst M Doc/library/zipfile.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.2.rst M Doc/whatsnew/3.3.rst diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 3155c6cb977b..4525acb04503 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -158,6 +158,7 @@ And a function to display the chores for a given day:: ... for chore, days in chores.items(): ... if day in days: ... print(chore) + ... >>> show_chores(chores_for_ethan, Weekday.SATURDAY) answer SO questions @@ -712,6 +713,7 @@ It is also possible to name the combinations:: ... W = 2 ... X = 1 ... RWX = 7 + ... >>> Perm.RWX <Perm.RWX: 7> >>> ~Perm.RWX diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index f8839d0986d0..e6c964864925 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -565,6 +565,7 @@ arguments they contain. For example:: >>> with open('args.txt', 'w', encoding=sys.getfilesystemencoding()) as fp: ... fp.write('-f\nbar') + ... >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') >>> parser.add_argument('-f') >>> parser.parse_args(['-f', 'foo', '@args.txt']) diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ae5a1598f84b..32df99869eb5 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -320,9 +320,11 @@ Writing and reading a bzip2-compressed file in binary mode: >>> with bz2.open("myfile.bz2", "wb") as f: ... # Write compressed data to file ... unused = f.write(data) + ... >>> with bz2.open("myfile.bz2", "rb") as f: ... # Decompress data from file ... content = f.read() + ... >>> content == data # Check equality to original object after round-trip True diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 53b4b69f84b7..2cffc2300a22 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -229,6 +229,7 @@ For example:: >>> cnt = Counter() >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: ... cnt[word] += 1 + ... >>> cnt Counter({'blue': 3, 'red': 2, 'green': 1}) @@ -818,6 +819,7 @@ zero): >>> def constant_factory(value): ... return lambda: value + ... >>> d = defaultdict(constant_factory('<missing>')) >>> d.update(name='John', action='ran') >>> '%(name)s %(action)s to %(object)s' % d diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index f7e2bb3f3c6d..0b39512a2d23 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -765,6 +765,7 @@ Example of counting days to an event:: >>> my_birthday = date(today.year, 6, 24) >>> if my_birthday < today: ... my_birthday = my_birthday.replace(year=today.year + 1) + ... >>> my_birthday datetime.date(2008, 6, 24) >>> time_to_birthday = abs(my_birthday - today) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 260108136df7..fec9b86864c5 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -2057,6 +2057,7 @@ to handle the :meth:`quantize` step: >>> def mul(x, y, fp=TWOPLACES): ... return (x * y).quantize(fp) + ... >>> def div(x, y, fp=TWOPLACES): ... return (x / y).quantize(fp) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index c106d5a3383a..b6dd7f7e1232 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -351,6 +351,7 @@ The fine print: >>> def f(x): ... r'''Backslashes in a raw docstring: m\n''' + ... >>> print(f.__doc__) Backslashes in a raw docstring: m\n @@ -360,6 +361,7 @@ The fine print: >>> def f(x): ... '''Backslashes in a raw docstring: m\\n''' + ... >>> print(f.__doc__) Backslashes in a raw docstring: m\n diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index bf53b9520fc7..2439dee676c9 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -97,6 +97,7 @@ file on disk and pass it to the system ``sendmail`` program on a Unix system: >>> from subprocess import Popen, PIPE >>> with open('mymsg.txt', 'rb') as f: ... msg = message_from_binary_file(f, policy=policy.default) + ... >>> p = Popen(['sendmail', msg['To'].addresses[0]], stdin=PIPE) >>> g = BytesGenerator(p.stdin, policy=msg.policy.clone(linesep='\r\n')) >>> g.flatten(msg) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index f8875440baf5..295152c69082 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -292,6 +292,7 @@ Data Types ... @classmethod ... def today(cls): ... print('today is %s' % cls(date.today().isoweekday()).name) + ... >>> dir(Weekday.SATURDAY) ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] @@ -312,6 +313,7 @@ Data Types ... return (count + 1) * 3 ... FIRST = auto() ... SECOND = auto() + ... >>> PowersOfThree.SECOND.value 6 @@ -336,6 +338,7 @@ Data Types ... if member.value == value: ... return member ... return None + ... >>> Build.DEBUG.value 'debug' >>> Build('deBUG') @@ -353,6 +356,7 @@ Data Types ... def __repr__(self): ... cls_name = self.__class__.__name__ ... return f'{cls_name}.{self.name}' + ... >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" (OtherStyle.ALTERNATE, 'OtherStyle.ALTERNATE', 'OtherStyle.ALTERNATE') @@ -367,6 +371,7 @@ Data Types ... SOMETHING_ELSE = auto() ... def __str__(self): ... return f'{self.name}' + ... >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" (<OtherStyle.ALTERNATE: 1>, 'ALTERNATE', 'ALTERNATE') @@ -381,6 +386,7 @@ Data Types ... SOMETHING_ELSE = auto() ... def __format__(self, spec): ... return f'{self.name}' + ... >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" (<OtherStyle.ALTERNATE: 1>, 'OtherStyle.ALTERNATE', 'ALTERNATE') @@ -403,6 +409,7 @@ Data Types ... ONE = 1 ... TWO = 2 ... THREE = 3 + ... >>> Numbers.THREE <Numbers.THREE: 3> >>> Numbers.ONE + Numbers.TWO @@ -463,6 +470,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> purple = Color.RED | Color.BLUE >>> white = Color.RED | Color.GREEN | Color.BLUE >>> Color.GREEN in purple @@ -570,6 +578,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> Color.RED & 2 <Color: 0> >>> Color.RED | 2 @@ -695,6 +704,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> StrictFlag(2**2 + 2**4) Traceback (most recent call last): ... @@ -712,6 +722,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> ConformFlag(2**2 + 2**4) <ConformFlag.BLUE: 4> @@ -725,6 +736,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> EjectFlag(2**2 + 2**4) 20 @@ -738,6 +750,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> KeepFlag(2**2 + 2**4) <KeepFlag.BLUE|16: 20> diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 110e7e5d7fb9..2110990d1889 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -462,6 +462,7 @@ are always available. They are listed here in alphabetical order. >>> class Shape: ... def __dir__(self): ... return ['area', 'perimeter', 'location'] + ... >>> s = Shape() >>> dir(s) ['area', 'location', 'perimeter'] diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 8e47312fe77b..f8d10c0c295c 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -497,6 +497,7 @@ update the hash: >>> h = blake2b() >>> for item in items: ... h.update(item) + ... >>> h.hexdigest() '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 9cb7a6f94e49..6705577551dc 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -715,6 +715,7 @@ function. >>> def test(a, b): ... pass + ... >>> sig = signature(test) >>> new_sig = sig.replace(return_annotation="new return anno") >>> str(new_sig) @@ -1054,6 +1055,7 @@ Classes and functions >>> from inspect import getcallargs >>> def f(a, b=1, *pos, **named): ... pass + ... >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} True >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} diff --git a/Doc/library/re.rst b/Doc/library/re.rst index e6e242320fd8..f7d46586cf75 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -973,6 +973,7 @@ Functions >>> def dashrepl(matchobj): ... if matchobj.group(0) == '-': return ' ' ... else: return '-' + ... >>> re.sub('-{1,2}', dashrepl, 'pro----gram-files') 'pro--gram files' >>> re.sub(r'\sAND\s', ' & ', 'Baked Beans And Spam', flags=re.IGNORECASE) @@ -1672,6 +1673,7 @@ in each word of a sentence except for the first and last characters:: ... inner_word = list(m.group(2)) ... random.shuffle(inner_word) ... return m.group(1) + "".join(inner_word) + m.group(3) + ... >>> text = "Professor Abdolmalek, please report your absences promptly." >>> re.sub(r"(\w)(\w+)(\w)", repl, text) 'Poefsrosr Aealmlobdk, pslaee reorpt your abnseces plmrptoy.' diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 2b6387cbfb52..3622864a4b06 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -397,6 +397,7 @@ Module functions >>> con = sqlite3.connect(":memory:") >>> def evil_trace(stmt): ... 5/0 + ... >>> con.set_trace_callback(evil_trace) >>> def debug(unraisable): ... print(f"{unraisable.exc_value!r} in callback {unraisable.object.__name__}") diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 88a887960edb..f934b0e0319d 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -996,6 +996,7 @@ probability that the Python room will stay within its capacity limits? >>> seed(8675309) >>> def trial(): ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') + ... >>> mean(trial() <= k for i in range(10_000)) 0.8398 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 785b76a11f2f..73debe5ceeaf 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4459,6 +4459,7 @@ can be used interchangeably to index the same dictionary entry. >>> class Counter(dict): ... def __missing__(self, key): ... return 0 + ... >>> c = Counter() >>> c['red'] 0 @@ -4716,6 +4717,7 @@ An example of dictionary view usage:: >>> n = 0 >>> for val in values: ... n += val + ... >>> print(n) 504 diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index b768557e6075..e009f303fef3 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1604,6 +1604,7 @@ decorator: >>> @patch.dict(foo, {'newkey': 'newvalue'}) ... def test(): ... assert foo == {'newkey': 'newvalue'} + ... >>> test() >>> assert foo == {} diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 2fe0d2e082fb..876de29b17ca 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1212,6 +1212,7 @@ Example of changing the attribute "target" of every link in first paragraph:: [<Element 'a' at 0xb77ec2ac>, <Element 'a' at 0xb77ec1cc>] >>> for i in links: # Iterates through all found links ... i.attrib["target"] = "blank" + ... >>> tree.write("output.xhtml") .. _elementtree-qname-objects: diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4dd9fa961a8d..82709dbc9249 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -672,6 +672,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the >>> def notests(s): ... fn = os.path.basename(s) ... return (not (fn == 'test' or fn.startswith('test_'))) + ... >>> zf.writepy('myprog', filterfunc=notests) The :meth:`writepy` method makes archives with file names like diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 276ab63b97f8..810a2cd2537c 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1331,6 +1331,7 @@ changes, or look through the Subversion logs for all the details. >>> from inspect import getcallargs >>> def f(a, b=1, *pos, **named): ... pass + ... >>> getcallargs(f, 1, 2, 3) {'a': 1, 'b': 2, 'pos': (3,), 'named': {}} >>> getcallargs(f, a=2, x=4) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 6037db9f954d..1b1455b72b92 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -468,6 +468,7 @@ Some smaller changes made to the core Python language are: >>> class LowerCasedDict(dict): ... def __getitem__(self, key): ... return dict.__getitem__(self, key.lower()) + ... >>> lcd = LowerCasedDict(part='widgets', quantity=10) >>> 'There are {QUANTITY} {Part} in stock'.format_map(lcd) 'There are 10 widgets in stock' @@ -475,6 +476,7 @@ Some smaller changes made to the core Python language are: >>> class PlaceholderDict(dict): ... def __missing__(self, key): ... return '<{}>'.format(key) + ... >>> 'Hello {name}, welcome to {location}'.format_map(PlaceholderDict()) 'Hello <name>, welcome to <location>' @@ -1886,6 +1888,7 @@ inspect >>> from inspect import getgeneratorstate >>> def gen(): ... yield 'demo' + ... >>> g = gen() >>> getgeneratorstate(g) 'GEN_CREATED' diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 96a632577b2c..9e8d42469b01 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -560,6 +560,7 @@ Example with (non-bound) methods:: >>> class C: ... def meth(self): ... pass + ... >>> C.meth.__name__ 'meth' >>> C.meth.__qualname__ From webhook-mailer at python.org Thu Dec 8 22:52:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 09 Dec 2022 03:52:24 -0000 Subject: [Python-checkins] GH-100113: remove remaining `yield from` usage from `asyncio` tests (#100114) Message-ID: <mailman.2875.1670557945.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0448deac70be94792616c0fb0c9cb524de9a09b8 commit: 0448deac70be94792616c0fb0c9cb524de9a09b8 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-09T09:22:18+05:30 summary: GH-100113: remove remaining `yield from` usage from `asyncio` tests (#100114) files: M Lib/test/test_asyncio/test_tasks.py diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index bb1ffdf2df74..5168b8250ef0 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2092,8 +2092,8 @@ def test_cancel_gather_1(self): async def create(): # The indirection fut->child_coro is needed since otherwise the # gathering task is done at the same time as the child future - def child_coro(): - return (yield from fut) + async def child_coro(): + return await fut gather_future = asyncio.gather(child_coro()) return asyncio.ensure_future(gather_future) gather_task = loop.run_until_complete(create()) From webhook-mailer at python.org Fri Dec 9 05:27:07 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 09 Dec 2022 10:27:07 -0000 Subject: [Python-checkins] GH-100110: Specialize FOR_ITER for tuples (GH-100109) Message-ID: <mailman.2876.1670581628.3313.python-checkins@python.org> https://github.com/python/cpython/commit/748c6c0921ee02a19e01a35f03ce5f4d9cfde5a6 commit: 748c6c0921ee02a19e01a35f03ce5f4d9cfde5a6 branch: main author: Ken Jin <kenjin at python.org> committer: markshannon <mark at hotpy.org> date: 2022-12-09T10:27:01Z summary: GH-100110: Specialize FOR_ITER for tuples (GH-100109) * Specialize FOR_ITER for tuples files: A Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst M Include/internal/pycore_opcode.h M Include/internal/pycore_tuple.h M Include/opcode.h M Lib/opcode.py M Objects/tupleobject.c M Python/bytecodes.c M Python/generated_cases.c.h M Python/opcode_targets.h M Python/specialize.c diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 0d31ca166a7d..da8a272f2fa2 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -125,6 +125,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [FOR_ITER_GEN] = FOR_ITER, [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, + [FOR_ITER_TUPLE] = FOR_ITER, [GET_AITER] = GET_AITER, [GET_ANEXT] = GET_ANEXT, [GET_AWAITABLE] = GET_AWAITABLE, @@ -293,31 +294,31 @@ static const char *const _PyOpcode_OpName[263] = { [FOR_ITER_LIST] = "FOR_ITER_LIST", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [STOPITERATION_ERROR] = "STOPITERATION_ERROR", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -344,7 +345,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -352,7 +353,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -372,9 +373,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -384,24 +385,24 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [170] = "<170>", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [173] = "<173>", @@ -498,7 +499,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 170: \ case 173: \ case 174: \ case 175: \ diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 504c36338d9e..edc70843b575 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -67,6 +67,13 @@ struct _Py_tuple_state { extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); + +typedef struct { + PyObject_HEAD + Py_ssize_t it_index; + PyTupleObject *it_seq; /* Set to NULL when iterator is exhausted */ +} _PyTupleIterObject; + #ifdef __cplusplus } #endif diff --git a/Include/opcode.h b/Include/opcode.h index f284313d2ed7..888250ed37e8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -162,34 +162,35 @@ extern "C" { #define COMPARE_OP_INT_JUMP 57 #define COMPARE_OP_STR_JUMP 58 #define FOR_ITER_LIST 59 -#define FOR_ITER_RANGE 62 -#define FOR_ITER_GEN 64 -#define LOAD_ATTR_CLASS 65 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 72 -#define LOAD_ATTR_PROPERTY 73 -#define LOAD_ATTR_SLOT 76 -#define LOAD_ATTR_WITH_HINT 77 -#define LOAD_ATTR_METHOD_LAZY_DICT 78 -#define LOAD_ATTR_METHOD_NO_DICT 79 -#define LOAD_ATTR_METHOD_WITH_DICT 80 -#define LOAD_ATTR_METHOD_WITH_VALUES 81 -#define LOAD_CONST__LOAD_FAST 86 -#define LOAD_FAST__LOAD_CONST 113 -#define LOAD_FAST__LOAD_FAST 121 -#define LOAD_GLOBAL_BUILTIN 141 -#define LOAD_GLOBAL_MODULE 143 -#define STORE_ATTR_INSTANCE_VALUE 153 -#define STORE_ATTR_SLOT 154 -#define STORE_ATTR_WITH_HINT 158 -#define STORE_FAST__LOAD_FAST 159 -#define STORE_FAST__STORE_FAST 160 -#define STORE_SUBSCR_DICT 161 -#define STORE_SUBSCR_LIST_INT 166 -#define UNPACK_SEQUENCE_LIST 167 -#define UNPACK_SEQUENCE_TUPLE 168 -#define UNPACK_SEQUENCE_TWO_TUPLE 169 +#define FOR_ITER_TUPLE 62 +#define FOR_ITER_RANGE 64 +#define FOR_ITER_GEN 65 +#define LOAD_ATTR_CLASS 66 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 +#define LOAD_ATTR_INSTANCE_VALUE 72 +#define LOAD_ATTR_MODULE 73 +#define LOAD_ATTR_PROPERTY 76 +#define LOAD_ATTR_SLOT 77 +#define LOAD_ATTR_WITH_HINT 78 +#define LOAD_ATTR_METHOD_LAZY_DICT 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_WITH_DICT 81 +#define LOAD_ATTR_METHOD_WITH_VALUES 86 +#define LOAD_CONST__LOAD_FAST 113 +#define LOAD_FAST__LOAD_CONST 121 +#define LOAD_FAST__LOAD_FAST 141 +#define LOAD_GLOBAL_BUILTIN 143 +#define LOAD_GLOBAL_MODULE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 160 +#define STORE_FAST__STORE_FAST 161 +#define STORE_SUBSCR_DICT 166 +#define STORE_SUBSCR_LIST_INT 167 +#define UNPACK_SEQUENCE_LIST 168 +#define UNPACK_SEQUENCE_TUPLE 169 +#define UNPACK_SEQUENCE_TWO_TUPLE 170 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index fa6dbe5d2417..fc57affbac58 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -320,6 +320,7 @@ def pseudo_op(name, op, real_ops): ], "FOR_ITER": [ "FOR_ITER_LIST", + "FOR_ITER_TUPLE", "FOR_ITER_RANGE", "FOR_ITER_GEN", ], diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst new file mode 100644 index 000000000000..e494f44fd42c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst @@ -0,0 +1 @@ +Specialize ``FOR_ITER`` for tuples. diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 4405125d45e7..e1b9953226c0 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -995,14 +995,9 @@ _PyTuple_ClearFreeList(PyInterpreterState *interp) /*********************** Tuple Iterator **************************/ -typedef struct { - PyObject_HEAD - Py_ssize_t it_index; - PyTupleObject *it_seq; /* Set to NULL when iterator is exhausted */ -} tupleiterobject; static void -tupleiter_dealloc(tupleiterobject *it) +tupleiter_dealloc(_PyTupleIterObject *it) { _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); @@ -1010,14 +1005,14 @@ tupleiter_dealloc(tupleiterobject *it) } static int -tupleiter_traverse(tupleiterobject *it, visitproc visit, void *arg) +tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) { Py_VISIT(it->it_seq); return 0; } static PyObject * -tupleiter_next(tupleiterobject *it) +tupleiter_next(_PyTupleIterObject *it) { PyTupleObject *seq; PyObject *item; @@ -1040,7 +1035,7 @@ tupleiter_next(tupleiterobject *it) } static PyObject * -tupleiter_len(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_len(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) { Py_ssize_t len = 0; if (it->it_seq) @@ -1051,7 +1046,7 @@ tupleiter_len(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -tupleiter_reduce(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) { if (it->it_seq) return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), @@ -1061,7 +1056,7 @@ tupleiter_reduce(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -tupleiter_setstate(tupleiterobject *it, PyObject *state) +tupleiter_setstate(_PyTupleIterObject *it, PyObject *state) { Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) @@ -1089,7 +1084,7 @@ static PyMethodDef tupleiter_methods[] = { PyTypeObject PyTupleIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "tuple_iterator", /* tp_name */ - sizeof(tupleiterobject), /* tp_basicsize */ + sizeof(_PyTupleIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)tupleiter_dealloc, /* tp_dealloc */ @@ -1122,13 +1117,13 @@ PyTypeObject PyTupleIter_Type = { static PyObject * tuple_iter(PyObject *seq) { - tupleiterobject *it; + _PyTupleIterObject *it; if (!PyTuple_Check(seq)) { PyErr_BadInternalCall(); return NULL; } - it = PyObject_GC_New(tupleiterobject, &PyTupleIter_Type); + it = PyObject_GC_New(_PyTupleIterObject, &PyTupleIter_Type); if (it == NULL) return NULL; it->it_index = 0; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7a1bfdcc9e26..5807bd5dc2d2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2523,6 +2523,29 @@ dummy_func( end_for_iter_list: } + // stack effect: ( -- __0) + inst(FOR_ITER_TUPLE) { + assert(cframe.use_tracing == 0); + _PyTupleIterObject *it = (_PyTupleIterObject *)TOP(); + DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq) { + if (it->it_index < PyTuple_GET_SIZE(seq)) { + PyObject *next = PyTuple_GET_ITEM(seq, it->it_index++); + PUSH(Py_NewRef(next)); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); + goto end_for_iter_tuple; // End of this instruction + } + it->it_seq = NULL; + Py_DECREF(seq); + } + STACK_SHRINK(1); + Py_DECREF(it); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + end_for_iter_tuple: + } + // stack effect: ( -- __0) inst(FOR_ITER_RANGE) { assert(cframe.use_tracing == 0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 76eb6085ec57..59e70b722269 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2759,6 +2759,29 @@ DISPATCH(); } + TARGET(FOR_ITER_TUPLE) { + assert(cframe.use_tracing == 0); + _PyTupleIterObject *it = (_PyTupleIterObject *)TOP(); + DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq) { + if (it->it_index < PyTuple_GET_SIZE(seq)) { + PyObject *next = PyTuple_GET_ITEM(seq, it->it_index++); + PUSH(Py_NewRef(next)); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); + goto end_for_iter_tuple; // End of this instruction + } + it->it_seq = NULL; + Py_DECREF(seq); + } + STACK_SHRINK(1); + Py_DECREF(it); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + end_for_iter_tuple: + DISPATCH(); + } + TARGET(FOR_ITER_RANGE) { assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)TOP(); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 3aba4e7556a6..be3ad01c151c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -61,31 +61,31 @@ static void *opcode_targets[256] = { &&TARGET_FOR_ITER_LIST, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_FOR_ITER_RANGE, + &&TARGET_FOR_ITER_TUPLE, &&TARGET_STOPITERATION_ERROR, + &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index cd09b188b7fa..7545a7712493 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2132,6 +2132,10 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) _Py_SET_OPCODE(*instr, FOR_ITER_LIST); goto success; } + else if (tp == &PyTupleIter_Type) { + _Py_SET_OPCODE(*instr, FOR_ITER_TUPLE); + goto success; + } else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { _Py_SET_OPCODE(*instr, FOR_ITER_RANGE); goto success; From webhook-mailer at python.org Fri Dec 9 06:16:21 2022 From: webhook-mailer at python.org (zooba) Date: Fri, 09 Dec 2022 11:16:21 -0000 Subject: [Python-checkins] gh-88267: Avoid DLL exporting functions from static builds on Windows(GH-99888) Message-ID: <mailman.2877.1670584582.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3c5355496b54fa0a4ea0e22344d008528e45682c commit: 3c5355496b54fa0a4ea0e22344d008528e45682c branch: main author: Christian Rendina <christian.rendina at gmail.com> committer: zooba <steve.dower at microsoft.com> date: 2022-12-09T11:16:15Z summary: gh-88267: Avoid DLL exporting functions from static builds on Windows(GH-99888) files: A Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst M Include/exports.h diff --git a/Include/exports.h b/Include/exports.h index fc1a5c5ead62..59373c39ff75 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -2,9 +2,15 @@ #define Py_EXPORTS_H #if defined(_WIN32) || defined(__CYGWIN__) - #define Py_IMPORTED_SYMBOL __declspec(dllimport) - #define Py_EXPORTED_SYMBOL __declspec(dllexport) - #define Py_LOCAL_SYMBOL + #if defined(Py_ENABLE_SHARED) + #define Py_IMPORTED_SYMBOL __declspec(dllimport) + #define Py_EXPORTED_SYMBOL __declspec(dllexport) + #define Py_LOCAL_SYMBOL + #else + #define Py_IMPORTED_SYMBOL + #define Py_EXPORTED_SYMBOL + #define Py_LOCAL_SYMBOL + #endif #else /* * If we only ever used gcc >= 5, we could use __has_attribute(visibility) diff --git a/Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst b/Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst new file mode 100644 index 000000000000..29c3f5a1d763 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst @@ -0,0 +1 @@ +Avoid exporting Python symbols in linked Windows applications when the core is built as static. From webhook-mailer at python.org Fri Dec 9 07:19:17 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 09 Dec 2022 12:19:17 -0000 Subject: [Python-checkins] GH-98522: Add version number to code objects. (GH-98525) Message-ID: <mailman.2878.1670588358.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fb713b21833a17cba8022af0fa4c486512157d4b commit: fb713b21833a17cba8022af0fa4c486512157d4b branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-09T12:18:45Z summary: GH-98522: Add version number to code objects. (GH-98525) * Add version number to code object for better versioning of functions. * Improves specialization for closures and list comprehensions. files: A Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst M Include/cpython/code.h M Include/internal/pycore_code.h M Objects/codeobject.c M Objects/funcobject.c M Programs/_bootstrap_python.c M Programs/_freeze_module.c M Python/bytecodes.c M Python/generated_cases.c.h M Tools/build/deepfreeze.py diff --git a/Include/cpython/code.h b/Include/cpython/code.h index f11d099e0379..fc7c5ed70243 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -87,6 +87,7 @@ typedef struct { int co_nplaincellvars; /* number of non-arg cell variables */ \ int co_ncellvars; /* total number of cell variables */ \ int co_nfreevars; /* number of free variables */ \ + uint32_t co_version; /* version number */ \ \ PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \ PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 357fc85a95cf..f22fd45f8319 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -474,6 +474,8 @@ typedef struct _PyShimCodeDef { extern PyCodeObject * _Py_MakeShimCode(const _PyShimCodeDef *code); +extern uint32_t _Py_next_func_version; + #ifdef __cplusplus } diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst new file mode 100644 index 000000000000..d923af198f82 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst @@ -0,0 +1,3 @@ +Add an internal version number to code objects, to give better versioning of +inner functions and comprehensions, and thus better specialization of those +functions. This change is invisible to both Python and C extensions. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0c197d767b0a..c92c7deaf808 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -11,7 +11,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "clinic/codeobject.c.h" - static void notify_code_watchers(PyCodeEvent event, PyCodeObject *co) { @@ -398,7 +397,10 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - + co->co_version = _Py_next_func_version; + if (_Py_next_func_version != 0) { + _Py_next_func_version++; + } /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index bf97edc53ad7..9df06520586a 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -3,7 +3,7 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() -#include "pycore_function.h" // FUNC_MAX_WATCHERS +#include "pycore_code.h" // _Py_next_func_version #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "structmember.h" // PyMemberDef @@ -64,7 +64,6 @@ PyFunction_ClearWatcher(int watcher_id) interp->active_func_watchers &= ~(1 << watcher_id); return 0; } - PyFunctionObject * _PyFunction_FromConstructor(PyFrameConstructor *constr) { diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index bbac0c4e1a8a..6e1593a0b599 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -14,6 +14,8 @@ #include "Python/frozen_modules/importlib._bootstrap_external.h" /* End includes */ +uint32_t _Py_next_func_version = 1; + /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 9e2169f32e92..90fc2dc6e87d 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -9,6 +9,7 @@ Keep this file in sync with Programs/_freeze_module.py. */ + #include <Python.h> #include <marshal.h> #include "pycore_fileutils.h" // _Py_stat_struct @@ -22,6 +23,8 @@ #include <unistd.h> #endif +uint32_t _Py_next_func_version = 1; + /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5807bd5dc2d2..c56f1d3ef9f4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3452,6 +3452,7 @@ dummy_func( func->func_defaults = POP(); } + func->func_version = ((PyCodeObject *)codeobj)->co_version; PUSH((PyObject *)func); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 59e70b722269..45382a466b1c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3693,6 +3693,7 @@ func->func_defaults = POP(); } + func->func_version = ((PyCodeObject *)codeobj)->co_version; PUSH((PyObject *)func); DISPATCH(); } diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 2eef649437a6..7f4e24280133 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -44,6 +44,7 @@ def make_string_literal(b: bytes) -> str: CO_FAST_CELL = 0x40 CO_FAST_FREE = 0x80 +next_code_version = 1 def get_localsplus(code: types.CodeType): a = collections.defaultdict(int) @@ -227,6 +228,7 @@ def generate_unicode(self, name: str, s: str) -> str: def generate_code(self, name: str, code: types.CodeType) -> str: + global next_code_version # The ordering here matches PyCode_NewWithPosOnlyArgs() # (but see below). co_consts = self.generate(name + "_consts", code.co_consts) @@ -268,6 +270,8 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_nplaincellvars = {nplaincellvars},") self.write(f".co_ncellvars = {ncellvars},") self.write(f".co_nfreevars = {nfreevars},") + self.write(f".co_version = {next_code_version},") + next_code_version += 1 self.write(f".co_localsplusnames = {co_localsplusnames},") self.write(f".co_localspluskinds = {co_localspluskinds},") self.write(f".co_filename = {co_filename},") @@ -461,6 +465,7 @@ def generate(args: list[str], output: TextIO) -> None: with printer.block(f"if ({p} < 0)"): printer.write("return -1;") printer.write("return 0;") + printer.write(f"\nuint32_t _Py_next_func_version = {next_code_version};\n") if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") From webhook-mailer at python.org Fri Dec 9 07:47:33 2022 From: webhook-mailer at python.org (zooba) Date: Fri, 09 Dec 2022 12:47:33 -0000 Subject: [Python-checkins] bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) Message-ID: <mailman.2879.1670590054.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1 commit: a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1 branch: main author: Shreyan Avigyan <shreyan.avigyan at gmail.com> committer: zooba <steve.dower at microsoft.com> date: 2022-12-09T12:47:18Z summary: bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) files: A Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst M Lib/test/test_winreg.py M PC/winreg.c diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 8157c2da6efa..769ab67b0f56 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -113,7 +113,6 @@ def _write_test_data(self, root_key, subkeystr="sub_key", "does not close the actual key!") except OSError: pass - def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): # Check we can get default value for this key. val = QueryValue(root_key, test_key_name) @@ -340,6 +339,23 @@ def test_setvalueex_value_range(self): finally: DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_return_value(self): # Test for Issue #16759, return unsigned int from QueryValueEx. # Reg2Py, which gets called by QueryValueEx, was returning a value diff --git a/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst new file mode 100644 index 000000000000..a5975b2d00c7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst @@ -0,0 +1,3 @@ +:meth:`winreg.SetValueEx` now leaves the target value untouched in the case of conversion errors. +Previously, ``-1`` would be written in case of such errors. + diff --git a/PC/winreg.c b/PC/winreg.c index df34e8cf5a77..63b37be526ab 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -561,42 +561,54 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) { Py_ssize_t i,j; switch (typ) { - case REG_DWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD); - if (value == Py_None) { - DWORD zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD)); - } - else { - DWORD d = PyLong_AsUnsignedLong(value); + case REG_DWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLong(value); + if (d == (DWORD)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD)); + *retDataSize = sizeof(DWORD); + break; } - break; - case REG_QWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD64); - if (value == Py_None) { - DWORD64 zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD64)); - } - else { - DWORD64 d = PyLong_AsUnsignedLongLong(value); + case REG_QWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD64 d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLongLong(value); + if (d == (DWORD64)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD64)); + *retDataSize = sizeof(DWORD64); + break; } - break; case REG_SZ: case REG_EXPAND_SZ: { From webhook-mailer at python.org Fri Dec 9 08:13:13 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 09 Dec 2022 13:13:13 -0000 Subject: [Python-checkins] bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) Message-ID: <mailman.2880.1670591593.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3939a4b7d95a403ee587ff5d222db53ec83bec2c commit: 3939a4b7d95a403ee587ff5d222db53ec83bec2c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-09T05:12:43-08:00 summary: bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) (cherry picked from commit a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1) Co-authored-by: Shreyan Avigyan <shreyan.avigyan at gmail.com> files: A Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst M Lib/test/test_winreg.py M PC/winreg.c diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 8157c2da6efa..769ab67b0f56 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -113,7 +113,6 @@ def _write_test_data(self, root_key, subkeystr="sub_key", "does not close the actual key!") except OSError: pass - def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): # Check we can get default value for this key. val = QueryValue(root_key, test_key_name) @@ -340,6 +339,23 @@ def test_setvalueex_value_range(self): finally: DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_return_value(self): # Test for Issue #16759, return unsigned int from QueryValueEx. # Reg2Py, which gets called by QueryValueEx, was returning a value diff --git a/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst new file mode 100644 index 000000000000..a5975b2d00c7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst @@ -0,0 +1,3 @@ +:meth:`winreg.SetValueEx` now leaves the target value untouched in the case of conversion errors. +Previously, ``-1`` would be written in case of such errors. + diff --git a/PC/winreg.c b/PC/winreg.c index 817717a50b3d..f668cf3c19ca 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -564,42 +564,54 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) { Py_ssize_t i,j; switch (typ) { - case REG_DWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD); - if (value == Py_None) { - DWORD zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD)); - } - else { - DWORD d = PyLong_AsUnsignedLong(value); + case REG_DWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLong(value); + if (d == (DWORD)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD)); + *retDataSize = sizeof(DWORD); + break; } - break; - case REG_QWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD64); - if (value == Py_None) { - DWORD64 zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD64)); - } - else { - DWORD64 d = PyLong_AsUnsignedLongLong(value); + case REG_QWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD64 d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLongLong(value); + if (d == (DWORD64)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD64)); + *retDataSize = sizeof(DWORD64); + break; } - break; case REG_SZ: case REG_EXPAND_SZ: { From webhook-mailer at python.org Fri Dec 9 08:17:31 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 09 Dec 2022 13:17:31 -0000 Subject: [Python-checkins] bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) Message-ID: <mailman.2881.1670591851.3313.python-checkins@python.org> https://github.com/python/cpython/commit/580165d01c579c781c9ba0327f6ff7457201cea8 commit: 580165d01c579c781c9ba0327f6ff7457201cea8 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-09T05:17:25-08:00 summary: bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) (cherry picked from commit a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1) Co-authored-by: Shreyan Avigyan <shreyan.avigyan at gmail.com> files: A Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst M Lib/test/test_winreg.py M PC/winreg.c diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 8157c2da6efa..769ab67b0f56 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -113,7 +113,6 @@ def _write_test_data(self, root_key, subkeystr="sub_key", "does not close the actual key!") except OSError: pass - def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): # Check we can get default value for this key. val = QueryValue(root_key, test_key_name) @@ -340,6 +339,23 @@ def test_setvalueex_value_range(self): finally: DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_return_value(self): # Test for Issue #16759, return unsigned int from QueryValueEx. # Reg2Py, which gets called by QueryValueEx, was returning a value diff --git a/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst new file mode 100644 index 000000000000..a5975b2d00c7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst @@ -0,0 +1,3 @@ +:meth:`winreg.SetValueEx` now leaves the target value untouched in the case of conversion errors. +Previously, ``-1`` would be written in case of such errors. + diff --git a/PC/winreg.c b/PC/winreg.c index 4fefcdcc942f..29b4029c320e 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -564,42 +564,54 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) { Py_ssize_t i,j; switch (typ) { - case REG_DWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD); - if (value == Py_None) { - DWORD zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD)); - } - else { - DWORD d = PyLong_AsUnsignedLong(value); + case REG_DWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLong(value); + if (d == (DWORD)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD)); + *retDataSize = sizeof(DWORD); + break; } - break; - case REG_QWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD64); - if (value == Py_None) { - DWORD64 zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD64)); - } - else { - DWORD64 d = PyLong_AsUnsignedLongLong(value); + case REG_QWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD64 d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLongLong(value); + if (d == (DWORD64)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD64)); + *retDataSize = sizeof(DWORD64); + break; } - break; case REG_SZ: case REG_EXPAND_SZ: { From webhook-mailer at python.org Fri Dec 9 11:14:39 2022 From: webhook-mailer at python.org (iritkatriel) Date: Fri, 09 Dec 2022 16:14:39 -0000 Subject: [Python-checkins] bpo-44512: Fix handling of extrasactions arg to csv.DictWriter with mixed or upper case (#26924) Message-ID: <mailman.2882.1670602479.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d0679c12398579fe9f10e78b6332dded119e4697 commit: d0679c12398579fe9f10e78b6332dded119e4697 branch: main author: andrei kulakov <andrei.avk at gmail.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-09T16:14:33Z summary: bpo-44512: Fix handling of extrasactions arg to csv.DictWriter with mixed or upper case (#26924) files: A Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst M Lib/csv.py M Lib/test/test_csv.py diff --git a/Lib/csv.py b/Lib/csv.py index 309a8f3f4863..4ef8be45ca9e 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -139,7 +139,8 @@ def __init__(self, f, fieldnames, restval="", extrasaction="raise", fieldnames = list(fieldnames) self.fieldnames = fieldnames # list of keys for the dict self.restval = restval # for writing short dicts - if extrasaction.lower() not in ("raise", "ignore"): + extrasaction = extrasaction.lower() + if extrasaction not in ("raise", "ignore"): raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'" % extrasaction) self.extrasaction = extrasaction diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index d64bff13a44e..8289ddb1c3a5 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -762,6 +762,10 @@ def test_write_field_not_in_field_names_raise(self): dictrow = {'f0': 0, 'f1': 1, 'f2': 2, 'f3': 3} self.assertRaises(ValueError, csv.DictWriter.writerow, writer, dictrow) + # see bpo-44512 (differently cased 'raise' should not result in 'ignore') + writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="RAISE") + self.assertRaises(ValueError, csv.DictWriter.writerow, writer, dictrow) + def test_write_field_not_in_field_names_ignore(self): fileobj = StringIO() writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="ignore") @@ -769,6 +773,10 @@ def test_write_field_not_in_field_names_ignore(self): csv.DictWriter.writerow(writer, dictrow) self.assertEqual(fileobj.getvalue(), "1,2\r\n") + # bpo-44512 + writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="IGNORE") + csv.DictWriter.writerow(writer, dictrow) + def test_dict_reader_fieldnames_accepts_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() diff --git a/Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst b/Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst new file mode 100644 index 000000000000..7f290605934d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst @@ -0,0 +1,2 @@ +Fixes inconsistent handling of case sensitivity of *extrasaction* arg in +:class:`csv.DictWriter`. From webhook-mailer at python.org Fri Dec 9 12:02:41 2022 From: webhook-mailer at python.org (rhettinger) Date: Fri, 09 Dec 2022 17:02:41 -0000 Subject: [Python-checkins] GH-98363: Shrink the physical size as well as the logical size (GH-100138) Message-ID: <mailman.2883.1670605362.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8c215466dbcdfbdedf0faba7c8ae64461069042b commit: 8c215466dbcdfbdedf0faba7c8ae64461069042b branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-09T11:02:35-06:00 summary: GH-98363: Shrink the physical size as well as the logical size (GH-100138) files: M Modules/itertoolsmodule.c diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 60ec11c32d01..c1f1e7320db7 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -182,8 +182,7 @@ batched_next(batchedobject *bo) Py_DECREF(result); return NULL; } - /* Elements in result[i:] are still NULL */ - Py_SET_SIZE(result, i); + _PyTuple_Resize(&result, i); return result; } From webhook-mailer at python.org Fri Dec 9 12:18:14 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 17:18:14 -0000 Subject: [Python-checkins] gh-81057: Fix the wasm32-wasi Buildbot (gh-100139) Message-ID: <mailman.2884.1670606295.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8d0bd93ae28299f9b2cd11d3dd1a49599b62430a commit: 8d0bd93ae28299f9b2cd11d3dd1a49599b62430a branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-09T10:17:54-07:00 summary: gh-81057: Fix the wasm32-wasi Buildbot (gh-100139) The build was broken by gh-100084. https://github.com/python/cpython/issues/81057 files: M Include/internal/pycore_pythread.h M Python/thread_pthread.h diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 4aeb285b89a8..f53921494c15 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -33,7 +33,7 @@ extern "C" { # endif /* _POSIX_THREADS */ #endif /* _POSIX_THREADS */ -#if defined(_POSIX_THREADS) && !defined(HAVE_PTHREAD_STUBS) +#if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) # define _USE_PTHREADS #endif diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index ae312e987bd6..76d6f3bcdf9c 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -132,7 +132,8 @@ init_condattr(void) if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) { condattr_monotonic = &ca; // Use monotonic clock } -#endif +# undef ca +#endif // CONDATTR_MONOTONIC } int From webhook-mailer at python.org Fri Dec 9 12:18:35 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 17:18:35 -0000 Subject: [Python-checkins] gh-81057: Fix a Reference Leak in the posix Module (gh-100140) Message-ID: <mailman.2885.1670606316.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7a0f3c1d92ef0768e082ace19d970b0ef12e7346 commit: 7a0f3c1d92ef0768e082ace19d970b0ef12e7346 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-09T10:18:29-07:00 summary: gh-81057: Fix a Reference Leak in the posix Module (gh-100140) The leak was introduced in gh-100082. https://github.com/python/cpython/issues/81057 files: M Modules/posixmodule.c diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f5175350e12a..4817973262f4 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2242,6 +2242,7 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } _posixstate *state = get_posix_state(mod); + Py_DECREF(mod); if (state == NULL) { return NULL; } From webhook-mailer at python.org Sat Dec 10 04:16:06 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 09:16:06 -0000 Subject: [Python-checkins] gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (#100052) Message-ID: <mailman.2886.1670663767.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7c0fb71fbfa8682f56c15832e2c793a6180f2ec0 commit: 7c0fb71fbfa8682f56c15832e2c793a6180f2ec0 branch: main author: ram vikram singh <ramvikrams243 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T14:46:00+05:30 summary: gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (#100052) files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 73debe5ceeaf..40f787ffe639 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4743,7 +4743,7 @@ An example of dictionary view usage:: >>> # get back a read-only proxy for the original dictionary >>> values.mapping - mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}) + mappingproxy({'bacon': 1, 'spam': 500}) >>> values.mapping['spam'] 500 From webhook-mailer at python.org Sat Dec 10 04:23:29 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 09:23:29 -0000 Subject: [Python-checkins] gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) Message-ID: <mailman.2887.1670664211.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9aca00341a75b03f08ed232f0a697eddefa5c8eb commit: 9aca00341a75b03f08ed232f0a697eddefa5c8eb branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T01:23:24-08:00 summary: gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) (cherry picked from commit 7c0fb71fbfa8682f56c15832e2c793a6180f2ec0) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 75550d6e4698..03264ebbeab9 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4733,7 +4733,7 @@ An example of dictionary view usage:: >>> # get back a read-only proxy for the original dictionary >>> values.mapping - mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}) + mappingproxy({'bacon': 1, 'spam': 500}) >>> values.mapping['spam'] 500 From webhook-mailer at python.org Sat Dec 10 04:24:51 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 09:24:51 -0000 Subject: [Python-checkins] gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) Message-ID: <mailman.2888.1670664292.3313.python-checkins@python.org> https://github.com/python/cpython/commit/92dd53bf94a2cf7686681c0486f003775692bd4a commit: 92dd53bf94a2cf7686681c0486f003775692bd4a branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T01:24:46-08:00 summary: gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) (cherry picked from commit 7c0fb71fbfa8682f56c15832e2c793a6180f2ec0) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8f90bd3efa6c..a6a7eea9f071 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4693,7 +4693,7 @@ An example of dictionary view usage:: >>> # get back a read-only proxy for the original dictionary >>> values.mapping - mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}) + mappingproxy({'bacon': 1, 'spam': 500}) >>> values.mapping['spam'] 500 From webhook-mailer at python.org Sat Dec 10 04:36:01 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 09:36:01 -0000 Subject: [Python-checkins] gh-99582: freeze `zipimport` into `_bootstrap_python` (#99583) Message-ID: <mailman.2889.1670664962.3313.python-checkins@python.org> https://github.com/python/cpython/commit/228c92eb5c126130316a32b44a0ce8f28cc5d544 commit: 228c92eb5c126130316a32b44a0ce8f28cc5d544 branch: main author: Kai Zhang <kylerzhang11 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:05:56+05:30 summary: gh-99582: freeze `zipimport` into `_bootstrap_python` (#99583) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst M Makefile.pre.in M Programs/_bootstrap_python.c diff --git a/Makefile.pre.in b/Makefile.pre.in index de42d684f166..815df69ad48c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -976,7 +976,8 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ - Python/frozen_modules/importlib._bootstrap_external.h + Python/frozen_modules/importlib._bootstrap_external.h \ + Python/frozen_modules/zipimport.h Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst new file mode 100644 index 000000000000..320d47cb9cf6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst @@ -0,0 +1 @@ +Freeze :mod:`zipimport` module into ``_bootstrap_python``. diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6e1593a0b599..6c388fc7033d 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -12,6 +12,7 @@ /* Includes for frozen modules: */ #include "Python/frozen_modules/importlib._bootstrap.h" #include "Python/frozen_modules/importlib._bootstrap_external.h" +#include "Python/frozen_modules/zipimport.h" /* End includes */ uint32_t _Py_next_func_version = 1; @@ -32,6 +33,7 @@ _Py_Deepfreeze_Fini(void) static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)}, + {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { From webhook-mailer at python.org Sat Dec 10 05:05:29 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 10:05:29 -0000 Subject: [Python-checkins] Fix potential flakiness in `test_run_until_complete_baseexception` (#100148) Message-ID: <mailman.2890.1670666730.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a9bad4d28413666edc57551dd439bca6a6a59dd9 commit: a9bad4d28413666edc57551dd439bca6a6a59dd9 branch: main author: Fantix King <fantix.king at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:35:24+05:30 summary: Fix potential flakiness in `test_run_until_complete_baseexception` (#100148) files: From webhook-mailer at python.org Sat Dec 10 05:12:46 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 10:12:46 -0000 Subject: [Python-checkins] gh-99970 Adding missing `optionflags` parameter in the documentation of `doctest` (#99971) Message-ID: <mailman.2891.1670667167.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e477348f36eff3f63ba509582622ea620fa9ae5b commit: e477348f36eff3f63ba509582622ea620fa9ae5b branch: main author: busywhitespace <busywhitespace at tuta.io> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:42:40+05:30 summary: gh-99970 Adding missing `optionflags` parameter in the documentation of `doctest` (#99971) files: M Doc/library/doctest.rst diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index b6dd7f7e1232..d6e4dca08606 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1057,7 +1057,7 @@ from text files and modules with doctests: from a text file using :func:`DocFileSuite`. -.. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, checker=None) +.. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, optionflags=0, checker=None) Convert doctest tests for a module to a :class:`unittest.TestSuite`. From webhook-mailer at python.org Sat Dec 10 05:20:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 10:20:23 -0000 Subject: [Python-checkins] gh-99728: correct typo in `datetime` format codes documentation (#99750) Message-ID: <mailman.2892.1670667624.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d5f8a2b6ad408368e728a389da918cead3ef7ee9 commit: d5f8a2b6ad408368e728a389da918cead3ef7ee9 branch: main author: Brad Wolfe <brad.wolfe at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:50:18+05:30 summary: gh-99728: correct typo in `datetime` format codes documentation (#99750) files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 0b39512a2d23..8bfed19d3fd2 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2602,7 +2602,7 @@ Notes: (9) When used with the :meth:`strptime` method, the leading zero is optional - for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. .. rubric:: Footnotes From webhook-mailer at python.org Sat Dec 10 05:28:56 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 10:28:56 -0000 Subject: [Python-checkins] gh-99728: correct typo in `datetime` format codes documentation (GH-99750) Message-ID: <mailman.2893.1670668137.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e6b0bd59481b9bc4570736c1f5ef291dbbe06b8e commit: e6b0bd59481b9bc4570736c1f5ef291dbbe06b8e branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T02:28:51-08:00 summary: gh-99728: correct typo in `datetime` format codes documentation (GH-99750) (cherry picked from commit d5f8a2b6ad408368e728a389da918cead3ef7ee9) Co-authored-by: Brad Wolfe <brad.wolfe at gmail.com> files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c2d7715a52cf..f222ec9bb771 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2570,7 +2570,7 @@ Notes: (9) When used with the :meth:`strptime` method, the leading zero is optional - for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. .. rubric:: Footnotes From webhook-mailer at python.org Sat Dec 10 05:29:10 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 10:29:10 -0000 Subject: [Python-checkins] gh-99728: correct typo in `datetime` format codes documentation (GH-99750) Message-ID: <mailman.2894.1670668151.3313.python-checkins@python.org> https://github.com/python/cpython/commit/606adb4b891b52c8b9a53d29d594e996f117c0b3 commit: 606adb4b891b52c8b9a53d29d594e996f117c0b3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T02:29:04-08:00 summary: gh-99728: correct typo in `datetime` format codes documentation (GH-99750) (cherry picked from commit d5f8a2b6ad408368e728a389da918cead3ef7ee9) Co-authored-by: Brad Wolfe <brad.wolfe at gmail.com> files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 8a7d99999f5d..e700b0a1347a 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2589,7 +2589,7 @@ Notes: (9) When used with the :meth:`strptime` method, the leading zero is optional - for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. .. rubric:: Footnotes From webhook-mailer at python.org Sat Dec 10 18:07:11 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 10 Dec 2022 23:07:11 -0000 Subject: [Python-checkins] gh-99941: Ensure that asyncio.Protocol.data_received receives immutable bytes (#100053) Message-ID: <mailman.2895.1670713631.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1bb68ba6d9de6bb7f00aee11d135123163f15887 commit: 1bb68ba6d9de6bb7f00aee11d135123163f15887 branch: main author: DarioDaF <dario.fagotto at gmail.com> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-10T15:07:02-08:00 summary: gh-99941: Ensure that asyncio.Protocol.data_received receives immutable bytes (#100053) files: A Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst M Lib/asyncio/proactor_events.py M Lib/asyncio/streams.py M Lib/test/test_asyncio/test_proactor_events.py diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index c6aab408fc74..1e2a730cf368 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -288,7 +288,8 @@ def _loop_reading(self, fut=None): # we got end-of-file so no need to reschedule a new read return - data = self._data[:length] + # It's a new slice so make it immutable so protocols upstream don't have problems + data = bytes(memoryview(self._data)[:length]) else: # the future will be replaced by next proactor.recv call fut.cancel() diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 3bd99043d096..0f9098b41956 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -688,7 +688,7 @@ async def read(self, n=-1): await self._wait_for_data('read') # This will work right even if buffer is less than n bytes - data = bytes(self._buffer[:n]) + data = bytes(memoryview(self._buffer)[:n]) del self._buffer[:n] self._maybe_resume_transport() @@ -730,7 +730,7 @@ async def readexactly(self, n): data = bytes(self._buffer) self._buffer.clear() else: - data = bytes(self._buffer[:n]) + data = bytes(memoryview(self._buffer)[:n]) del self._buffer[:n] self._maybe_resume_transport() return data diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index ae30185cef77..6cb7dc300c53 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -75,7 +75,10 @@ def test_loop_reading_data(self): called_buf = bytearray(self.buffer_size) called_buf[:len(buf)] = buf self.loop._proactor.recv_into.assert_called_with(self.sock, called_buf) - self.protocol.data_received.assert_called_with(bytearray(buf)) + self.protocol.data_received.assert_called_with(buf) + # assert_called_with maps bytearray and bytes to the same thing so check manually + # regression test for https://github.com/python/cpython/issues/99941 + self.assertIsInstance(self.protocol.data_received.call_args.args[0], bytes) @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") def test_loop_reading_no_data(self): diff --git a/Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst b/Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst new file mode 100644 index 000000000000..a019d7287207 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst @@ -0,0 +1,2 @@ +Ensure that :func:`asyncio.Protocol.data_received` receives an immutable +:class:`bytes` object (as documented), instead of :class:`bytearray`. From webhook-mailer at python.org Sat Dec 10 19:17:44 2022 From: webhook-mailer at python.org (gpshead) Date: Sun, 11 Dec 2022 00:17:44 -0000 Subject: [Python-checkins] gh-88500: Reduce memory use of `urllib.unquote` (#96763) Message-ID: <mailman.2896.1670717866.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2e279e85fece187b6058718ac7e82d1692461e26 commit: 2e279e85fece187b6058718ac7e82d1692461e26 branch: main author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-10T16:17:39-08:00 summary: gh-88500: Reduce memory use of `urllib.unquote` (#96763) `urllib.unquote_to_bytes` and `urllib.unquote` could both potentially generate `O(len(string))` intermediate `bytes` or `str` objects while computing the unquoted final result depending on the input provided. As Python objects are relatively large, this could consume a lot of ram. This switches the implementation to using an expanding `bytearray` and a generator internally instead of precomputed `split()` style operations. Microbenchmarks with some antagonistic inputs like `mess = "\u0141%%%20a%fe"*1000` show this is 10-20% slower for unquote and unquote_to_bytes and no different for typical inputs that are short or lack much unicode or % escaping. But the functions are already quite fast anyways so not a big deal. The slowdown scales consistently linear with input size as expected. Memory usage observed manually using `/usr/bin/time -v` on `python -m timeit` runs of larger inputs. Unittesting memory consumption is difficult and does not seem worthwhile. Observed memory usage is ~1/2 for `unquote()` and <1/3 for `unquote_to_bytes()` using `python -m timeit -s 'from urllib.parse import unquote, unquote_to_bytes; v="\u0141%01\u0161%20"*500_000' 'unquote_to_bytes(v)'` as a test. files: A Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst M Lib/test/test_urllib.py M Lib/urllib/parse.py diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index f067560ca6ca..2df74f5e6f99 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1104,6 +1104,8 @@ def test_unquoting(self): self.assertEqual(result.count('%'), 1, "using unquote(): not all characters escaped: " "%s" % result) + + def test_unquote_rejects_none_and_tuple(self): self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, None) self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, ()) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 4f6867accbc0..5f95c5ff7f9c 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -600,6 +600,9 @@ def urldefrag(url): def unquote_to_bytes(string): """unquote_to_bytes('abc%20def') -> b'abc def'.""" + return bytes(_unquote_impl(string)) + +def _unquote_impl(string: bytes | bytearray | str) -> bytes | bytearray: # Note: strings are encoded as UTF-8. This is only an issue if it contains # unescaped non-ASCII characters, which URIs should not. if not string: @@ -611,8 +614,8 @@ def unquote_to_bytes(string): bits = string.split(b'%') if len(bits) == 1: return string - res = [bits[0]] - append = res.append + res = bytearray(bits[0]) + append = res.extend # Delay the initialization of the table to not waste memory # if the function is never called global _hextobyte @@ -626,10 +629,20 @@ def unquote_to_bytes(string): except KeyError: append(b'%') append(item) - return b''.join(res) + return res _asciire = re.compile('([\x00-\x7f]+)') +def _generate_unquoted_parts(string, encoding, errors): + previous_match_end = 0 + for ascii_match in _asciire.finditer(string): + start, end = ascii_match.span() + yield string[previous_match_end:start] # Non-ASCII + # The ascii_match[1] group == string[start:end]. + yield _unquote_impl(ascii_match[1]).decode(encoding, errors) + previous_match_end = end + yield string[previous_match_end:] # Non-ASCII tail + def unquote(string, encoding='utf-8', errors='replace'): """Replace %xx escapes by their single-character equivalent. The optional encoding and errors parameters specify how to decode percent-encoded @@ -641,21 +654,16 @@ def unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ if isinstance(string, bytes): - return unquote_to_bytes(string).decode(encoding, errors) + return _unquote_impl(string).decode(encoding, errors) if '%' not in string: + # Is it a string-like object? string.split return string if encoding is None: encoding = 'utf-8' if errors is None: errors = 'replace' - bits = _asciire.split(string) - res = [bits[0]] - append = res.append - for i in range(1, len(bits), 2): - append(unquote_to_bytes(bits[i]).decode(encoding, errors)) - append(bits[i + 1]) - return ''.join(res) + return ''.join(_generate_unquoted_parts(string, encoding, errors)) def parse_qs(qs, keep_blank_values=False, strict_parsing=False, diff --git a/Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst b/Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst new file mode 100644 index 000000000000..ad01f5e16b16 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst @@ -0,0 +1,2 @@ +Reduced the memory usage of :func:`urllib.parse.unquote` and +:func:`urllib.parse.unquote_to_bytes` on large values. From webhook-mailer at python.org Sun Dec 11 18:21:31 2022 From: webhook-mailer at python.org (ethanfurman) Date: Sun, 11 Dec 2022 23:21:31 -0000 Subject: [Python-checkins] gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Message-ID: <mailman.2897.1670800892.3313.python-checkins@python.org> https://github.com/python/cpython/commit/868bab0fdc514cfa70ce97e484a689aee8cb5a36 commit: 868bab0fdc514cfa70ce97e484a689aee8cb5a36 branch: main author: Beweeted <Beweeted at users.noreply.github.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-11T15:20:59-08:00 summary: gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Changed from multiples of 3 to powers of 3 to match the class name. files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 295152c69082..25a6e1f0b616 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -310,12 +310,12 @@ Data Types >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): - ... return (count + 1) * 3 + ... return 3 ** (count + 1) ... FIRST = auto() ... SECOND = auto() ... >>> PowersOfThree.SECOND.value - 6 + 9 .. method:: Enum.__init_subclass__(cls, **kwds) From webhook-mailer at python.org Sun Dec 11 18:30:33 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 11 Dec 2022 23:30:33 -0000 Subject: [Python-checkins] gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Message-ID: <mailman.2898.1670801434.3313.python-checkins@python.org> https://github.com/python/cpython/commit/593c5a02470c063e1dcaf5df5f397294067d0a80 commit: 593c5a02470c063e1dcaf5df5f397294067d0a80 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-11T15:30:25-08:00 summary: gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Changed from multiples of 3 to powers of 3 to match the class name. (cherry picked from commit 868bab0fdc514cfa70ce97e484a689aee8cb5a36) Co-authored-by: Beweeted <Beweeted at users.noreply.github.com> files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index be0686203bd1..b7b4b949790b 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -316,11 +316,11 @@ Data Types >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): - ... return (count + 1) * 3 + ... return 3 ** (count + 1) ... FIRST = auto() ... SECOND = auto() >>> PowersOfThree.SECOND.value - 6 + 9 .. method:: Enum.__init_subclass__(cls, **kwds) From webhook-mailer at python.org Sun Dec 11 18:44:34 2022 From: webhook-mailer at python.org (sweeneyde) Date: Sun, 11 Dec 2022 23:44:34 -0000 Subject: [Python-checkins] gh-99688: Fix outdated tests in test_unary (#99712) Message-ID: <mailman.2899.1670802275.3313.python-checkins@python.org> https://github.com/python/cpython/commit/54289f85b2af1ecf046089ddf535dda1bdf6af24 commit: 54289f85b2af1ecf046089ddf535dda1bdf6af24 branch: main author: Yesung(Isaac) Lee <yy0221ss at gmail.com> committer: sweeneyde <36520290+sweeneyde at users.noreply.github.com> date: 2022-12-11T18:44:29-05:00 summary: gh-99688: Fix outdated tests in test_unary (#99712) * Remove duplicates from "L" suffix removal * test_invert now tests `~`. files: M Lib/test/test_unary.py diff --git a/Lib/test/test_unary.py b/Lib/test/test_unary.py index c3c17cc9f611..a45fbf6bd6bc 100644 --- a/Lib/test/test_unary.py +++ b/Lib/test/test_unary.py @@ -8,7 +8,6 @@ def test_negative(self): self.assertTrue(-2 == 0 - 2) self.assertEqual(-0, 0) self.assertEqual(--2, 2) - self.assertTrue(-2 == 0 - 2) self.assertTrue(-2.0 == 0 - 2.0) self.assertTrue(-2j == 0 - 2j) @@ -16,15 +15,13 @@ def test_positive(self): self.assertEqual(+2, 2) self.assertEqual(+0, 0) self.assertEqual(++2, 2) - self.assertEqual(+2, 2) self.assertEqual(+2.0, 2.0) self.assertEqual(+2j, 2j) def test_invert(self): - self.assertTrue(-2 == 0 - 2) - self.assertEqual(-0, 0) - self.assertEqual(--2, 2) - self.assertTrue(-2 == 0 - 2) + self.assertTrue(~2 == -(2+1)) + self.assertEqual(~0, -1) + self.assertEqual(~~2, 2) def test_no_overflow(self): nines = "9" * 32 From webhook-mailer at python.org Sun Dec 11 23:16:02 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Mon, 12 Dec 2022 04:16:02 -0000 Subject: [Python-checkins] gh-70393: Clarify mention of "middle" scope (#98839) Message-ID: <mailman.2900.1670818563.3313.python-checkins@python.org> https://github.com/python/cpython/commit/70be5e42f6e288de32e0df3c77ac22a9ddf1a74b commit: 70be5e42f6e288de32e0df3c77ac22a9ddf1a74b branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-11T20:15:55-08:00 summary: gh-70393: Clarify mention of "middle" scope (#98839) files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 9ecbf8b87efb..0e5a9402bc50 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -119,12 +119,12 @@ directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contains non-local, but also non-global names + nearest enclosing scope, contain non-local, but also non-global names * the next-to-last scope contains the current module's global names * the outermost scope (searched last) is the namespace containing built-in names If a name is declared global, then all references and assignments go directly to -the middle scope containing the module's global names. To rebind variables +the next-to-last scope containing the module's global names. To rebind variables found outside of the innermost scope, the :keyword:`nonlocal` statement can be used; if not declared nonlocal, those variables are read-only (an attempt to write to such a variable will simply create a *new* local variable in the From webhook-mailer at python.org Sun Dec 11 23:25:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 04:25:11 -0000 Subject: [Python-checkins] gh-70393: Clarify mention of "middle" scope (GH-98839) Message-ID: <mailman.2901.1670819112.3313.python-checkins@python.org> https://github.com/python/cpython/commit/71b032635d757348cb915c2e8d99392f5bbcb059 commit: 71b032635d757348cb915c2e8d99392f5bbcb059 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-11T20:25:06-08:00 summary: gh-70393: Clarify mention of "middle" scope (GH-98839) (cherry picked from commit 70be5e42f6e288de32e0df3c77ac22a9ddf1a74b) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index f27abe48b2d4..d7a24b4893fb 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -119,12 +119,12 @@ directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contains non-local, but also non-global names + nearest enclosing scope, contain non-local, but also non-global names * the next-to-last scope contains the current module's global names * the outermost scope (searched last) is the namespace containing built-in names If a name is declared global, then all references and assignments go directly to -the middle scope containing the module's global names. To rebind variables +the next-to-last scope containing the module's global names. To rebind variables found outside of the innermost scope, the :keyword:`nonlocal` statement can be used; if not declared nonlocal, those variables are read-only (an attempt to write to such a variable will simply create a *new* local variable in the From webhook-mailer at python.org Sun Dec 11 23:26:36 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 04:26:36 -0000 Subject: [Python-checkins] gh-70393: Clarify mention of "middle" scope (GH-98839) Message-ID: <mailman.2902.1670819196.3313.python-checkins@python.org> https://github.com/python/cpython/commit/eece1bdca7d04184a298a7045f389d89f3e5e8a8 commit: eece1bdca7d04184a298a7045f389d89f3e5e8a8 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-11T20:26:30-08:00 summary: gh-70393: Clarify mention of "middle" scope (GH-98839) (cherry picked from commit 70be5e42f6e288de32e0df3c77ac22a9ddf1a74b) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index f27abe48b2d4..d7a24b4893fb 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -119,12 +119,12 @@ directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contains non-local, but also non-global names + nearest enclosing scope, contain non-local, but also non-global names * the next-to-last scope contains the current module's global names * the outermost scope (searched last) is the namespace containing built-in names If a name is declared global, then all references and assignments go directly to -the middle scope containing the module's global names. To rebind variables +the next-to-last scope containing the module's global names. To rebind variables found outside of the innermost scope, the :keyword:`nonlocal` statement can be used; if not declared nonlocal, those variables are read-only (an attempt to write to such a variable will simply create a *new* local variable in the From webhook-mailer at python.org Mon Dec 12 06:40:00 2022 From: webhook-mailer at python.org (hugovk) Date: Mon, 12 Dec 2022 11:40:00 -0000 Subject: [Python-checkins] clarify the 4300-digit limit on int-str conversion (#100175) Message-ID: <mailman.2903.1670845201.3313.python-checkins@python.org> https://github.com/python/cpython/commit/935ef593211a627526b2b869ce1fc2a5e67e6cdd commit: 935ef593211a627526b2b869ce1fc2a5e67e6cdd branch: main author: Ned Batchelder <ned at nedbatchelder.com> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-12T13:39:54+02:00 summary: clarify the 4300-digit limit on int-str conversion (#100175) files: M Doc/library/stdtypes.rst M Objects/longobject.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 40f787ffe639..c785336944f5 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5503,7 +5503,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> _ = int('2' * 5432) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. >>> i = int('2' * 4300) >>> len(str(i)) 4300 @@ -5511,7 +5511,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> len(str(i_squared)) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. >>> len(hex(i_squared)) 7144 >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. diff --git a/Objects/longobject.c b/Objects/longobject.c index c84b4d3f316d..8596ce9797b5 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -36,8 +36,8 @@ medium_value(PyLongObject *x) #define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) -#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" -#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d digits) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" /* If defined, use algorithms from the _pylong.py module */ #define WITH_PYLONG_MODULE 1 From webhook-mailer at python.org Mon Dec 12 06:52:33 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 12 Dec 2022 11:52:33 -0000 Subject: [Python-checkins] gh-100176: remove incorrect version compatibility check from argument clinic (#100190) Message-ID: <mailman.2904.1670845955.3313.python-checkins@python.org> https://github.com/python/cpython/commit/621a1790c4d9e316c7228a13735bf71f2b0f5372 commit: 621a1790c4d9e316c7228a13735bf71f2b0f5372 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-12T17:22:12+05:30 summary: gh-100176: remove incorrect version compatibility check from argument clinic (#100190) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0ece814e8f18..fdf8041e14bb 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5212,10 +5212,6 @@ def state_terminal(self, line): def main(argv): import sys - - if sys.version_info.major < 3 or sys.version_info.minor < 3: - sys.exit("Error: clinic.py requires Python 3.3 or greater.") - import argparse cmdline = argparse.ArgumentParser( description="""Preprocessor for CPython C files. From webhook-mailer at python.org Mon Dec 12 06:59:34 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 12 Dec 2022 11:59:34 -0000 Subject: [Python-checkins] gh-96715 Remove redundant NULL check in `profile_trampoline` function (#96716) Message-ID: <mailman.2905.1670846374.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3221b0de6792145cb4d0d461065a956db82acc93 commit: 3221b0de6792145cb4d0d461065a956db82acc93 branch: main author: chgnrdv <52372310+chgnrdv at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-12T17:29:27+05:30 summary: gh-96715 Remove redundant NULL check in `profile_trampoline` function (#96716) Closes #96715 files: M Python/sysmodule.c diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 88f806e616f2..91f5c487c98f 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -950,10 +950,6 @@ static int profile_trampoline(PyObject *self, PyFrameObject *frame, int what, PyObject *arg) { - if (arg == NULL) { - arg = Py_None; - } - PyThreadState *tstate = _PyThreadState_GET(); PyObject *result = call_trampoline(tstate, self, frame, what, arg); if (result == NULL) { From webhook-mailer at python.org Mon Dec 12 07:06:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:06:55 -0000 Subject: [Python-checkins] clarify the 4300-digit limit on int-str conversion (GH-100175) Message-ID: <mailman.2906.1670846815.3313.python-checkins@python.org> https://github.com/python/cpython/commit/17bc55e0430f2fd08bd006364bd127ae1ff63161 commit: 17bc55e0430f2fd08bd006364bd127ae1ff63161 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:06:48-08:00 summary: clarify the 4300-digit limit on int-str conversion (GH-100175) (cherry picked from commit 935ef593211a627526b2b869ce1fc2a5e67e6cdd) Co-authored-by: Ned Batchelder <ned at nedbatchelder.com> files: M Doc/library/stdtypes.rst M Objects/longobject.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 03264ebbeab9..426eec620566 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5493,7 +5493,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> _ = int('2' * 5432) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. >>> i = int('2' * 4300) >>> len(str(i)) 4300 @@ -5501,7 +5501,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> len(str(i_squared)) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. >>> len(hex(i_squared)) 7144 >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. diff --git a/Objects/longobject.c b/Objects/longobject.c index e93461af7275..1b289d5468bd 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -36,8 +36,8 @@ medium_value(PyLongObject *x) #define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) -#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" -#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d digits) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" static inline void _Py_DECREF_INT(PyLongObject *op) From webhook-mailer at python.org Mon Dec 12 07:25:28 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:25:28 -0000 Subject: [Python-checkins] Fix: typo (Indention) (GH-99904) Message-ID: <mailman.2907.1670847929.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8711b59f7ac1803307d340e357e025043fbe2f39 commit: 8711b59f7ac1803307d340e357e025043fbe2f39 branch: main author: jarrodcolburn <jcourtlandcolburn at gmail.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:25:22-08:00 summary: Fix: typo (Indention) (GH-99904) Example needed to be indented. Was trying to call a context manger `pr` (from ` with cProfile.Profile() as pr:`) wot perform ` pr.print_stats()` once it had already exited. Automerge-Triggered-By: GH:AlexWaygood files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 2d95096f4cb8..c2189e02656c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -274,7 +274,7 @@ functions: with cProfile.Profile() as pr: # ... do something ... - pr.print_stats() + pr.print_stats() .. versionchanged:: 3.8 Added context manager support. From webhook-mailer at python.org Mon Dec 12 07:38:45 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:38:45 -0000 Subject: [Python-checkins] Fix: typo (Indention) (GH-99904) Message-ID: <mailman.2908.1670848726.3313.python-checkins@python.org> https://github.com/python/cpython/commit/60383a3cd8739cf386ed64477924d0ccca00fe9a commit: 60383a3cd8739cf386ed64477924d0ccca00fe9a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:38:39-08:00 summary: Fix: typo (Indention) (GH-99904) Example needed to be indented. Was trying to call a context manger `pr` (from ` with cProfile.Profile() as pr:`) wot perform ` pr.print_stats()` once it had already exited. (cherry picked from commit 8711b59f7ac1803307d340e357e025043fbe2f39) Co-authored-by: jarrodcolburn <jcourtlandcolburn at gmail.com> Automerge-Triggered-By: GH:AlexWaygood files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 2d95096f4cb8..c2189e02656c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -274,7 +274,7 @@ functions: with cProfile.Profile() as pr: # ... do something ... - pr.print_stats() + pr.print_stats() .. versionchanged:: 3.8 Added context manager support. From webhook-mailer at python.org Mon Dec 12 07:40:15 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:40:15 -0000 Subject: [Python-checkins] Fix: typo (Indention) (GH-99904) Message-ID: <mailman.2909.1670848815.3313.python-checkins@python.org> https://github.com/python/cpython/commit/24ed439462bdcbca6a23d0517970d9b5d195b2d6 commit: 24ed439462bdcbca6a23d0517970d9b5d195b2d6 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:40:09-08:00 summary: Fix: typo (Indention) (GH-99904) Example needed to be indented. Was trying to call a context manger `pr` (from ` with cProfile.Profile() as pr:`) wot perform ` pr.print_stats()` once it had already exited. (cherry picked from commit 8711b59f7ac1803307d340e357e025043fbe2f39) Co-authored-by: jarrodcolburn <jcourtlandcolburn at gmail.com> Automerge-Triggered-By: GH:AlexWaygood files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index cf324a57e79f..6f01d1e027ec 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -273,7 +273,7 @@ functions: with cProfile.Profile() as pr: # ... do something ... - pr.print_stats() + pr.print_stats() .. versionchanged:: 3.8 Added context manager support. From webhook-mailer at python.org Mon Dec 12 08:39:31 2022 From: webhook-mailer at python.org (zooba) Date: Mon, 12 Dec 2022 13:39:31 -0000 Subject: [Python-checkins] gh-79218: Define `MS_WIN64` macro for Mingw-w64 64bit on Windows (GH-100137) Message-ID: <mailman.2910.1670852371.3313.python-checkins@python.org> https://github.com/python/cpython/commit/158b8a07212cea6066afe8bb91f1cd542d922dba commit: 158b8a07212cea6066afe8bb91f1cd542d922dba branch: main author: GalaxySnail <me at glxys.nl> committer: zooba <steve.dower at microsoft.com> date: 2022-12-12T13:39:23Z summary: gh-79218: Define `MS_WIN64` macro for Mingw-w64 64bit on Windows (GH-100137) files: A Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst M PC/pyconfig.h diff --git a/Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst b/Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst new file mode 100644 index 000000000000..e2e6ca3c7796 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst @@ -0,0 +1 @@ +Define ``MS_WIN64`` for Mingw-w64 64bit, fix cython compilation failure. diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 1a33d4c5a1e4..1d8408b363a6 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -209,6 +209,16 @@ typedef int pid_t; #endif /* _MSC_VER */ +/* ------------------------------------------------------------------------*/ +/* mingw and mingw-w64 define __MINGW32__ */ +#ifdef __MINGW32__ + +#ifdef _WIN64 +#define MS_WIN64 +#endif + +#endif /* __MINGW32__*/ + /* ------------------------------------------------------------------------*/ /* egcs/gnu-win32 defines __GNUC__ and _WIN32 */ #if defined(__GNUC__) && defined(_WIN32) From webhook-mailer at python.org Mon Dec 12 09:22:56 2022 From: webhook-mailer at python.org (iritkatriel) Date: Mon, 12 Dec 2022 14:22:56 -0000 Subject: [Python-checkins] gh-99955: standardize return values of functions in compiler's code-gen (#100010) Message-ID: <mailman.2911.1670854977.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e4ea33b17807d99ed737f800d9b0006957c008d2 commit: e4ea33b17807d99ed737f800d9b0006957c008d2 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-12T14:22:15Z summary: gh-99955: standardize return values of functions in compiler's code-gen (#100010) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst M Python/compile.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst new file mode 100644 index 000000000000..e9867b39ad7d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst @@ -0,0 +1 @@ +Internal compiler functions (in compile.c) now consistently return -1 on error and 0 on success. diff --git a/Python/compile.c b/Python/compile.c index d6ed6941ac1e..17b164a4d06e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -55,6 +55,14 @@ */ #define STACK_USE_GUIDELINE 30 +#define SUCCESS 0 +#define ERROR -1 + +#define RETURN_IF_ERROR(X) \ + if ((X) == -1) { \ + return ERROR; \ + } + /* If we exceed this limit, it should * be considered a compiler bug. * Currently it should be impossible @@ -498,7 +506,7 @@ static int compiler_annassign(struct compiler *, stmt_ty); static int compiler_subscript(struct compiler *, expr_ty); static int compiler_slice(struct compiler *, expr_ty); -static int are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); +static bool are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); static int compiler_with(struct compiler *, stmt_ty, int); @@ -610,18 +618,18 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, { c->c_const_cache = PyDict_New(); if (!c->c_const_cache) { - return 0; + return ERROR; } c->c_stack = PyList_New(0); if (!c->c_stack) { - return 0; + return ERROR; } c->c_filename = Py_NewRef(filename); c->c_arena = arena; if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { - return 0; + return ERROR; } int merged = c->c_future.ff_features | flags.cf_flags; c->c_future.ff_features = merged; @@ -635,16 +643,16 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, state.ff_features = merged; if (!_PyAST_Optimize(mod, arena, &state)) { - return 0; + return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); if (c->c_st == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_SystemError, "no symtable"); } - return 0; + return ERROR; } - return 1; + return SUCCESS; } static struct compiler* @@ -656,7 +664,7 @@ new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, if (c == NULL) { return NULL; } - if (!compiler_setup(c, mod, filename, flags, optimize, arena)) { + if (compiler_setup(c, mod, filename, flags, optimize, arena) < 0) { compiler_free(c); return NULL; } @@ -800,11 +808,12 @@ cfg_builder_init(cfg_builder *g) { g->g_block_list = NULL; basicblock *block = cfg_builder_new_block(g); - if (block == NULL) - return 0; + if (block == NULL) { + return ERROR; + } g->g_curblock = g->g_entryblock = block; g->g_current_label = NO_LABEL; - return 1; + return SUCCESS; } static void @@ -862,8 +871,10 @@ compiler_set_qualname(struct compiler *c) || u->u_scope_type == COMPILER_SCOPE_CLASS) { assert(u->u_name); mangled = _Py_Mangle(parent->u_private, u->u_name); - if (!mangled) - return 0; + if (!mangled) { + return ERROR; + } + scope = _PyST_GetScope(parent->u_ste, mangled); Py_DECREF(mangled); assert(scope != GLOBAL_IMPLICIT); @@ -879,8 +890,9 @@ compiler_set_qualname(struct compiler *c) _Py_DECLARE_STR(dot_locals, ".<locals>"); base = PyUnicode_Concat(parent->u_qualname, &_Py_STR(dot_locals)); - if (base == NULL) - return 0; + if (base == NULL) { + return ERROR; + } } else { base = Py_NewRef(parent->u_qualname); @@ -892,18 +904,20 @@ compiler_set_qualname(struct compiler *c) _Py_DECLARE_STR(dot, "."); name = PyUnicode_Concat(base, &_Py_STR(dot)); Py_DECREF(base); - if (name == NULL) - return 0; + if (name == NULL) { + return ERROR; + } PyUnicode_Append(&name, u->u_name); - if (name == NULL) - return 0; + if (name == NULL) { + return ERROR; + } } else { name = Py_NewRef(u->u_name); } u->u_qualname = name; - return 1; + return SUCCESS; } static jump_target_label @@ -1304,10 +1318,6 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return stack_effect(opcode, oparg, -1); } -/* Add an opcode with no argument. - Returns 0 on failure, 1 on success. -*/ - static int basicblock_addop(basicblock *b, int opcode, int oparg, location loc) { @@ -1318,7 +1328,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) int off = basicblock_next_instr(b); if (off < 0) { - return 0; + return ERROR; } struct instr *i = &b->b_instr[off]; i->i_opcode = opcode; @@ -1326,7 +1336,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) i->i_target = NULL; i->i_loc = loc; - return 1; + return SUCCESS; } static bool @@ -1522,8 +1532,9 @@ static int compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) { Py_ssize_t arg = compiler_add_const(c, o); - if (arg < 0) - return 0; + if (arg < 0) { + return ERROR; + } return cfg_builder_addop_i(CFG_BUILDER(c), LOAD_CONST, arg, loc); } @@ -1532,8 +1543,9 @@ compiler_addop_o(struct compiler *c, location loc, int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); - if (arg < 0) - return 0; + if (arg < 0) { + return ERROR; + } return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); } @@ -1544,12 +1556,14 @@ compiler_addop_name(struct compiler *c, location loc, Py_ssize_t arg; PyObject *mangled = _Py_Mangle(c->u->u_private, o); - if (!mangled) - return 0; + if (!mangled) { + return ERROR; + } arg = dict_add_o(dict, mangled); Py_DECREF(mangled); - if (arg < 0) - return 0; + if (arg < 0) { + return ERROR; + } if (opcode == LOAD_ATTR) { arg <<= 1; } @@ -1561,9 +1575,7 @@ compiler_addop_name(struct compiler *c, location loc, return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); } -/* Add an opcode with an integer argument. - Returns 0 on failure, 1 on success. -*/ +/* Add an opcode with an integer argument */ static int cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) { @@ -1588,94 +1600,79 @@ cfg_builder_addop_j(cfg_builder *g, location loc, return cfg_builder_addop(g, opcode, target.id, loc); } - -#define ADDOP(C, LOC, OP) { \ - if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) \ - return 0; \ -} +#define ADDOP(C, LOC, OP) \ + RETURN_IF_ERROR(cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) #define ADDOP_IN_SCOPE(C, LOC, OP) { \ - if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) { \ + if (cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC)) < 0) { \ compiler_exit_scope(c); \ - return 0; \ + return -1; \ } \ } -#define ADDOP_LOAD_CONST(C, LOC, O) { \ - if (!compiler_addop_load_const((C), (LOC), (O))) \ - return 0; \ -} +#define ADDOP_LOAD_CONST(C, LOC, O) \ + RETURN_IF_ERROR(compiler_addop_load_const((C), (LOC), (O))) /* Same as ADDOP_LOAD_CONST, but steals a reference. */ #define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ PyObject *__new_const = (O); \ if (__new_const == NULL) { \ - return 0; \ + return ERROR; \ } \ - if (!compiler_addop_load_const((C), (LOC), __new_const)) { \ + if (compiler_addop_load_const((C), (LOC), __new_const) < 0) { \ Py_DECREF(__new_const); \ - return 0; \ + return ERROR; \ } \ Py_DECREF(__new_const); \ } #define ADDOP_N(C, LOC, OP, O, TYPE) { \ assert(!HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (!compiler_addop_o((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) { \ + if (compiler_addop_o((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O)) < 0) { \ Py_DECREF((O)); \ - return 0; \ + return ERROR; \ } \ Py_DECREF((O)); \ } -#define ADDOP_NAME(C, LOC, OP, O, TYPE) { \ - if (!compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) \ - return 0; \ -} +#define ADDOP_NAME(C, LOC, OP, O, TYPE) \ + RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) -#define ADDOP_I(C, LOC, OP, O) { \ - if (!cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) \ - return 0; \ -} +#define ADDOP_I(C, LOC, OP, O) \ + RETURN_IF_ERROR(cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) -#define ADDOP_JUMP(C, LOC, OP, O) { \ - if (!cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) \ - return 0; \ -} +#define ADDOP_JUMP(C, LOC, OP, O) \ + RETURN_IF_ERROR(cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) -#define ADDOP_COMPARE(C, LOC, CMP) { \ - if (!compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) \ - return 0; \ -} +#define ADDOP_COMPARE(C, LOC, CMP) \ + RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) #define ADDOP_BINARY(C, LOC, BINOP) \ - RETURN_IF_FALSE(addop_binary((C), (LOC), (BINOP), false)) + RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false)) #define ADDOP_INPLACE(C, LOC, BINOP) \ - RETURN_IF_FALSE(addop_binary((C), (LOC), (BINOP), true)) + RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true)) #define ADD_YIELD_FROM(C, LOC, await) \ - RETURN_IF_FALSE(compiler_add_yield_from((C), (LOC), (await))) + RETURN_IF_ERROR(compiler_add_yield_from((C), (LOC), (await))) #define POP_EXCEPT_AND_RERAISE(C, LOC) \ - RETURN_IF_FALSE(compiler_pop_except_and_reraise((C), (LOC))) + RETURN_IF_ERROR(compiler_pop_except_and_reraise((C), (LOC))) #define ADDOP_YIELD(C, LOC) \ - RETURN_IF_FALSE(addop_yield((C), (LOC))) + RETURN_IF_ERROR(addop_yield((C), (LOC))) /* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. */ -#define VISIT(C, TYPE, V) {\ - if (!compiler_visit_ ## TYPE((C), (V))) \ - return 0; \ -} +#define VISIT(C, TYPE, V) \ + RETURN_IF_ERROR(compiler_visit_ ## TYPE((C), (V))); #define VISIT_IN_SCOPE(C, TYPE, V) {\ - if (!compiler_visit_ ## TYPE((C), (V))) { \ + if (compiler_visit_ ## TYPE((C), (V)) < 0) { \ compiler_exit_scope(c); \ - return 0; \ + return ERROR; \ } \ } @@ -1684,8 +1681,8 @@ cfg_builder_addop_j(cfg_builder *g, location loc, asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - if (!compiler_visit_ ## TYPE((C), elt)) \ - return 0; \ + if (compiler_visit_ ## TYPE((C), elt) < 0) \ + return ERROR; \ } \ } @@ -1694,17 +1691,13 @@ cfg_builder_addop_j(cfg_builder *g, location loc, asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - if (!compiler_visit_ ## TYPE((C), elt)) { \ + if (compiler_visit_ ## TYPE((C), elt) < 0) { \ compiler_exit_scope(c); \ - return 0; \ + return ERROR; \ } \ } \ } -#define RETURN_IF_FALSE(X) \ - if (!(X)) { \ - return 0; \ - } static int compiler_enter_scope(struct compiler *c, identifier name, @@ -1718,7 +1711,7 @@ compiler_enter_scope(struct compiler *c, identifier name, struct compiler_unit)); if (!u) { PyErr_NoMemory(); - return 0; + return ERROR; } u->u_scope_type = scope_type; u->u_argcount = 0; @@ -1727,14 +1720,14 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_ste = PySymtable_Lookup(c->c_st, key); if (!u->u_ste) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_name = Py_NewRef(name); u->u_varnames = list2dict(u->u_ste->ste_varnames); u->u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, 0, 0); if (!u->u_varnames || !u->u_cellvars) { compiler_unit_free(u); - return 0; + return ERROR; } if (u->u_ste->ste_needs_class_closure) { /* Cook up an implicit __class__ cell. */ @@ -1745,7 +1738,7 @@ compiler_enter_scope(struct compiler *c, identifier name, _PyLong_GetZero()); if (res < 0) { compiler_unit_free(u); - return 0; + return ERROR; } } @@ -1753,7 +1746,7 @@ compiler_enter_scope(struct compiler *c, identifier name, PyDict_GET_SIZE(u->u_cellvars)); if (!u->u_freevars) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_nfblocks = 0; @@ -1761,12 +1754,12 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_consts = PyDict_New(); if (!u->u_consts) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_names = PyDict_New(); if (!u->u_names) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_private = NULL; @@ -1777,7 +1770,7 @@ compiler_enter_scope(struct compiler *c, identifier name, if (!capsule || PyList_Append(c->c_stack, capsule) < 0) { Py_XDECREF(capsule); compiler_unit_free(u); - return 0; + return ERROR; } Py_DECREF(capsule); u->u_private = Py_XNewRef(c->u->u_private); @@ -1787,23 +1780,20 @@ compiler_enter_scope(struct compiler *c, identifier name, c->c_nestlevel++; cfg_builder *g = CFG_BUILDER(c); - if (!cfg_builder_init(g)) { - return 0; - } + RETURN_IF_ERROR(cfg_builder_init(g)); if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = 0; } else { - if (!compiler_set_qualname(c)) - return 0; + RETURN_IF_ERROR(compiler_set_qualname(c)); } ADDOP_I(c, loc, RESUME, 0); if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = -1; } - return 1; + return SUCCESS; } static void @@ -1837,7 +1827,7 @@ compiler_exit_scope(struct compiler *c) /* Search if variable annotations are present statically in a block. */ -static int +static bool find_ann(asdl_stmt_seq *stmts) { int i, j, res = 0; @@ -1847,7 +1837,7 @@ find_ann(asdl_stmt_seq *stmts) st = (stmt_ty)asdl_seq_GET(stmts, i); switch (st->kind) { case AnnAssign_kind: - return 1; + return true; case For_kind: res = find_ann(st->v.For.body) || find_ann(st->v.For.orelse); @@ -1875,7 +1865,7 @@ find_ann(asdl_stmt_seq *stmts) excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( st->v.Try.handlers, j); if (find_ann(handler->v.ExceptHandler.body)) { - return 1; + return true; } } res = find_ann(st->v.Try.body) || @@ -1887,7 +1877,7 @@ find_ann(asdl_stmt_seq *stmts) excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( st->v.TryStar.handlers, j); if (find_ann(handler->v.ExceptHandler.body)) { - return 1; + return true; } } res = find_ann(st->v.TryStar.body) || @@ -1895,7 +1885,7 @@ find_ann(asdl_stmt_seq *stmts) find_ann(st->v.TryStar.orelse); break; default: - res = 0; + res = false; } if (res) { break; @@ -1922,7 +1912,7 @@ compiler_push_fblock(struct compiler *c, location loc, f->fb_block = block_label; f->fb_exit = exit; f->fb_datum = datum; - return 1; + return SUCCESS; } static void @@ -1942,7 +1932,7 @@ compiler_call_exit_with_nones(struct compiler *c, location loc) ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_I(c, loc, CALL, 2); - return 1; + return SUCCESS; } static int @@ -1966,7 +1956,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) ADDOP(c, loc, CLEANUP_THROW); USE_LABEL(c, exit); - return 1; + return SUCCESS; } static int @@ -1982,7 +1972,7 @@ compiler_pop_except_and_reraise(struct compiler *c, location loc) ADDOP_I(c, loc, COPY, 3); ADDOP(c, loc, POP_EXCEPT); ADDOP_I(c, loc, RERAISE, 1); - return 1; + return SUCCESS; } /* Unwind a frame block. If preserve_tos is true, the TOS before @@ -1999,7 +1989,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, case EXCEPTION_HANDLER: case EXCEPTION_GROUP_HANDLER: case ASYNC_COMPREHENSION_GENERATOR: - return 1; + return SUCCESS; case FOR_LOOP: /* Pop the iterator */ @@ -2007,19 +1997,18 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, ADDOP_I(c, *ploc, SWAP, 2); } ADDOP(c, *ploc, POP_TOP); - return 1; + return SUCCESS; case TRY_EXCEPT: ADDOP(c, *ploc, POP_BLOCK); - return 1; + return SUCCESS; case FINALLY_TRY: /* This POP_BLOCK gets the line number of the unwinding statement */ ADDOP(c, *ploc, POP_BLOCK); if (preserve_tos) { - if (!compiler_push_fblock(c, *ploc, POP_VALUE, NO_LABEL, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, *ploc, POP_VALUE, NO_LABEL, NO_LABEL, NULL)); } /* Emit the finally block */ VISIT_SEQ(c, stmt, info->fb_datum); @@ -2030,7 +2019,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, * statement causing the unwinding, so make the unwinding * instruction artificial */ *ploc = NO_LOCATION; - return 1; + return SUCCESS; case FINALLY_END: if (preserve_tos) { @@ -2042,7 +2031,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, } ADDOP(c, *ploc, POP_BLOCK); ADDOP(c, *ploc, POP_EXCEPT); - return 1; + return SUCCESS; case WITH: case ASYNC_WITH: @@ -2051,9 +2040,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, if (preserve_tos) { ADDOP_I(c, *ploc, SWAP, 2); } - if(!compiler_call_exit_with_nones(c, *ploc)) { - return 0; - } + RETURN_IF_ERROR(compiler_call_exit_with_nones(c, *ploc)); if (info->fb_type == ASYNC_WITH) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, *ploc, Py_None); @@ -2064,7 +2051,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, * statement causing the unwinding, so make the unwinding * instruction artificial */ *ploc = NO_LOCATION; - return 1; + return SUCCESS; case HANDLER_CLEANUP: { if (info->fb_datum) { @@ -2077,17 +2064,17 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, ADDOP(c, *ploc, POP_EXCEPT); if (info->fb_datum) { ADDOP_LOAD_CONST(c, *ploc, Py_None); - compiler_nameop(c, *ploc, info->fb_datum, Store); - compiler_nameop(c, *ploc, info->fb_datum, Del); + RETURN_IF_ERROR(compiler_nameop(c, *ploc, info->fb_datum, Store)); + RETURN_IF_ERROR(compiler_nameop(c, *ploc, info->fb_datum, Del)); } - return 1; + return SUCCESS; } case POP_VALUE: { if (preserve_tos) { ADDOP_I(c, *ploc, SWAP, 2); } ADDOP(c, *ploc, POP_TOP); - return 1; + return SUCCESS; } } Py_UNREACHABLE(); @@ -2099,7 +2086,7 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, int preserve_tos, struct fblockinfo **loop) { if (c->u->u_nfblocks == 0) { - return 1; + return SUCCESS; } struct fblockinfo *top = &c->u->u_fblock[c->u->u_nfblocks-1]; if (top->fb_type == EXCEPTION_GROUP_HANDLER) { @@ -2108,19 +2095,15 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, } if (loop != NULL && (top->fb_type == WHILE_LOOP || top->fb_type == FOR_LOOP)) { *loop = top; - return 1; + return SUCCESS; } struct fblockinfo copy = *top; c->u->u_nfblocks--; - if (!compiler_unwind_fblock(c, ploc, ©, preserve_tos)) { - return 0; - } - if (!compiler_unwind_fblock_stack(c, ploc, preserve_tos, loop)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock(c, ploc, ©, preserve_tos)); + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, ploc, preserve_tos, loop)); c->u->u_fblock[c->u->u_nfblocks] = copy; c->u->u_nfblocks++; - return 1; + return SUCCESS; } /* Compile a sequence of statements, checking for a docstring @@ -2145,8 +2128,9 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) if (find_ann(stmts)) { ADDOP(c, loc, SETUP_ANNOTATIONS); } - if (!asdl_seq_LEN(stmts)) - return 1; + if (!asdl_seq_LEN(stmts)) { + return SUCCESS; + } /* if not -OO mode, set docstring */ if (c->c_optimize < 2) { docstring = _PyAST_GetDocString(stmts); @@ -2155,29 +2139,29 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) st = (stmt_ty)asdl_seq_GET(stmts, 0); assert(st->kind == Expr_kind); VISIT(c, expr, st->v.Expr.value); - if (!compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)) - return 0; + RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } - for (; i < asdl_seq_LEN(stmts); i++) + for (; i < asdl_seq_LEN(stmts); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); - return 1; + } + return SUCCESS; } static int compiler_codegen(struct compiler *c, mod_ty mod) { _Py_DECLARE_STR(anon_module, "<module>"); - if (!compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, + mod, 1)); + location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { case Module_kind: - if (!compiler_body(c, loc, mod->v.Module.body)) { + if (compiler_body(c, loc, mod->v.Module.body) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } break; case Interactive_kind: @@ -2194,16 +2178,16 @@ compiler_codegen(struct compiler *c, mod_ty mod) PyErr_Format(PyExc_SystemError, "module kind %d should not be possible", mod->kind); - return 0; + return ERROR; } - return 1; + return SUCCESS; } static PyCodeObject * compiler_mod(struct compiler *c, mod_ty mod) { int addNone = mod->kind != Expression_kind; - if (!compiler_codegen(c, mod)) { + if (compiler_codegen(c, mod) < 0) { return NULL; } PyCodeObject *co = assemble(c, addNone); @@ -2270,7 +2254,7 @@ compiler_make_closure(struct compiler *c, location loc, */ int reftype = get_ref_type(c, name); if (reftype == -1) { - return 0; + return ERROR; } int arg; if (reftype == CELL) { @@ -2293,7 +2277,7 @@ compiler_make_closure(struct compiler *c, location loc, co->co_name, freevars); Py_DECREF(freevars); - return 0; + return ERROR; } ADDOP_I(c, loc, LOAD_CLOSURE, arg); } @@ -2302,34 +2286,34 @@ compiler_make_closure(struct compiler *c, location loc, } ADDOP_LOAD_CONST(c, loc, (PyObject*)co); ADDOP_I(c, loc, MAKE_FUNCTION, flags); - return 1; + return SUCCESS; } static int compiler_decorators(struct compiler *c, asdl_expr_seq* decos) { - int i; - - if (!decos) - return 1; + if (!decos) { + return SUCCESS; + } - for (i = 0; i < asdl_seq_LEN(decos); i++) { + for (Py_ssize_t i = 0; i < asdl_seq_LEN(decos); i++) { VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i)); } - return 1; + return SUCCESS; } static int compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) { - if (!decos) - return 1; + if (!decos) { + return SUCCESS; + } for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); ADDOP_I(c, loc, CALL, 0); } - return 1; + return SUCCESS; } static int @@ -2338,7 +2322,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, { /* Push a dict of keyword-only default values. - Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed. + Return -1 on error, 0 if no dict pushed, 1 if a dict is pushed. */ int i; PyObject *keys = NULL; @@ -2355,7 +2339,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, keys = PyList_New(1); if (keys == NULL) { Py_DECREF(mangled); - return 0; + return ERROR; } PyList_SET_ITEM(keys, 0, mangled); } @@ -2366,7 +2350,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, goto error; } } - if (!compiler_visit_expr(c, default_)) { + if (compiler_visit_expr(c, default_) < 0) { goto error; } } @@ -2381,12 +2365,12 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, return 1; } else { - return -1; + return 0; } error: Py_XDECREF(keys); - return 0; + return ERROR; } static int @@ -2394,7 +2378,7 @@ compiler_visit_annexpr(struct compiler *c, expr_ty annotation) { location loc = LOC(annotation); ADDOP_LOAD_CONST_NEW(c, loc, _PyAST_ExprAsUnicode(annotation)); - return 1; + return SUCCESS; } static int @@ -2402,11 +2386,11 @@ compiler_visit_argannotation(struct compiler *c, identifier id, expr_ty annotation, Py_ssize_t *annotations_len, location loc) { if (!annotation) { - return 1; + return SUCCESS; } PyObject *mangled = _Py_Mangle(c->u->u_private, id); if (!mangled) { - return 0; + return ERROR; } ADDOP_LOAD_CONST(c, loc, mangled); Py_DECREF(mangled); @@ -2428,7 +2412,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, } } *annotations_len += 2; - return 1; + return SUCCESS; } static int @@ -2438,15 +2422,15 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, int i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (!compiler_visit_argannotation( + RETURN_IF_ERROR( + compiler_visit_argannotation( c, arg->arg, arg->annotation, annotations_len, - loc)) - return 0; + loc)); } - return 1; + return SUCCESS; } static int @@ -2456,36 +2440,40 @@ compiler_visit_annotations(struct compiler *c, location loc, /* Push arg annotation names and values. The expressions are evaluated out-of-order wrt the source code. - Return 0 on error, -1 if no annotations pushed, 1 if a annotations is pushed. + Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed. */ Py_ssize_t annotations_len = 0; - if (!compiler_visit_argannotations(c, args->args, &annotations_len, loc)) - return 0; - if (!compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)) - return 0; - if (args->vararg && args->vararg->annotation && - !compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len, loc)) - return 0; - if (!compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)) - return 0; - if (args->kwarg && args->kwarg->annotation && - !compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len, loc)) - return 0; + RETURN_IF_ERROR( + compiler_visit_argannotations(c, args->args, &annotations_len, loc)); - if (!compiler_visit_argannotation(c, &_Py_ID(return), returns, - &annotations_len, loc)) { - return 0; + RETURN_IF_ERROR( + compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)); + + if (args->vararg && args->vararg->annotation) { + RETURN_IF_ERROR( + compiler_visit_argannotation(c, args->vararg->arg, + args->vararg->annotation, &annotations_len, loc)); } + RETURN_IF_ERROR( + compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)); + + if (args->kwarg && args->kwarg->annotation) { + RETURN_IF_ERROR( + compiler_visit_argannotation(c, args->kwarg->arg, + args->kwarg->annotation, &annotations_len, loc)); + } + + RETURN_IF_ERROR( + compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc)); + if (annotations_len) { ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); return 1; } - return -1; + return 0; } static int @@ -2494,7 +2482,7 @@ compiler_visit_defaults(struct compiler *c, arguments_ty args, { VISIT_SEQ(c, expr, args->defaults); ADDOP_I(c, loc, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); - return 1; + return SUCCESS; } static Py_ssize_t @@ -2503,47 +2491,45 @@ compiler_default_arguments(struct compiler *c, location loc, { Py_ssize_t funcflags = 0; if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { - if (!compiler_visit_defaults(c, args, loc)) - return -1; + RETURN_IF_ERROR(compiler_visit_defaults(c, args, loc)); funcflags |= 0x01; } if (args->kwonlyargs) { int res = compiler_visit_kwonlydefaults(c, loc, args->kwonlyargs, args->kw_defaults); - if (res == 0) { - return -1; - } - else if (res > 0) { + RETURN_IF_ERROR(res); + if (res > 0) { funcflags |= 0x02; } } return funcflags; } -static int +static bool forbidden_name(struct compiler *c, location loc, identifier name, expr_context_ty ctx) { if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { compiler_error(c, loc, "cannot assign to __debug__"); - return 1; + return true; } if (ctx == Del && _PyUnicode_EqualToASCIIString(name, "__debug__")) { compiler_error(c, loc, "cannot delete __debug__"); - return 1; + return true; } - return 0; + return false; } static int compiler_check_debug_one_arg(struct compiler *c, arg_ty arg) { if (arg != NULL) { - if (forbidden_name(c, LOC(arg), arg->arg, Store)) - return 0; + if (forbidden_name(c, LOC(arg), arg->arg, Store)) { + return ERROR; + } } - return 1; + return SUCCESS; } static int @@ -2551,39 +2537,32 @@ compiler_check_debug_args_seq(struct compiler *c, asdl_arg_seq *args) { if (args != NULL) { for (Py_ssize_t i = 0, n = asdl_seq_LEN(args); i < n; i++) { - if (!compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))) - return 0; + RETURN_IF_ERROR( + compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))); } } - return 1; + return SUCCESS; } static int compiler_check_debug_args(struct compiler *c, arguments_ty args) { - if (!compiler_check_debug_args_seq(c, args->posonlyargs)) - return 0; - if (!compiler_check_debug_args_seq(c, args->args)) - return 0; - if (!compiler_check_debug_one_arg(c, args->vararg)) - return 0; - if (!compiler_check_debug_args_seq(c, args->kwonlyargs)) - return 0; - if (!compiler_check_debug_one_arg(c, args->kwarg)) - return 0; - return 1; + RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->posonlyargs)); + RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->args)); + RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->vararg)); + RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->kwonlyargs)); + RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->kwarg)); + return SUCCESS; } static inline int insert_instruction(basicblock *block, int pos, struct instr *instr) { - if (basicblock_next_instr(block) < 0) { - return -1; - } + RETURN_IF_ERROR(basicblock_next_instr(block)); for (int i = block->b_iused - 1; i > pos; i--) { block->b_instr[i] = block->b_instr[i-1]; } block->b_instr[pos] = *instr; - return 0; + return SUCCESS; } static int @@ -2598,16 +2577,15 @@ wrap_in_stopiteration_handler(struct compiler *c) .i_loc = NO_LOCATION, .i_target = NULL, }; - if (insert_instruction(c->u->u_cfg_builder.g_entryblock, 0, &setup)) { - return 0; - } + RETURN_IF_ERROR( + insert_instruction(c->u->u_cfg_builder.g_entryblock, 0, &setup)); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); ADDOP(c, NO_LOCATION, RETURN_VALUE); USE_LABEL(c, handler); ADDOP(c, NO_LOCATION, STOPITERATION_ERROR); ADDOP_I(c, NO_LOCATION, RERAISE, 1); - return 1; + return SUCCESS; } static int @@ -2647,11 +2625,8 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) scope_type = COMPILER_SCOPE_FUNCTION; } - if (!compiler_check_debug_args(c, args)) - return 0; - - if (!compiler_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_check_debug_args(c, args)); + RETURN_IF_ERROR(compiler_decorators(c, decos)); firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2661,19 +2636,16 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) location loc = LOC(s); funcflags = compiler_default_arguments(c, loc, args); if (funcflags == -1) { - return 0; + return ERROR; } annotations = compiler_visit_annotations(c, loc, args, returns); - if (annotations == 0) { - return 0; - } - else if (annotations > 0) { + RETURN_IF_ERROR(annotations); + if (annotations > 0) { funcflags |= 0x04; } - if (!compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)); /* if not -OO mode, add docstring */ if (c->c_optimize < 2) { @@ -2681,7 +2653,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } c->u->u_argcount = asdl_seq_LEN(args->args); @@ -2691,9 +2663,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } if (c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator) { - if (!wrap_in_stopiteration_handler(c)) { + if (wrap_in_stopiteration_handler(c) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } } co = assemble(c, 1); @@ -2702,18 +2674,17 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (co == NULL) { Py_XDECREF(qualname); Py_XDECREF(co); - return 0; + return ERROR; } - if (!compiler_make_closure(c, loc, co, funcflags, qualname)) { + if (compiler_make_closure(c, loc, co, funcflags, qualname) < 0) { Py_DECREF(qualname); Py_DECREF(co); - return 0; + return ERROR; } Py_DECREF(qualname); Py_DECREF(co); - if (!compiler_apply_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); return compiler_nameop(c, loc, name, Store); } @@ -2724,8 +2695,7 @@ compiler_class(struct compiler *c, stmt_ty s) int i, firstlineno; asdl_expr_seq *decos = s->v.ClassDef.decorator_list; - if (!compiler_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_decorators(c, decos)); firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2743,35 +2713,35 @@ compiler_class(struct compiler *c, stmt_ty s) This borrows from compiler_call. */ /* 1. compile the class body into a code object */ - if (!compiler_enter_scope(c, s->v.ClassDef.name, - COMPILER_SCOPE_CLASS, (void *)s, firstlineno)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, s->v.ClassDef.name, + COMPILER_SCOPE_CLASS, (void *)s, firstlineno)); + /* this block represents what we do in the new scope */ { location loc = LOCATION(firstlineno, firstlineno, 0, 0); /* use the class name for name mangling */ Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); /* load (global) __name__ ... */ - if (!compiler_nameop(c, loc, &_Py_ID(__name__), Load)) { + if (compiler_nameop(c, loc, &_Py_ID(__name__), Load) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } /* ... and store it as __module__ */ - if (!compiler_nameop(c, loc, &_Py_ID(__module__), Store)) { + if (compiler_nameop(c, loc, &_Py_ID(__module__), Store) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } assert(c->u->u_qualname); ADDOP_LOAD_CONST(c, loc, c->u->u_qualname); - if (!compiler_nameop(c, loc, &_Py_ID(__qualname__), Store)) { + if (compiler_nameop(c, loc, &_Py_ID(__qualname__), Store) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } /* compile the body proper */ - if (!compiler_body(c, loc, s->v.ClassDef.body)) { + if (compiler_body(c, loc, s->v.ClassDef.body) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } /* The following code is artificial */ /* Return __classcell__ if it is referenced, otherwise return None */ @@ -2780,14 +2750,14 @@ compiler_class(struct compiler *c, stmt_ty s) i = compiler_lookup_arg(c->u->u_cellvars, &_Py_ID(__class__)); if (i < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } assert(i == 0); ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); ADDOP_I(c, NO_LOCATION, COPY, 1); - if (!compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store)) { + if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } } else { @@ -2801,8 +2771,9 @@ compiler_class(struct compiler *c, stmt_ty s) } /* leave the new scope */ compiler_exit_scope(c); - if (co == NULL) - return 0; + if (co == NULL) { + return ERROR; + } location loc = LOC(s); /* 2. load the 'build_class' function */ @@ -2810,9 +2781,9 @@ compiler_class(struct compiler *c, stmt_ty s) ADDOP(c, loc, LOAD_BUILD_CLASS); /* 3. load a function (or closure) made from the code object */ - if (!compiler_make_closure(c, loc, co, 0, NULL)) { + if (compiler_make_closure(c, loc, co, 0, NULL) < 0) { Py_DECREF(co); - return 0; + return ERROR; } Py_DECREF(co); @@ -2820,27 +2791,25 @@ compiler_class(struct compiler *c, stmt_ty s) ADDOP_LOAD_CONST(c, loc, s->v.ClassDef.name); /* 5. generate the rest of the code for the call */ - if (!compiler_call_helper(c, loc, 2, - s->v.ClassDef.bases, - s->v.ClassDef.keywords)) - return 0; + RETURN_IF_ERROR(compiler_call_helper(c, loc, 2, + s->v.ClassDef.bases, + s->v.ClassDef.keywords)); + /* 6. apply decorators */ - if (!compiler_apply_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); /* 7. store into <name> */ - if (!compiler_nameop(c, loc, s->v.ClassDef.name, Store)) - return 0; - return 1; + RETURN_IF_ERROR(compiler_nameop(c, loc, s->v.ClassDef.name, Store)); + return SUCCESS; } -/* Return 0 if the expression is a constant value except named singletons. - Return 1 otherwise. */ -static int +/* Return false if the expression is a constant value except named singletons. + Return true otherwise. */ +static bool check_is_arg(expr_ty e) { if (e->kind != Constant_kind) { - return 1; + return true; } PyObject *value = e->v.Constant.value; return (value == Py_None @@ -2849,19 +2818,18 @@ check_is_arg(expr_ty e) || value == Py_Ellipsis); } -/* Check operands of identity chacks ("is" and "is not"). +/* Check operands of identity checks ("is" and "is not"). Emit a warning if any operand is a constant except named singletons. - Return 0 on error. */ static int check_compare(struct compiler *c, expr_ty e) { Py_ssize_t i, n; - int left = check_is_arg(e->v.Compare.left); + bool left = check_is_arg(e->v.Compare.left); n = asdl_seq_LEN(e->v.Compare.ops); for (i = 0; i < n; i++) { cmpop_ty op = (cmpop_ty)asdl_seq_GET(e->v.Compare.ops, i); - int right = check_is_arg((expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); + bool right = check_is_arg((expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); if (op == Is || op == IsNot) { if (!right || !left) { const char *msg = (op == Is) @@ -2872,7 +2840,7 @@ check_compare(struct compiler *c, expr_ty e) } left = right; } - return 1; + return SUCCESS; } static int compiler_addcompare(struct compiler *c, location loc, @@ -2900,21 +2868,21 @@ static int compiler_addcompare(struct compiler *c, location loc, break; case Is: ADDOP_I(c, loc, IS_OP, 0); - return 1; + return SUCCESS; case IsNot: ADDOP_I(c, loc, IS_OP, 1); - return 1; + return SUCCESS; case In: ADDOP_I(c, loc, CONTAINS_OP, 0); - return 1; + return SUCCESS; case NotIn: ADDOP_I(c, loc, CONTAINS_OP, 1); - return 1; + return SUCCESS; default: Py_UNREACHABLE(); } ADDOP_I(c, loc, COMPARE_OP, cmp); - return 1; + return SUCCESS; } @@ -2941,43 +2909,36 @@ compiler_jump_if(struct compiler *c, location loc, next2 = new_next2; } for (i = 0; i < n; ++i) { - if (!compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)) { - return 0; - } - } - if (!compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)) { - return 0; + RETURN_IF_ERROR( + compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); } + RETURN_IF_ERROR( + compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); if (!SAME_LABEL(next2, next)) { USE_LABEL(c, next2); } - return 1; + return SUCCESS; } case IfExp_kind: { NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next2); - if (!compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)) { - return 0; - } - if (!compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)); + RETURN_IF_ERROR( + compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)); ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, next2); - if (!compiler_jump_if(c, loc, e->v.IfExp.orelse, next, cond)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); USE_LABEL(c, end); - return 1; + return SUCCESS; } case Compare_kind: { Py_ssize_t n = asdl_seq_LEN(e->v.Compare.ops) - 1; if (n > 0) { - if (!check_compare(c, e)) { - return 0; - } + RETURN_IF_ERROR(check_compare(c, e)); NEW_JUMP_TARGET_LABEL(c, cleanup); VISIT(c, expr, e->v.Compare.left); for (Py_ssize_t i = 0; i < n; i++) { @@ -3001,7 +2962,7 @@ compiler_jump_if(struct compiler *c, location loc, } USE_LABEL(c, end); - return 1; + return SUCCESS; } /* fallback to general implementation */ break; @@ -3014,7 +2975,7 @@ compiler_jump_if(struct compiler *c, location loc, /* general implementation */ VISIT(c, expr, e); ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); - return 1; + return SUCCESS; } static int @@ -3024,9 +2985,9 @@ compiler_ifexp(struct compiler *c, expr_ty e) NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next); - if (!compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); + VISIT(c, expr, e->v.IfExp.body); ADDOP_JUMP(c, NO_LOCATION, JUMP, end); @@ -3034,7 +2995,7 @@ compiler_ifexp(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.IfExp.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3046,8 +3007,7 @@ compiler_lambda(struct compiler *c, expr_ty e) arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); - if (!compiler_check_debug_args(c, args)) - return 0; + RETURN_IF_ERROR(compiler_check_debug_args(c, args)); location loc = LOC(e); funcflags = compiler_default_arguments(c, loc, args); @@ -3056,14 +3016,13 @@ compiler_lambda(struct compiler *c, expr_ty e) } _Py_DECLARE_STR(anon_lambda, "<lambda>"); - if (!compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, - (void *)e, e->lineno)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, + (void *)e, e->lineno)); + /* Make None the first constant, so the lambda can't have a docstring. */ - if (compiler_add_const(c, Py_None) < 0) - return 0; + RETURN_IF_ERROR(compiler_add_const(c, Py_None)); c->u->u_argcount = asdl_seq_LEN(args->args); c->u->u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); @@ -3081,18 +3040,18 @@ compiler_lambda(struct compiler *c, expr_ty e) compiler_exit_scope(c); if (co == NULL) { Py_DECREF(qualname); - return 0; + return ERROR; } - if (!compiler_make_closure(c, loc, co, funcflags, qualname)) { + if (compiler_make_closure(c, loc, co, funcflags, qualname) < 0) { Py_DECREF(qualname); Py_DECREF(co); - return 0; + return ERROR; } Py_DECREF(qualname); Py_DECREF(co); - return 1; + return SUCCESS; } static int @@ -3108,9 +3067,9 @@ compiler_if(struct compiler *c, stmt_ty s) else { next = end; } - if (!compiler_jump_if(c, LOC(s), s->v.If.test, next, 0)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, LOC(s), s->v.If.test, next, 0)); + VISIT_SEQ(c, stmt, s->v.If.body); if (asdl_seq_LEN(s->v.If.orelse)) { ADDOP_JUMP(c, NO_LOCATION, JUMP, end); @@ -3120,7 +3079,7 @@ compiler_if(struct compiler *c, stmt_ty s) } USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3132,9 +3091,8 @@ compiler_for(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup); NEW_JUMP_TARGET_LABEL(c, end); - if (!compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); + VISIT(c, expr, s->v.For.iter); ADDOP(c, loc, GET_ITER); @@ -3155,7 +3113,7 @@ compiler_for(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.For.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } @@ -3177,9 +3135,8 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, loc, GET_AITER); USE_LABEL(c, start); - if (!compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); + /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); @@ -3207,7 +3164,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.For.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3219,18 +3176,13 @@ compiler_while(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, anchor); USE_LABEL(c, loop); - if (!compiler_push_fblock(c, LOC(s), WHILE_LOOP, loop, end, NULL)) { - return 0; - } - if (!compiler_jump_if(c, LOC(s), s->v.While.test, anchor, 0)) { - return 0; - } + + RETURN_IF_ERROR(compiler_push_fblock(c, LOC(s), WHILE_LOOP, loop, end, NULL)); + RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); USE_LABEL(c, body); VISIT_SEQ(c, stmt, s->v.While.body); - if (!compiler_jump_if(c, LOC(s), s->v.While.test, body, 1)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, body, 1)); compiler_pop_fblock(c, WHILE_LOOP, loop); @@ -3240,7 +3192,7 @@ compiler_while(struct compiler *c, stmt_ty s) } USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3249,13 +3201,13 @@ compiler_return(struct compiler *c, stmt_ty s) location loc = LOC(s); int preserve_tos = ((s->v.Return.value != NULL) && (s->v.Return.value->kind != Constant_kind)); - if (c->u->u_ste->ste_type != FunctionBlock) + if (c->u->u_ste->ste_type != FunctionBlock) { return compiler_error(c, loc, "'return' outside function"); + } if (s->v.Return.value != NULL && c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) { - return compiler_error( - c, loc, "'return' with value in async generator"); + return compiler_error(c, loc, "'return' with value in async generator"); } if (preserve_tos) { @@ -3272,8 +3224,7 @@ compiler_return(struct compiler *c, stmt_ty s) ADDOP(c, loc, NOP); } - if (!compiler_unwind_fblock_stack(c, &loc, preserve_tos, NULL)) - return 0; + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, preserve_tos, NULL)); if (s->v.Return.value == NULL) { ADDOP_LOAD_CONST(c, loc, Py_None); } @@ -3282,7 +3233,7 @@ compiler_return(struct compiler *c, stmt_ty s) } ADDOP(c, loc, RETURN_VALUE); - return 1; + return SUCCESS; } static int @@ -3291,17 +3242,13 @@ compiler_break(struct compiler *c, location loc) struct fblockinfo *loop = NULL; /* Emit instruction with line number */ ADDOP(c, loc, NOP); - if (!compiler_unwind_fblock_stack(c, &loc, 0, &loop)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, 0, &loop)); if (loop == NULL) { return compiler_error(c, loc, "'break' outside loop"); } - if (!compiler_unwind_fblock(c, &loc, loop, 0)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock(c, &loc, loop, 0)); ADDOP_JUMP(c, loc, JUMP, loop->fb_exit); - return 1; + return SUCCESS; } static int @@ -3310,14 +3257,12 @@ compiler_continue(struct compiler *c, location loc) struct fblockinfo *loop = NULL; /* Emit instruction with line number */ ADDOP(c, loc, NOP); - if (!compiler_unwind_fblock_stack(c, &loc, 0, &loop)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, 0, &loop)); if (loop == NULL) { return compiler_error(c, loc, "'continue' not properly in loop"); } ADDOP_JUMP(c, loc, JUMP, loop->fb_block); - return 1; + return SUCCESS; } @@ -3376,11 +3321,12 @@ compiler_try_finally(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, end); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, FINALLY_TRY, body, end, s->v.Try.finalbody)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_TRY, body, end, + s->v.Try.finalbody)); + if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { - if (!compiler_try_except(c, s)) - return 0; + RETURN_IF_ERROR(compiler_try_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.Try.body); @@ -3397,8 +3343,8 @@ compiler_try_finally(struct compiler *c, stmt_ty s) loc = NO_LOCATION; ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); - if (!compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)); VISIT_SEQ(c, stmt, s->v.Try.finalbody); loc = location_of_last_executing_statement(s->v.Try.finalbody); compiler_pop_fblock(c, FINALLY_END, end); @@ -3409,7 +3355,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) POP_EXCEPT_AND_RERAISE(c, loc); USE_LABEL(c, exit); - return 1; + return SUCCESS; } static int @@ -3425,13 +3371,12 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, end); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, FINALLY_TRY, body, end, s->v.TryStar.finalbody)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_TRY, body, end, + s->v.TryStar.finalbody)); + if (s->v.TryStar.handlers && asdl_seq_LEN(s->v.TryStar.handlers)) { - if (!compiler_try_star_except(c, s)) { - return 0; - } + RETURN_IF_ERROR(compiler_try_star_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.TryStar.body); @@ -3448,9 +3393,9 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) loc = NO_LOCATION; ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); - if (!compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)); + VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); loc = location_of_last_executing_statement(s->v.Try.finalbody); @@ -3461,7 +3406,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) POP_EXCEPT_AND_RERAISE(c, loc); USE_LABEL(c, exit); - return 1; + return SUCCESS; } @@ -3507,8 +3452,8 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, except); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)); VISIT_SEQ(c, stmt, s->v.Try.body); compiler_pop_fblock(c, TRY_EXCEPT, body); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -3522,9 +3467,11 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); + /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, loc, EXCEPTION_HANDLER, NO_LABEL, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, EXCEPTION_HANDLER, NO_LABEL, NO_LABEL, NULL)); + for (i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( s->v.Try.handlers, i); @@ -3543,7 +3490,8 @@ compiler_try_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup_end); NEW_JUMP_TARGET_LABEL(c, cleanup_body); - compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store); + RETURN_IF_ERROR( + compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store)); /* try: @@ -3560,10 +3508,9 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, - NO_LABEL, handler->v.ExceptHandler.name)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, + NO_LABEL, handler->v.ExceptHandler.name)); /* second # body */ VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); @@ -3573,8 +3520,10 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP(c, NO_LOCATION, POP_BLOCK); ADDOP(c, NO_LOCATION, POP_EXCEPT); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); ADDOP_JUMP(c, NO_LOCATION, JUMP, end); /* except: */ @@ -3582,8 +3531,10 @@ compiler_try_except(struct compiler *c, stmt_ty s) /* name = None; del name; # artificial */ ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); ADDOP_I(c, NO_LOCATION, RERAISE, 1); } @@ -3593,8 +3544,10 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP(c, loc, POP_TOP); /* exc_value */ USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, + NO_LABEL, NULL)); + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -3612,7 +3565,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); USE_LABEL(c, end); - return 1; + return SUCCESS; } /* @@ -3679,9 +3632,8 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, except); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)); VISIT_SEQ(c, stmt, s->v.TryStar.body); compiler_pop_fblock(c, TRY_EXCEPT, body); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -3692,11 +3644,12 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); + /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, loc, EXCEPTION_GROUP_HANDLER, - NO_LABEL, NO_LABEL, "except handler")) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, EXCEPTION_GROUP_HANDLER, + NO_LABEL, NO_LABEL, "except handler")); + for (Py_ssize_t i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( s->v.TryStar.handlers, i); @@ -3736,7 +3689,8 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup_body); if (handler->v.ExceptHandler.name) { - compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store); + RETURN_IF_ERROR( + compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store)); } else { ADDOP(c, loc, POP_TOP); // match @@ -3756,9 +3710,9 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, NO_LABEL, handler->v.ExceptHandler.name)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, + NO_LABEL, handler->v.ExceptHandler.name)); /* second # body */ VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); @@ -3767,8 +3721,10 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP(c, NO_LOCATION, POP_BLOCK); if (handler->v.ExceptHandler.name) { ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); } ADDOP_JUMP(c, NO_LOCATION, JUMP, except); @@ -3778,8 +3734,10 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) /* name = None; del name; # artificial */ if (handler->v.ExceptHandler.name) { ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); } /* add exception raised to the res list */ @@ -3823,7 +3781,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.TryStar.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3857,19 +3815,22 @@ compiler_import_as(struct compiler *c, location loc, */ Py_ssize_t len = PyUnicode_GET_LENGTH(name); Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, len, 1); - if (dot == -2) - return 0; + if (dot == -2) { + return ERROR; + } if (dot != -1) { /* Consume the base module name to get the first attribute */ while (1) { Py_ssize_t pos = dot + 1; PyObject *attr; dot = PyUnicode_FindChar(name, '.', pos, len, 1); - if (dot == -2) - return 0; + if (dot == -2) { + return ERROR; + } attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); - if (!attr) - return 0; + if (!attr) { + return ERROR; + } ADDOP_N(c, loc, IMPORT_FROM, attr, names); if (dot == -1) { break; @@ -3877,11 +3838,9 @@ compiler_import_as(struct compiler *c, location loc, ADDOP_I(c, loc, SWAP, 2); ADDOP(c, loc, POP_TOP); } - if (!compiler_nameop(c, loc, asname, Store)) { - return 0; - } + RETURN_IF_ERROR(compiler_nameop(c, loc, asname, Store)); ADDOP(c, loc, POP_TOP); - return 1; + return SUCCESS; } return compiler_nameop(c, loc, asname, Store); } @@ -3910,8 +3869,7 @@ compiler_import(struct compiler *c, stmt_ty s) if (alias->asname) { r = compiler_import_as(c, loc, alias->name, alias->asname); - if (!r) - return r; + RETURN_IF_ERROR(r); } else { identifier tmp = alias->name; @@ -3919,18 +3877,18 @@ compiler_import(struct compiler *c, stmt_ty s) alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1); if (dot != -1) { tmp = PyUnicode_Substring(alias->name, 0, dot); - if (tmp == NULL) - return 0; + if (tmp == NULL) { + return ERROR; + } } r = compiler_nameop(c, loc, tmp, Store); if (dot != -1) { Py_DECREF(tmp); } - if (!r) - return r; + RETURN_IF_ERROR(r); } } - return 1; + return SUCCESS; } static int @@ -3942,7 +3900,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) PyObject *names = PyTuple_New(n); if (!names) { - return 0; + return ERROR; } /* build up the names */ @@ -3975,7 +3933,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') { assert(n == 1); ADDOP(c, LOC(s), IMPORT_STAR); - return 1; + return SUCCESS; } ADDOP_NAME(c, LOC(s), IMPORT_FROM, alias->name, names); @@ -3984,13 +3942,11 @@ compiler_from_import(struct compiler *c, stmt_ty s) store_name = alias->asname; } - if (!compiler_nameop(c, LOC(s), store_name, Store)) { - return 0; - } + RETURN_IF_ERROR(compiler_nameop(c, LOC(s), store_name, Store)); } /* remove imported module */ ADDOP(c, LOC(s), POP_TOP); - return 1; + return SUCCESS; } static int @@ -4003,19 +3959,15 @@ compiler_assert(struct compiler *c, stmt_ty s) PyTuple_Check(s->v.Assert.test->v.Constant.value) && PyTuple_Size(s->v.Assert.test->v.Constant.value) > 0)) { - if (!compiler_warn(c, LOC(s), "assertion is always true, " - "perhaps remove parentheses?")) - { - return 0; - } + RETURN_IF_ERROR( + compiler_warn(c, LOC(s), "assertion is always true, " + "perhaps remove parentheses?")); } if (c->c_optimize) { - return 1; + return SUCCESS; } NEW_JUMP_TARGET_LABEL(c, end); - if (!compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); ADDOP(c, LOC(s), LOAD_ASSERTION_ERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); @@ -4024,7 +3976,7 @@ compiler_assert(struct compiler *c, stmt_ty s) ADDOP_I(c, LOC(s), RAISE_VARARGS, 1); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -4033,18 +3985,18 @@ compiler_stmt_expr(struct compiler *c, location loc, expr_ty value) if (c->c_interactive && c->c_nestlevel <= 1) { VISIT(c, expr, value); ADDOP(c, loc, PRINT_EXPR); - return 1; + return SUCCESS; } if (value->kind == Constant_kind) { /* ignore constant statement */ ADDOP(c, loc, NOP); - return 1; + return SUCCESS; } VISIT(c, expr, value); ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ - return 1; + return SUCCESS; } static int @@ -4140,7 +4092,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) return compiler_async_for(c, s); } - return 1; + return SUCCESS; } static int @@ -4210,10 +4162,10 @@ addop_binary(struct compiler *c, location loc, operator_ty binop, default: PyErr_Format(PyExc_SystemError, "%s op %d should not be possible", inplace ? "inplace" : "binary", binop); - return 0; + return ERROR; } ADDOP_I(c, loc, BINARY_OP, oparg); - return 1; + return SUCCESS; } @@ -4224,7 +4176,7 @@ addop_yield(struct compiler *c, location loc) { } ADDOP_I(c, loc, YIELD_VALUE, 0); ADDOP_I(c, loc, RESUME, 1); - return 1; + return SUCCESS; } static int @@ -4242,12 +4194,14 @@ compiler_nameop(struct compiler *c, location loc, !_PyUnicode_EqualToASCIIString(name, "True") && !_PyUnicode_EqualToASCIIString(name, "False")); - if (forbidden_name(c, loc, name, ctx)) - return 0; + if (forbidden_name(c, loc, name, ctx)) { + return ERROR; + } mangled = _Py_Mangle(c->u->u_private, name); - if (!mangled) - return 0; + if (!mangled) { + return ERROR; + } op = 0; optype = OP_NAME; @@ -4297,7 +4251,7 @@ compiler_nameop(struct compiler *c, location loc, case Del: op = DELETE_FAST; break; } ADDOP_N(c, loc, op, mangled, varnames); - return 1; + return SUCCESS; case OP_GLOBAL: switch (ctx) { case Load: op = LOAD_GLOBAL; break; @@ -4318,7 +4272,7 @@ compiler_nameop(struct compiler *c, location loc, arg = dict_add_o(dict, mangled); Py_DECREF(mangled); if (arg < 0) { - return 0; + return ERROR; } if (op == LOAD_GLOBAL) { arg <<= 1; @@ -4353,7 +4307,7 @@ compiler_boolop(struct compiler *c, expr_ty e) VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n)); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -4365,7 +4319,7 @@ starunpack_helper(struct compiler *c, location loc, if (n > 2 && are_all_items_const(elts, 0, n)) { PyObject *folded = PyTuple_New(n); if (folded == NULL) { - return 0; + return ERROR; } PyObject *val; for (Py_ssize_t i = 0; i < n; i++) { @@ -4378,7 +4332,7 @@ starunpack_helper(struct compiler *c, location loc, if (add == SET_ADD) { Py_SETREF(folded, PyFrozenSet_New(folded)); if (folded == NULL) { - return 0; + return ERROR; } } ADDOP_I(c, loc, build, pushed); @@ -4388,7 +4342,7 @@ starunpack_helper(struct compiler *c, location loc, ADDOP(c, loc, LIST_TO_TUPLE); } } - return 1; + return SUCCESS; } int big = n+pushed > STACK_USE_GUIDELINE; @@ -4410,7 +4364,7 @@ starunpack_helper(struct compiler *c, location loc, } else { ADDOP_I(c, loc, build, n+pushed); } - return 1; + return SUCCESS; } int sequence_built = 0; if (big) { @@ -4438,7 +4392,7 @@ starunpack_helper(struct compiler *c, location loc, if (tuple) { ADDOP(c, loc, LIST_TO_TUPLE); } - return 1; + return SUCCESS; } static int @@ -4450,10 +4404,11 @@ unpack_helper(struct compiler *c, location loc, asdl_expr_seq *elts) expr_ty elt = asdl_seq_GET(elts, i); if (elt->kind == Starred_kind && !seen_star) { if ((i >= (1 << 8)) || - (n-i-1 >= (INT_MAX >> 8))) + (n-i-1 >= (INT_MAX >> 8))) { return compiler_error(c, loc, "too many expressions in " "star-unpacking assignment"); + } ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); seen_star = 1; } @@ -4465,19 +4420,19 @@ unpack_helper(struct compiler *c, location loc, asdl_expr_seq *elts) if (!seen_star) { ADDOP_I(c, loc, UNPACK_SEQUENCE, n); } - return 1; + return SUCCESS; } static int assignment_helper(struct compiler *c, location loc, asdl_expr_seq *elts) { Py_ssize_t n = asdl_seq_LEN(elts); - RETURN_IF_FALSE(unpack_helper(c, loc, elts)); + RETURN_IF_ERROR(unpack_helper(c, loc, elts)); for (Py_ssize_t i = 0; i < n; i++) { expr_ty elt = asdl_seq_GET(elts, i); VISIT(c, expr, elt->kind != Starred_kind ? elt : elt->v.Starred.value); } - return 1; + return SUCCESS; } static int @@ -4492,9 +4447,10 @@ compiler_list(struct compiler *c, expr_ty e) return starunpack_helper(c, loc, elts, 0, BUILD_LIST, LIST_APPEND, LIST_EXTEND, 0); } - else + else { VISIT_SEQ(c, expr, elts); - return 1; + } + return SUCCESS; } static int @@ -4509,9 +4465,10 @@ compiler_tuple(struct compiler *c, expr_ty e) return starunpack_helper(c, loc, elts, 0, BUILD_LIST, LIST_APPEND, LIST_EXTEND, 1); } - else + else { VISIT_SEQ(c, expr, elts); - return 1; + } + return SUCCESS; } static int @@ -4522,16 +4479,16 @@ compiler_set(struct compiler *c, expr_ty e) BUILD_SET, SET_ADD, SET_UPDATE, 0); } -static int +static bool are_all_items_const(asdl_expr_seq *seq, Py_ssize_t begin, Py_ssize_t end) { - Py_ssize_t i; - for (i = begin; i < end; i++) { + for (Py_ssize_t i = begin; i < end; i++) { expr_ty key = (expr_ty)asdl_seq_GET(seq, i); - if (key == NULL || key->kind != Constant_kind) - return 0; + if (key == NULL || key->kind != Constant_kind) { + return false; + } } - return 1; + return true; } static int @@ -4547,7 +4504,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end } keys = PyTuple_New(n); if (keys == NULL) { - return 0; + return SUCCESS; } for (i = begin; i < end; i++) { key = ((expr_ty)asdl_seq_GET(e->v.Dict.keys, i))->v.Constant.value; @@ -4555,7 +4512,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end } ADDOP_LOAD_CONST_NEW(c, loc, keys); ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return 1; + return SUCCESS; } if (big) { ADDOP_I(c, loc, BUILD_MAP, 0); @@ -4570,7 +4527,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end if (!big) { ADDOP_I(c, loc, BUILD_MAP, n); } - return 1; + return SUCCESS; } static int @@ -4587,9 +4544,7 @@ compiler_dict(struct compiler *c, expr_ty e) is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; if (is_unpacking) { if (elements) { - if (!compiler_subdict(c, e, i - elements, i)) { - return 0; - } + RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4605,9 +4560,7 @@ compiler_dict(struct compiler *c, expr_ty e) } else { if (elements*2 > STACK_USE_GUIDELINE) { - if (!compiler_subdict(c, e, i - elements, i + 1)) { - return 0; - } + RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i + 1)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4620,9 +4573,7 @@ compiler_dict(struct compiler *c, expr_ty e) } } if (elements) { - if (!compiler_subdict(c, e, n - elements, n)) { - return 0; - } + RETURN_IF_ERROR(compiler_subdict(c, e, n - elements, n)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4631,7 +4582,7 @@ compiler_dict(struct compiler *c, expr_ty e) if (!have_dict) { ADDOP_I(c, loc, BUILD_MAP, 0); } - return 1; + return SUCCESS; } static int @@ -4640,9 +4591,7 @@ compiler_compare(struct compiler *c, expr_ty e) location loc = LOC(e); Py_ssize_t i, n; - if (!check_compare(c, e)) { - return 0; - } + RETURN_IF_ERROR(check_compare(c, e)); VISIT(c, expr, e->v.Compare.left); assert(asdl_seq_LEN(e->v.Compare.ops) > 0); n = asdl_seq_LEN(e->v.Compare.ops) - 1; @@ -4671,7 +4620,7 @@ compiler_compare(struct compiler *c, expr_ty e) USE_LABEL(c, end); } - return 1; + return SUCCESS; } static PyTypeObject * @@ -4724,7 +4673,7 @@ check_caller(struct compiler *c, expr_ty e) infer_type(e)->tp_name); } default: - return 1; + return SUCCESS; } } @@ -4740,7 +4689,7 @@ check_subscripter(struct compiler *c, expr_ty e) PyLong_Check(v) || PyFloat_Check(v) || PyComplex_Check(v) || PyAnySet_Check(v))) { - return 1; + return SUCCESS; } /* fall through */ case Set_kind: @@ -4753,7 +4702,7 @@ check_subscripter(struct compiler *c, expr_ty e) infer_type(e)->tp_name); } default: - return 1; + return SUCCESS; } } @@ -4766,14 +4715,14 @@ check_index(struct compiler *c, expr_ty e, expr_ty s) if (index_type == NULL || PyType_FastSubclass(index_type, Py_TPFLAGS_LONG_SUBCLASS) || index_type == &PySlice_Type) { - return 1; + return SUCCESS; } switch (e->kind) { case Constant_kind: v = e->v.Constant.value; if (!(PyUnicode_Check(v) || PyBytes_Check(v) || PyTuple_Check(v))) { - return 1; + return SUCCESS; } /* fall through */ case Tuple_kind: @@ -4789,7 +4738,7 @@ check_index(struct compiler *c, expr_ty e, expr_ty s) index_type->tp_name); } default: - return 1; + return SUCCESS; } } @@ -4838,7 +4787,7 @@ update_start_location_to_match_attr(struct compiler *c, location loc, return loc; } -// Return 1 if the method call was optimized, -1 if not, and 0 on error. +// Return 1 if the method call was optimized, 0 if not, and -1 on error. static int maybe_optimize_method_call(struct compiler *c, expr_ty e) { @@ -4849,32 +4798,32 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) /* Check that the call node is an attribute access */ if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load) { - return -1; + return 0; } /* Check that the base object is not something that is imported */ if (is_import_originated(c, meth->v.Attribute.value)) { - return -1; + return 0; } /* Check that there aren't too many arguments */ argsl = asdl_seq_LEN(args); kwdsl = asdl_seq_LEN(kwds); if (argsl + kwdsl + (kwdsl != 0) >= STACK_USE_GUIDELINE) { - return -1; + return 0; } /* Check that there are no *varargs types of arguments. */ for (i = 0; i < argsl; i++) { expr_ty elt = asdl_seq_GET(args, i); if (elt->kind == Starred_kind) { - return -1; + return 0; } } for (i = 0; i < kwdsl; i++) { keyword_ty kw = asdl_seq_GET(kwds, i); if (kw->arg == NULL) { - return -1; + return 0; } } /* Alright, we can optimize the code. */ @@ -4886,9 +4835,8 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) if (kwdsl) { VISIT_SEQ(c, keyword, kwds); - if (!compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)) { - return 0; - }; + RETURN_IF_ERROR( + compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)); } loc = update_start_location_to_match_attr(c, LOC(e), meth); ADDOP_I(c, loc, CALL, argsl + kwdsl); @@ -4906,32 +4854,31 @@ validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) } location loc = LOC(key); if (forbidden_name(c, loc, key->arg, Store)) { - return -1; + return ERROR; } for (Py_ssize_t j = i + 1; j < nkeywords; j++) { keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { compiler_error(c, LOC(other), "keyword argument repeated: %U", key->arg); - return -1; + return ERROR; } } } - return 0; + return SUCCESS; } static int compiler_call(struct compiler *c, expr_ty e) { - if (validate_keywords(c, e->v.Call.keywords) == -1) { - return 0; - } + RETURN_IF_ERROR(validate_keywords(c, e->v.Call.keywords)); int ret = maybe_optimize_method_call(c, e); - if (ret >= 0) { - return ret; + if (ret < 0) { + return ERROR; } - if (!check_caller(c, e->v.Call.func)) { - return 0; + if (ret == 1) { + return SUCCESS; } + RETURN_IF_ERROR(check_caller(c, e->v.Call.func)); location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); VISIT(c, expr, e->v.Call.func); @@ -4963,7 +4910,7 @@ compiler_joined_str(struct compiler *c, expr_ty e) ADDOP_I(c, loc, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); } } - return 1; + return SUCCESS; } /* Used to implement f-strings. Format a single value. */ @@ -4998,7 +4945,7 @@ compiler_formatted_value(struct compiler *c, expr_ty e) default: PyErr_Format(PyExc_SystemError, "Unrecognized conversion character %d", conversion); - return 0; + return ERROR; } if (e->v.FormattedValue.format_spec) { /* Evaluate the format spec, and update our opcode arg. */ @@ -5010,7 +4957,7 @@ compiler_formatted_value(struct compiler *c, expr_ty e) location loc = LOC(e); ADDOP_I(c, loc, FORMAT_VALUE, oparg); - return 1; + return SUCCESS; } static int @@ -5030,7 +4977,7 @@ compiler_subkwargs(struct compiler *c, location loc, } keys = PyTuple_New(n); if (keys == NULL) { - return 0; + return ERROR; } for (i = begin; i < end; i++) { key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg; @@ -5038,7 +4985,7 @@ compiler_subkwargs(struct compiler *c, location loc, } ADDOP_LOAD_CONST_NEW(c, loc, keys); ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return 1; + return SUCCESS; } if (big) { ADDOP_I(c, NO_LOCATION, BUILD_MAP, 0); @@ -5054,12 +5001,11 @@ compiler_subkwargs(struct compiler *c, location loc, if (!big) { ADDOP_I(c, loc, BUILD_MAP, n); } - return 1; + return SUCCESS; } /* Used by compiler_call_helper and maybe_optimize_method_call to emit * KW_NAMES before CALL. - * Returns 1 on success, 0 on error. */ static int compiler_call_simple_kw_helper(struct compiler *c, location loc, @@ -5068,7 +5014,7 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, PyObject *names; names = PyTuple_New(nkwelts); if (names == NULL) { - return 0; + return ERROR; } for (int i = 0; i < nkwelts; i++) { keyword_ty kw = asdl_seq_GET(keywords, i); @@ -5076,11 +5022,11 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, } Py_ssize_t arg = compiler_add_const(c, names); if (arg < 0) { - return 0; + return ERROR; } Py_DECREF(names); ADDOP_I(c, loc, KW_NAMES, arg); - return 1; + return SUCCESS; } @@ -5093,9 +5039,7 @@ compiler_call_helper(struct compiler *c, location loc, { Py_ssize_t i, nseen, nelts, nkwelts; - if (validate_keywords(c, keywords) == -1) { - return 0; - } + RETURN_IF_ERROR(validate_keywords(c, keywords)); nelts = asdl_seq_LEN(args); nkwelts = asdl_seq_LEN(keywords); @@ -5124,12 +5068,11 @@ compiler_call_helper(struct compiler *c, location loc, } if (nkwelts) { VISIT_SEQ(c, keyword, keywords); - if (!compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)) { - return 0; - }; + RETURN_IF_ERROR( + compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)); } ADDOP_I(c, loc, CALL, n + nelts + nkwelts); - return 1; + return SUCCESS; ex_call: @@ -5138,8 +5081,8 @@ compiler_call_helper(struct compiler *c, location loc, VISIT(c, expr, ((expr_ty)asdl_seq_GET(args, 0))->v.Starred.value); } else if (starunpack_helper(c, loc, args, n, BUILD_LIST, - LIST_APPEND, LIST_EXTEND, 1) == 0) { - return 0; + LIST_APPEND, LIST_EXTEND, 1) < 0) { + return ERROR; } /* Then keyword arguments */ if (nkwelts) { @@ -5152,9 +5095,7 @@ compiler_call_helper(struct compiler *c, location loc, if (kw->arg == NULL) { /* A keyword argument unpacking. */ if (nseen) { - if (!compiler_subkwargs(c, loc, keywords, i - nseen, i)) { - return 0; - } + RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, i - nseen, i)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5174,9 +5115,7 @@ compiler_call_helper(struct compiler *c, location loc, } if (nseen) { /* Pack up any trailing keyword arguments. */ - if (!compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)) { - return 0; - } + RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5185,7 +5124,7 @@ compiler_call_helper(struct compiler *c, location loc, assert(have_dict); } ADDOP_I(c, loc, CALL_FUNCTION_EX, nkwelts > 0); - return 1; + return SUCCESS; } @@ -5280,17 +5219,14 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - if (!compiler_jump_if(c, loc, e, if_cleanup, 0)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); } if (++gen_index < asdl_seq_LEN(generators)) { - if (!compiler_comprehension_generator(c, loc, - generators, gen_index, depth, - elt, val, type)) { - return 0; - } + RETURN_IF_ERROR( + compiler_comprehension_generator(c, loc, + generators, gen_index, depth, + elt, val, type)); } location elt_loc = LOC(elt); @@ -5324,7 +5260,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); break; default: - return 0; + return ERROR; } } @@ -5336,7 +5272,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, ADDOP(c, NO_LOCATION, END_FOR); } - return 1; + return SUCCESS; } static int @@ -5365,10 +5301,9 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, USE_LABEL(c, start); /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, loc, ASYNC_COMPREHENSION_GENERATOR, - start, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, ASYNC_COMPREHENSION_GENERATOR, + start, NO_LABEL, NULL)); ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); @@ -5380,18 +5315,15 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - if (!compiler_jump_if(c, loc, e, if_cleanup, 0)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); } depth++; if (++gen_index < asdl_seq_LEN(generators)) { - if (!compiler_comprehension_generator(c, loc, - generators, gen_index, depth, - elt, val, type)) { - return 0; - } + RETURN_IF_ERROR( + compiler_comprehension_generator(c, loc, + generators, gen_index, depth, + elt, val, type)); } location elt_loc = LOC(elt); @@ -5424,7 +5356,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); break; default: - return 0; + return ERROR; } } @@ -5437,7 +5369,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, ADDOP(c, loc, END_ASYNC_FOR); - return 1; + return SUCCESS; } static int @@ -5453,8 +5385,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); outermost = (comprehension_ty) asdl_seq_GET(generators, 0); - if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno)) + if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, + (void *)e, e->lineno) < 0) { goto error; } @@ -5493,8 +5425,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP_I(c, loc, op, 0); } - if (!compiler_comprehension_generator(c, loc, generators, 0, 0, - elt, val, type)) { + if (compiler_comprehension_generator(c, loc, generators, 0, 0, + elt, val, type) < 0) { goto error_in_scope; } @@ -5502,7 +5434,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP(c, LOC(e), RETURN_VALUE); } if (type == COMP_GENEXP) { - if (!wrap_in_stopiteration_handler(c)) { + if (wrap_in_stopiteration_handler(c) < 0) { goto error_in_scope; } } @@ -5513,11 +5445,12 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, if (is_top_level_await && is_async_generator){ c->u->u_ste->ste_coroutine = 1; } - if (co == NULL) + if (co == NULL) { goto error; + } loc = LOC(e); - if (!compiler_make_closure(c, loc, co, 0, qualname)) { + if (compiler_make_closure(c, loc, co, 0, qualname) < 0) { goto error; } Py_DECREF(qualname); @@ -5540,13 +5473,13 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADD_YIELD_FROM(c, loc, 1); } - return 1; + return SUCCESS; error_in_scope: compiler_exit_scope(c); error: Py_XDECREF(qualname); Py_XDECREF(co); - return 0; + return ERROR; } static int @@ -5595,7 +5528,7 @@ static int compiler_visit_keyword(struct compiler *c, keyword_ty k) { VISIT(c, expr, k->value); - return 1; + return SUCCESS; } @@ -5618,7 +5551,7 @@ compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); USE_LABEL(c, exit); - return 1; + return SUCCESS; } /* @@ -5675,15 +5608,13 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* SETUP_WITH pushes a finally block. */ USE_LABEL(c, block); - if (!compiler_push_fblock(c, loc, ASYNC_WITH, block, final, s)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, ASYNC_WITH, block, final, s)); if (item->optional_vars) { VISIT(c, expr, item->optional_vars); } else { - /* Discard result from context.__aenter__() */ + /* Discard result from context.__aenter__() */ ADDOP(c, loc, POP_TOP); } @@ -5692,8 +5623,8 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* BLOCK code */ VISIT_SEQ(c, stmt, s->v.AsyncWith.body) } - else if (!compiler_async_with(c, s, pos)) { - return 0; + else { + RETURN_IF_ERROR(compiler_async_with(c, s, pos)); } compiler_pop_fblock(c, ASYNC_WITH, block); @@ -5704,8 +5635,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - if(!compiler_call_exit_with_nones(c, loc)) - return 0; + RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -5723,10 +5653,10 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); - compiler_with_except_finish(c, cleanup); + RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); USE_LABEL(c, exit); - return 1; + return SUCCESS; } @@ -5772,9 +5702,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* SETUP_WITH pushes a finally block. */ USE_LABEL(c, block); - if (!compiler_push_fblock(c, loc, WITH, block, final, s)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, WITH, block, final, s)); if (item->optional_vars) { VISIT(c, expr, item->optional_vars); @@ -5785,11 +5713,13 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) } pos++; - if (pos == asdl_seq_LEN(s->v.With.items)) + if (pos == asdl_seq_LEN(s->v.With.items)) { /* BLOCK code */ VISIT_SEQ(c, stmt, s->v.With.body) - else if (!compiler_with(c, s, pos)) - return 0; + } + else { + RETURN_IF_ERROR(compiler_with(c, s, pos)); + } ADDOP(c, NO_LOCATION, POP_BLOCK); compiler_pop_fblock(c, WITH, block); @@ -5800,8 +5730,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) * call __exit__(None, None, None) */ loc = LOC(s); - if (!compiler_call_exit_with_nones(c, loc)) - return 0; + RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); ADDOP(c, loc, POP_TOP); ADDOP_JUMP(c, loc, JUMP, exit); @@ -5811,10 +5740,10 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); - compiler_with_except_finish(c, cleanup); + RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); USE_LABEL(c, exit); - return 1; + return SUCCESS; } static int @@ -5855,8 +5784,9 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case DictComp_kind: return compiler_dictcomp(c, e); case Yield_kind: - if (c->u->u_ste->ste_type != FunctionBlock) + if (c->u->u_ste->ste_type != FunctionBlock) { return compiler_error(c, loc, "'yield' outside function"); + } if (e->v.Yield.value) { VISIT(c, expr, e->v.Yield.value); } @@ -5866,12 +5796,12 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP_YIELD(c, loc); break; case YieldFrom_kind: - if (c->u->u_ste->ste_type != FunctionBlock) + if (c->u->u_ste->ste_type != FunctionBlock) { return compiler_error(c, loc, "'yield' outside function"); - - if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) + } + if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) { return compiler_error(c, loc, "'yield from' inside async function"); - + } VISIT(c, expr, e->v.YieldFrom.value); ADDOP(c, loc, GET_YIELD_FROM_ITER); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5884,7 +5814,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) } if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){ + c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) { return compiler_error(c, loc, "'await' outside async function"); } } @@ -5916,7 +5846,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) break; case Store: if (forbidden_name(c, loc, e->v.Attribute.attr, e->v.Attribute.ctx)) { - return 0; + return ERROR; } ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); break; @@ -5942,9 +5872,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case Slice_kind: { int n = compiler_slice(c, e); - if (n == 0) { - return 0; - } + RETURN_IF_ERROR(n); ADDOP_I(c, loc, BUILD_SLICE, n); break; } @@ -5956,7 +5884,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case Tuple_kind: return compiler_tuple(c, e); } - return 1; + return SUCCESS; } static int @@ -5991,9 +5919,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) case Subscript_kind: VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice)) { - if (!compiler_slice(c, e->v.Subscript.slice)) { - return 0; - } + RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); @@ -6007,14 +5933,13 @@ compiler_augassign(struct compiler *c, stmt_ty s) } break; case Name_kind: - if (!compiler_nameop(c, loc, e->v.Name.id, Load)) - return 0; + RETURN_IF_ERROR(compiler_nameop(c, loc, e->v.Name.id, Load)); break; default: PyErr_Format(PyExc_SystemError, "invalid node type (%d) for augmented assignment", e->kind); - return 0; + return ERROR; } loc = LOC(s); @@ -6048,7 +5973,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) default: Py_UNREACHABLE(); } - return 1; + return SUCCESS; } static int @@ -6056,7 +5981,7 @@ check_ann_expr(struct compiler *c, expr_ty e) { VISIT(c, expr, e); ADDOP(c, LOC(e), POP_TOP); - return 1; + return SUCCESS; } static int @@ -6065,7 +5990,7 @@ check_annotation(struct compiler *c, stmt_ty s) /* Annotations of complex targets does not produce anything under annotations future */ if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - return 1; + return SUCCESS; } /* Annotations are only evaluated in a module or class. */ @@ -6073,7 +5998,7 @@ check_annotation(struct compiler *c, stmt_ty s) c->u->u_scope_type == COMPILER_SCOPE_CLASS) { return check_ann_expr(c, s->v.AnnAssign.annotation); } - return 1; + return SUCCESS; } static int @@ -6082,26 +6007,24 @@ check_ann_subscr(struct compiler *c, expr_ty e) /* We check that everything in a subscript is defined at runtime. */ switch (e->kind) { case Slice_kind: - if (e->v.Slice.lower && !check_ann_expr(c, e->v.Slice.lower)) { - return 0; + if (e->v.Slice.lower && check_ann_expr(c, e->v.Slice.lower) < 0) { + return ERROR; } - if (e->v.Slice.upper && !check_ann_expr(c, e->v.Slice.upper)) { - return 0; + if (e->v.Slice.upper && check_ann_expr(c, e->v.Slice.upper) < 0) { + return ERROR; } - if (e->v.Slice.step && !check_ann_expr(c, e->v.Slice.step)) { - return 0; + if (e->v.Slice.step && check_ann_expr(c, e->v.Slice.step) < 0) { + return ERROR; } - return 1; + return SUCCESS; case Tuple_kind: { /* extended slice */ asdl_expr_seq *elts = e->v.Tuple.elts; Py_ssize_t i, n = asdl_seq_LEN(elts); for (i = 0; i < n; i++) { - if (!check_ann_subscr(c, asdl_seq_GET(elts, i))) { - return 0; - } + RETURN_IF_ERROR(check_ann_subscr(c, asdl_seq_GET(elts, i))); } - return 1; + return SUCCESS; } default: return check_ann_expr(c, e); @@ -6124,8 +6047,9 @@ compiler_annassign(struct compiler *c, stmt_ty s) } switch (targ->kind) { case Name_kind: - if (forbidden_name(c, loc, targ->v.Name.id, Store)) - return 0; + if (forbidden_name(c, loc, targ->v.Name.id, Store)) { + return ERROR; + } /* If we have a simple name in a module or class, store annotation. */ if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || @@ -6143,31 +6067,32 @@ compiler_annassign(struct compiler *c, stmt_ty s) } break; case Attribute_kind: - if (forbidden_name(c, loc, targ->v.Attribute.attr, Store)) - return 0; + if (forbidden_name(c, loc, targ->v.Attribute.attr, Store)) { + return ERROR; + } if (!s->v.AnnAssign.value && - !check_ann_expr(c, targ->v.Attribute.value)) { - return 0; + check_ann_expr(c, targ->v.Attribute.value) < 0) { + return ERROR; } break; case Subscript_kind: if (!s->v.AnnAssign.value && - (!check_ann_expr(c, targ->v.Subscript.value) || - !check_ann_subscr(c, targ->v.Subscript.slice))) { - return 0; + (check_ann_expr(c, targ->v.Subscript.value) < 0 || + check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { + return ERROR; } break; default: PyErr_Format(PyExc_SystemError, "invalid node type (%d) for annotated assignment", targ->kind); - return 0; + return ERROR; } /* Annotation is evaluated last. */ - if (!s->v.AnnAssign.simple && !check_annotation(c, s)) { - return 0; + if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + return ERROR; } - return 1; + return SUCCESS; } /* Raises a SyntaxError and returns 0. @@ -6183,7 +6108,7 @@ compiler_error(struct compiler *c, location loc, PyObject *msg = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg == NULL) { - return 0; + return ERROR; } PyObject *loc_obj = PyErr_ProgramTextObject(c->c_filename, loc.lineno); if (loc_obj == NULL) { @@ -6200,7 +6125,7 @@ compiler_error(struct compiler *c, location loc, exit: Py_DECREF(loc_obj); Py_XDECREF(args); - return 0; + return ERROR; } /* Emits a SyntaxWarning and returns 1 on success. @@ -6216,7 +6141,7 @@ compiler_warn(struct compiler *c, location loc, PyObject *msg = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg == NULL) { - return 0; + return ERROR; } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, c->c_filename, loc.lineno, NULL, NULL) < 0) @@ -6229,10 +6154,10 @@ compiler_warn(struct compiler *c, location loc, compiler_error(c, loc, PyUnicode_AsUTF8(msg)); } Py_DECREF(msg); - return 0; + return ERROR; } Py_DECREF(msg); - return 1; + return SUCCESS; } static int @@ -6243,19 +6168,13 @@ compiler_subscript(struct compiler *c, expr_ty e) int op = 0; if (ctx == Load) { - if (!check_subscripter(c, e->v.Subscript.value)) { - return 0; - } - if (!check_index(c, e->v.Subscript.value, e->v.Subscript.slice)) { - return 0; - } + RETURN_IF_ERROR(check_subscripter(c, e->v.Subscript.value)); + RETURN_IF_ERROR(check_index(c, e->v.Subscript.value, e->v.Subscript.slice)); } VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice) && ctx != Del) { - if (!compiler_slice(c, e->v.Subscript.slice)) { - return 0; - } + RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); if (ctx == Load) { ADDOP(c, loc, BINARY_SLICE); } @@ -6274,11 +6193,11 @@ compiler_subscript(struct compiler *c, expr_ty e) assert(op); ADDOP(c, loc, op); } - return 1; + return SUCCESS; } /* Returns the number of the values emitted, - * thus are needed to build the slice, or 0 if there is an error. */ + * thus are needed to build the slice, or -1 if there is an error. */ static int compiler_slice(struct compiler *c, expr_ty s) { @@ -6336,20 +6255,20 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) { Py_ssize_t size = n + 1; if (size <= pc->fail_pop_size) { - return 1; + return SUCCESS; } Py_ssize_t needed = sizeof(jump_target_label) * size; jump_target_label *resized = PyObject_Realloc(pc->fail_pop, needed); if (resized == NULL) { PyErr_NoMemory(); - return 0; + return ERROR; } pc->fail_pop = resized; while (pc->fail_pop_size < size) { NEW_JUMP_TARGET_LABEL(c, new_block); pc->fail_pop[pc->fail_pop_size++] = new_block; } - return 1; + return SUCCESS; } // Use op to jump to the correct fail_pop block. @@ -6360,9 +6279,9 @@ jump_to_fail_pop(struct compiler *c, location loc, // Pop any items on the top of the stack, plus any objects we were going to // capture on success: Py_ssize_t pops = pc->on_top + PyList_GET_SIZE(pc->stores); - RETURN_IF_FALSE(ensure_fail_pop(c, pc, pops)); + RETURN_IF_ERROR(ensure_fail_pop(c, pc, pops)); ADDOP_JUMP(c, loc, op, pc->fail_pop[pops]); - return 1; + return SUCCESS; } // Build all of the fail_pop blocks and reset fail_pop. @@ -6372,21 +6291,21 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, { if (!pc->fail_pop_size) { assert(pc->fail_pop == NULL); - return 1; + return SUCCESS; } while (--pc->fail_pop_size) { USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); - if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc)) { + if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; - return 0; + return ERROR; } } USE_LABEL(c, pc->fail_pop[0]); PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; - return 1; + return SUCCESS; } static int @@ -6403,7 +6322,7 @@ pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) while (1 < count) { ADDOP_I(c, loc, SWAP, count--); } - return 1; + return SUCCESS; } static int @@ -6412,23 +6331,22 @@ pattern_helper_store_name(struct compiler *c, location loc, { if (n == NULL) { ADDOP(c, loc, POP_TOP); - return 1; + return SUCCESS; } if (forbidden_name(c, loc, n, Store)) { - return 0; + return ERROR; } // Can't assign to the same name twice: int duplicate = PySequence_Contains(pc->stores, n); - if (duplicate < 0) { - return 0; - } + RETURN_IF_ERROR(duplicate); if (duplicate) { return compiler_error_duplicate_store(c, loc, n); } // Rotate this object underneath any items we need to preserve: Py_ssize_t rotations = pc->on_top + PyList_GET_SIZE(pc->stores) + 1; - RETURN_IF_FALSE(pattern_helper_rotate(c, loc, rotations)); - return !PyList_Append(pc->stores, n); + RETURN_IF_ERROR(pattern_helper_rotate(c, loc, rotations)); + RETURN_IF_ERROR(PyList_Append(pc->stores, n)); + return SUCCESS; } @@ -6442,10 +6360,11 @@ pattern_unpack_helper(struct compiler *c, location loc, pattern_ty elt = asdl_seq_GET(elts, i); if (elt->kind == MatchStar_kind && !seen_star) { if ((i >= (1 << 8)) || - (n-i-1 >= (INT_MAX >> 8))) + (n-i-1 >= (INT_MAX >> 8))) { return compiler_error(c, loc, "too many expressions in " "star-unpacking sequence pattern"); + } ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); seen_star = 1; } @@ -6457,7 +6376,7 @@ pattern_unpack_helper(struct compiler *c, location loc, if (!seen_star) { ADDOP_I(c, loc, UNPACK_SEQUENCE, n); } - return 1; + return SUCCESS; } static int @@ -6465,7 +6384,7 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, asdl_pattern_seq *patterns, Py_ssize_t star, pattern_context *pc) { - RETURN_IF_FALSE(pattern_unpack_helper(c, loc, patterns)); + RETURN_IF_ERROR(pattern_unpack_helper(c, loc, patterns)); Py_ssize_t size = asdl_seq_LEN(patterns); // We've now got a bunch of new subjects on the stack. They need to remain // there after each subpattern match: @@ -6474,9 +6393,9 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, // One less item to keep track of each time we loop through: pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } - return 1; + return SUCCESS; } // Like pattern_helper_sequence_unpack, but uses BINARY_SUBSCR instead of @@ -6511,12 +6430,12 @@ pattern_helper_sequence_subscr(struct compiler *c, location loc, ADDOP_BINARY(c, loc, Sub); } ADDOP(c, loc, BINARY_SUBSCR); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } // Pop the subject, we're done with it: pc->on_top--; ADDOP(c, loc, POP_TOP); - return 1; + return SUCCESS; } // Like compiler_pattern, but turn off checks for irrefutability. @@ -6526,9 +6445,9 @@ compiler_pattern_subpattern(struct compiler *c, { int allow_irrefutable = pc->allow_irrefutable; pc->allow_irrefutable = 1; - RETURN_IF_FALSE(compiler_pattern(c, p, pc)); + RETURN_IF_ERROR(compiler_pattern(c, p, pc)); pc->allow_irrefutable = allow_irrefutable; - return 1; + return SUCCESS; } static int @@ -6550,19 +6469,20 @@ compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to make a copy for (possibly) storing later: pc->on_top++; ADDOP_I(c, LOC(p), COPY, 1); - RETURN_IF_FALSE(compiler_pattern(c, p->v.MatchAs.pattern, pc)); + RETURN_IF_ERROR(compiler_pattern(c, p->v.MatchAs.pattern, pc)); // Success! Store it: pc->on_top--; - RETURN_IF_FALSE(pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); - return 1; + RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); + return SUCCESS; } static int compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchStar_kind); - RETURN_IF_FALSE(pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); - return 1; + RETURN_IF_ERROR( + pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); + return SUCCESS; } static int @@ -6575,18 +6495,18 @@ validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_ identifier attr = ((identifier)asdl_seq_GET(attrs, i)); location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); if (forbidden_name(c, loc, attr, Store)) { - return -1; + return ERROR; } for (Py_ssize_t j = i + 1; j < nattrs; j++) { identifier other = ((identifier)asdl_seq_GET(attrs, j)); if (!PyUnicode_Compare(attr, other)) { location loc = LOC((pattern_ty) asdl_seq_GET(patterns, j)); compiler_error(c, loc, "attribute name repeated in class pattern: %U", attr); - return -1; + return ERROR; } } } - return 0; + return SUCCESS; } static int @@ -6610,11 +6530,13 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) return compiler_error(c, LOC(p), e, p->v.MatchClass.cls); } if (nattrs) { - RETURN_IF_FALSE(!validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); + RETURN_IF_ERROR(validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); } VISIT(c, expr, p->v.MatchClass.cls); - PyObject *attr_names; - RETURN_IF_FALSE(attr_names = PyTuple_New(nattrs)); + PyObject *attr_names = PyTuple_New(nattrs); + if (attr_names == NULL) { + return ERROR; + } Py_ssize_t i; for (i = 0; i < nattrs; i++) { PyObject *name = asdl_seq_GET(kwd_attrs, i); @@ -6627,7 +6549,7 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) ADDOP_I(c, LOC(p), IS_OP, 1); // TOS is now a tuple of (nargs + nattrs) attributes (or None): pc->on_top++; - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, nargs + nattrs); pc->on_top += nargs + nattrs - 1; for (i = 0; i < nargs + nattrs; i++) { @@ -6645,10 +6567,10 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) ADDOP(c, LOC(p), POP_TOP); continue; } - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } // Success! Pop the tuple of attributes: - return 1; + return SUCCESS; } static int @@ -6671,19 +6593,19 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, // We need to keep the subject on top during the mapping and length checks: pc->on_top++; ADDOP(c, LOC(p), MATCH_MAPPING); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); if (!size && !star_target) { // If the pattern is just "{}", we're done! Pop the subject: pc->on_top--; ADDOP(c, LOC(p), POP_TOP); - return 1; + return SUCCESS; } if (size) { // If the pattern has any keys in it, perform a length check: ADDOP(c, LOC(p), GET_LEN); ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); ADDOP_COMPARE(c, LOC(p), GtE); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); } if (INT_MAX < size - 1) { return compiler_error(c, LOC(p), "too many sub-patterns in mapping pattern"); @@ -6695,7 +6617,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, // SyntaxError in the case of duplicates. PyObject *seen = PySet_New(NULL); if (seen == NULL) { - return 0; + return ERROR; } // NOTE: goto error on failure in the loop below to avoid leaking `seen` @@ -6729,7 +6651,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, compiler_error(c, LOC(p), e); goto error; } - if (!compiler_visit_expr(c, key)) { + if (compiler_visit_expr(c, key) < 0) { goto error; } } @@ -6744,7 +6666,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, ADDOP_I(c, LOC(p), COPY, 1); ADDOP_LOAD_CONST(c, LOC(p), Py_None); ADDOP_I(c, LOC(p), IS_OP, 1); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); // So far so good. Use that tuple of values on the stack to match // sub-patterns against: ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, size); @@ -6752,7 +6674,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, for (Py_ssize_t i = 0; i < size; i++) { pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } // If we get this far, it's a match! Whatever happens next should consume // the tuple of keys and the subject: @@ -6773,17 +6695,17 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, ADDOP_I(c, LOC(p), SWAP, 2); // [copy, keys..., copy, key] ADDOP(c, LOC(p), DELETE_SUBSCR); // [copy, keys...] } - RETURN_IF_FALSE(pattern_helper_store_name(c, LOC(p), star_target, pc)); + RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), star_target, pc)); } else { ADDOP(c, LOC(p), POP_TOP); // Tuple of keys. ADDOP(c, LOC(p), POP_TOP); // Subject. } - return 1; + return SUCCESS; error: Py_DECREF(seen); - return 0; + return ERROR; } static int @@ -6813,8 +6735,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; - if (!cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, LOC(alt)) || - !compiler_pattern(c, alt, pc)) { + if (cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, LOC(alt)) < 0 || + compiler_pattern(c, alt, pc) < 0) { goto error; } // Success! @@ -6868,7 +6790,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Do the same thing to the stack, using several // rotations: while (rotations--) { - if (!pattern_helper_rotate(c, LOC(alt), icontrol + 1)){ + if (pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { goto error; } } @@ -6876,8 +6798,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) } } assert(control); - if (!cfg_builder_addop_j(CFG_BUILDER(c), LOC(alt), JUMP, end) || - !emit_and_reset_fail_pop(c, LOC(alt), pc)) + if (cfg_builder_addop_j(CFG_BUILDER(c), LOC(alt), JUMP, end) < 0 || + emit_and_reset_fail_pop(c, LOC(alt), pc) < 0) { goto error; } @@ -6888,8 +6810,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to NULL this for the PyObject_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: - if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, LOC(p)) || - !jump_to_fail_pop(c, LOC(p), pc, JUMP)) { + if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, LOC(p)) < 0 || + jump_to_fail_pop(c, LOC(p), pc, JUMP) < 0) { goto error; } @@ -6904,7 +6826,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_ssize_t nrots = nstores + 1 + pc->on_top + PyList_GET_SIZE(pc->stores); for (Py_ssize_t i = 0; i < nstores; i++) { // Rotate this capture to its proper place on the stack: - if (!pattern_helper_rotate(c, LOC(p), nrots)) { + if (pattern_helper_rotate(c, LOC(p), nrots) < 0) { goto error; } // Update the list of previous stores with this new name, checking for @@ -6927,14 +6849,14 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // NOTE: Returning macros are safe again. // Pop the copy of the subject: ADDOP(c, LOC(p), POP_TOP); - return 1; + return SUCCESS; diff: compiler_error(c, LOC(p), "alternative patterns bind different names"); error: PyObject_Free(old_pc.fail_pop); Py_DECREF(old_pc.stores); Py_XDECREF(control); - return 0; + return ERROR; } @@ -6966,20 +6888,20 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, // We need to keep the subject on top during the sequence and length checks: pc->on_top++; ADDOP(c, LOC(p), MATCH_SEQUENCE); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); if (star < 0) { // No star: len(subject) == size ADDOP(c, LOC(p), GET_LEN); ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); ADDOP_COMPARE(c, LOC(p), Eq); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); } else if (size > 1) { // Star: len(subject) >= size - 1 ADDOP(c, LOC(p), GET_LEN); ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size - 1)); ADDOP_COMPARE(c, LOC(p), GtE); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); } // Whatever comes next should consume the subject: pc->on_top--; @@ -6988,12 +6910,12 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, ADDOP(c, LOC(p), POP_TOP); } else if (star_wildcard) { - RETURN_IF_FALSE(pattern_helper_sequence_subscr(c, LOC(p), patterns, star, pc)); + RETURN_IF_ERROR(pattern_helper_sequence_subscr(c, LOC(p), patterns, star, pc)); } else { - RETURN_IF_FALSE(pattern_helper_sequence_unpack(c, LOC(p), patterns, star, pc)); + RETURN_IF_ERROR(pattern_helper_sequence_unpack(c, LOC(p), patterns, star, pc)); } - return 1; + return SUCCESS; } static int @@ -7007,8 +6929,8 @@ compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) } VISIT(c, expr, value); ADDOP_COMPARE(c, LOC(p), Eq); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - return 1; + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + return SUCCESS; } static int @@ -7017,8 +6939,8 @@ compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc assert(p->kind == MatchSingleton_kind); ADDOP_LOAD_CONST(c, LOC(p), p->v.MatchSingleton.value); ADDOP_COMPARE(c, LOC(p), Is); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - return 1; + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + return SUCCESS; } static int @@ -7063,32 +6985,35 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) if (i != cases - has_default - 1) { ADDOP_I(c, LOC(m->pattern), COPY, 1); } - RETURN_IF_FALSE(pc->stores = PyList_New(0)); + pc->stores = PyList_New(0); + if (pc->stores == NULL) { + return ERROR; + } // Irrefutable cases must be either guarded, last, or both: pc->allow_irrefutable = m->guard != NULL || i == cases - 1; pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; // NOTE: Can't use returning macros here (they'll leak pc->stores)! - if (!compiler_pattern(c, m->pattern, pc)) { + if (compiler_pattern(c, m->pattern, pc) < 0) { Py_DECREF(pc->stores); - return 0; + return ERROR; } assert(!pc->on_top); // It's a match! Store all of the captured names (they're on the stack). Py_ssize_t nstores = PyList_GET_SIZE(pc->stores); for (Py_ssize_t n = 0; n < nstores; n++) { PyObject *name = PyList_GET_ITEM(pc->stores, n); - if (!compiler_nameop(c, LOC(m->pattern), name, Store)) { + if (compiler_nameop(c, LOC(m->pattern), name, Store) < 0) { Py_DECREF(pc->stores); - return 0; + return ERROR; } } Py_DECREF(pc->stores); // NOTE: Returning macros are safe again. if (m->guard) { - RETURN_IF_FALSE(ensure_fail_pop(c, pc, 0)); - RETURN_IF_FALSE(compiler_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); + RETURN_IF_ERROR(ensure_fail_pop(c, pc, 0)); + RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); } // Success! Pop the subject off, we're done with it: if (i != cases - has_default - 1) { @@ -7099,7 +7024,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) // If the pattern fails to match, we want the line number of the // cleanup to be associated with the failed pattern, not the last line // of the body - RETURN_IF_FALSE(emit_and_reset_fail_pop(c, LOC(m->pattern), pc)); + RETURN_IF_ERROR(emit_and_reset_fail_pop(c, LOC(m->pattern), pc)); } if (has_default) { // A trailing "case _" is common, and lets us save a bit of redundant @@ -7114,12 +7039,12 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) ADDOP(c, LOC(m->pattern), NOP); } if (m->guard) { - RETURN_IF_FALSE(compiler_jump_if(c, LOC(m->pattern), m->guard, end, 0)); + RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, end, 0)); } VISIT_SEQ(c, stmt, m->body); } USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -8792,6 +8717,19 @@ prepare_localsplus(struct compiler* c, int code_flags) return nlocalsplus; } +static int +add_return_at_end_of_block(struct compiler *c, int addNone) +{ + /* Make sure every block that falls off the end returns None. */ + if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { + if (addNone) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + } + ADDOP(c, NO_LOCATION, RETURN_VALUE); + } + return SUCCESS; +} + static PyCodeObject * assemble(struct compiler *c, int addNone) { @@ -8805,12 +8743,8 @@ assemble(struct compiler *c, int addNone) return NULL; } - /* Make sure every block that falls off the end returns None. */ - if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { - if (addNone) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - } - ADDOP(c, NO_LOCATION, RETURN_VALUE); + if (add_return_at_end_of_block(c, addNone) < 0) { + return NULL; } int nblocks = 0; @@ -9992,7 +9926,7 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) if (PyErr_Occurred()) { return -1; } - if (!cfg_builder_addop(g, opcode, oparg, loc)) { + if (cfg_builder_addop(g, opcode, oparg, loc) < 0) { return -1; } } @@ -10077,7 +10011,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - if (!compiler_codegen(c, mod)) { + if (compiler_codegen(c, mod) < 0) { goto finally; } From webhook-mailer at python.org Mon Dec 12 09:51:05 2022 From: webhook-mailer at python.org (markshannon) Date: Mon, 12 Dec 2022 14:51:05 -0000 Subject: [Python-checkins] GH-100143: Improve collecting pystats for parts of runs (GH-100144) Message-ID: <mailman.2912.1670856667.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1583c6e326a8454d3c806763620e1329bf6b7cbe commit: 1583c6e326a8454d3c806763620e1329bf6b7cbe branch: main author: Michael Droettboom <mdboom at gmail.com> committer: markshannon <mark at hotpy.org> date: 2022-12-12T14:50:43Z summary: GH-100143: Improve collecting pystats for parts of runs (GH-100144) * pystats off by default * Add -Xpystats flag * Always dump pystats, even if turned off files: A Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst M Python/initconfig.c M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst new file mode 100644 index 000000000000..20a25f8b03d1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst @@ -0,0 +1,3 @@ +When built with ``--enable-pystats``, stats collection is now off by +default. To enable it early at startup, pass the ``-Xpystats`` flag. Stats +are now always dumped, even if switched off. diff --git a/Python/initconfig.c b/Python/initconfig.c index 64ae987b3f34..d05099cd9977 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -129,7 +129,14 @@ The following implementation-specific options are available:\n\ \n\ -X int_max_str_digits=number: limit the size of int<->str conversions.\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ - The default is sys.int_info.default_max_str_digits. 0 disables."; + The default is sys.int_info.default_max_str_digits. 0 disables." + +#ifdef Py_STATS +"\n\ +\n\ +-X pystats: Enable pystats collection at startup." +#endif +; /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = @@ -2186,6 +2193,12 @@ config_read(PyConfig *config, int compute_path_config) config->show_ref_count = 1; } +#ifdef Py_STATS + if (config_get_xoption(config, L"pystats")) { + _py_stats = &_py_stats_struct; + } +#endif + status = config_read_complex_options(config); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Python/specialize.c b/Python/specialize.c index 7545a7712493..785088eac8c5 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -18,7 +18,7 @@ #ifdef Py_STATS PyStats _py_stats_struct = { 0 }; -PyStats *_py_stats = &_py_stats_struct; +PyStats *_py_stats = NULL; #define ADD_STAT_TO_DICT(res, field) \ do { \ @@ -205,9 +205,6 @@ _Py_StatsClear(void) void _Py_PrintSpecializationStats(int to_file) { - if (_py_stats == NULL) { - return; - } FILE *out = stderr; if (to_file) { /* Write to a file instead of stderr. */ @@ -238,7 +235,7 @@ _Py_PrintSpecializationStats(int to_file) else { fprintf(out, "Specialization stats:\n"); } - print_stats(out, _py_stats); + print_stats(out, &_py_stats_struct); if (out != stderr) { fclose(out); } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index c15501bdc761..c30a60e9514b 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -34,6 +34,16 @@ TOTAL = "specialization.deferred", "specialization.hit", "specialization.miss", "execution_count" +def format_ratio(num, den): + """ + Format a ratio as a percentage. When the denominator is 0, returns the empty + string. + """ + if den == 0: + return "" + else: + return f"{num/den:.01%}" + def join_rows(a_rows, b_rows): """ Joins two tables together, side-by-side, where the first column in each is a @@ -87,7 +97,7 @@ def calculate_specialization_stats(family_stats, total): continue else: label = key - rows.append((f"{label:>12}", f"{family_stats[key]:>12}", f"{100*family_stats[key]/total:0.1f}%")) + rows.append((f"{label:>12}", f"{family_stats[key]:>12}", format_ratio(family_stats[key], total))) return rows def calculate_specialization_success_failure(family_stats): @@ -100,7 +110,7 @@ def calculate_specialization_success_failure(family_stats): label = key[len("specialization."):] label = label[0].upper() + label[1:] val = family_stats.get(key, 0) - rows.append((label, val, f"{100*val/total_attempts:0.1f}%")) + rows.append((label, val, format_ratio(val, total_attempts))) return rows def calculate_specialization_failure_kinds(name, family_stats, defines): @@ -118,7 +128,7 @@ def calculate_specialization_failure_kinds(name, family_stats, defines): for value, index in failures: if not value: continue - rows.append((kind_to_text(index, defines, name), value, f"{100*value/total_failures:0.1f}%")) + rows.append((kind_to_text(index, defines, name), value, format_ratio(value, total_failures))) return rows def print_specialization_stats(name, family_stats, defines): @@ -318,11 +328,11 @@ def calculate_execution_counts(opcode_stats, total): for (count, name, miss) in counts: cumulative += count if miss: - miss = f"{100*miss/count:0.1f}%" + miss = format_ratio(miss, count) else: miss = "" - rows.append((name, count, f"{100*count/total:0.1f}%", - f"{100*cumulative/total:0.1f}%", miss)) + rows.append((name, count, format_ratio(count, total), + format_ratio(cumulative, total), miss)) return rows def emit_execution_counts(opcode_stats, total): @@ -386,9 +396,9 @@ def emit_comparative_specialization_stats(base_opcode_stats, head_opcode_stats): def calculate_specialization_effectiveness(opcode_stats, total): basic, not_specialized, specialized = categorized_counts(opcode_stats) return [ - ("Basic", basic, f"{basic*100/total:0.1f}%"), - ("Not specialized", not_specialized, f"{not_specialized*100/total:0.1f}%"), - ("Specialized", specialized, f"{specialized*100/total:0.1f}%"), + ("Basic", basic, format_ratio(basic, total)), + ("Not specialized", not_specialized, format_ratio(not_specialized, total)), + ("Specialized", specialized, format_ratio(specialized, total)), ] def emit_specialization_overview(opcode_stats, total): @@ -405,7 +415,7 @@ def emit_specialization_overview(opcode_stats, total): counts.sort(reverse=True) if total: with Section(f"{title} by instruction", 3): - rows = [ (name, count, f"{100*count/total:0.1f}%") for (count, name) in counts[:10] ] + rows = [ (name, count, format_ratio(count, total)) for (count, name) in counts[:10] ] emit_table(("Name", "Count:", "Ratio:"), rows) def emit_comparative_specialization_overview(base_opcode_stats, base_total, head_opcode_stats, head_total): @@ -432,15 +442,15 @@ def calculate_call_stats(stats): rows = [] for key, value in stats.items(): if "Calls to" in key: - rows.append((key, value, f"{100*value/total:0.1f}%")) + rows.append((key, value, format_ratio(value, total))) elif key.startswith("Calls "): name, index = key[:-1].split("[") index = int(index) label = name + " (" + pretty(defines[index][0]) + ")" - rows.append((label, value, f"{100*value/total:0.1f}%")) + rows.append((label, value, format_ratio(value, total))) for key, value in stats.items(): if key.startswith("Frame"): - rows.append((key, value, f"{100*value/total:0.1f}%")) + rows.append((key, value, format_ratio(value, total))) return rows def emit_call_stats(stats): @@ -468,13 +478,13 @@ def calculate_object_stats(stats): for key, value in stats.items(): if key.startswith("Object"): if "materialize" in key: - ratio = f"{100*value/total_materializations:0.1f}%" + ratio = format_ratio(value, total_materializations) elif "allocations" in key: - ratio = f"{100*value/total_allocations:0.1f}%" + ratio = format_ratio(value, total_allocations) elif "increfs" in key: - ratio = f"{100*value/total_increfs:0.1f}%" + ratio = format_ratio(value, total_increfs) elif "decrefs" in key: - ratio = f"{100*value/total_decrefs:0.1f}%" + ratio = format_ratio(value, total_decrefs) else: ratio = "" label = key[6:].strip() @@ -517,8 +527,8 @@ def emit_pair_counts(opcode_stats, total): for (count, pair) in itertools.islice(pair_counts, 100): i, j = pair cumulative += count - rows.append((opname[i] + " " + opname[j], count, f"{100*count/total:0.1f}%", - f"{100*cumulative/total:0.1f}%")) + rows.append((opname[i] + " " + opname[j], count, format_ratio(count, total), + format_ratio(cumulative, total))) emit_table(("Pair", "Count:", "Self:", "Cumulative:"), rows ) From webhook-mailer at python.org Mon Dec 12 10:44:31 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 15:44:31 -0000 Subject: [Python-checkins] gh-81057: Move tracemalloc Globals to _PyRuntimeState (gh-100151) Message-ID: <mailman.2913.1670859873.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8790d4d31fcd3abaccf31d27f72a8684adfc9dee commit: 8790d4d31fcd3abaccf31d27f72a8684adfc9dee branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T08:44:23-07:00 summary: gh-81057: Move tracemalloc Globals to _PyRuntimeState (gh-100151) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_tracemalloc.h M Include/internal/pycore_pymem.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M Modules/_tracemalloc.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 5749af7465f6..4cc953d8d779 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -90,28 +90,6 @@ PyAPI_FUNC(int) _PyMem_GetAllocatorName( PYMEM_ALLOCATOR_NOT_SET does nothing. */ PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator); -struct _PyTraceMalloc_Config { - /* Module initialized? - Variable protected by the GIL */ - enum { - TRACEMALLOC_NOT_INITIALIZED, - TRACEMALLOC_INITIALIZED, - TRACEMALLOC_FINALIZED - } initialized; - - /* Is tracemalloc tracing memory allocations? - Variable protected by the GIL */ - int tracing; - - /* limit of the number of frames in a traceback, 1 by default. - Variable protected by the GIL. */ - int max_nframe; -}; - -#define _PyTraceMalloc_Config_INIT \ - {.initialized = TRACEMALLOC_NOT_INITIALIZED, \ - .tracing = 0, \ - .max_nframe = 1} #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index fe2de5feb47a..99ec6fc8862b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -23,6 +23,7 @@ extern "C" { #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state #include "pycore_time.h" // struct _time_runtime_state +#include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -137,11 +138,9 @@ typedef struct pyruntimestate { struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; struct _getargs_runtime_state getargs; - struct { - struct _PyTraceMalloc_Config config; - } tracemalloc; struct _dtoa_runtime_state dtoa; struct _fileutils_state fileutils; + struct _tracemalloc_runtime_state tracemalloc; PyPreConfig preconfig; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b569e5833f1d..029357dedf3e 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -50,13 +50,11 @@ extern "C" { in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ }, \ - .tracemalloc = { \ - .config = _PyTraceMalloc_Config_INIT, \ - }, \ .dtoa = _dtoa_runtime_state_INIT(runtime), \ .fileutils = { \ .force_ascii = -1, \ }, \ + .tracemalloc = _tracemalloc_runtime_state_INIT, \ .float_state = { \ .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h new file mode 100644 index 000000000000..08d7d1096c78 --- /dev/null +++ b/Include/internal/pycore_tracemalloc.h @@ -0,0 +1,121 @@ +#ifndef Py_INTERNAL_TRACEMALLOC_H +#define Py_INTERNAL_TRACEMALLOC_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_hashtable.h" // _Py_hashtable_t + + +/* Trace memory blocks allocated by PyMem_RawMalloc() */ +#define TRACE_RAW_MALLOC + + +struct _PyTraceMalloc_Config { + /* Module initialized? + Variable protected by the GIL */ + enum { + TRACEMALLOC_NOT_INITIALIZED, + TRACEMALLOC_INITIALIZED, + TRACEMALLOC_FINALIZED + } initialized; + + /* Is tracemalloc tracing memory allocations? + Variable protected by the GIL */ + int tracing; + + /* limit of the number of frames in a traceback, 1 by default. + Variable protected by the GIL. */ + int max_nframe; +}; + + +/* Pack the frame_t structure to reduce the memory footprint on 64-bit + architectures: 12 bytes instead of 16. */ +struct +#ifdef __GNUC__ +__attribute__((packed)) +#elif defined(_MSC_VER) +#pragma pack(push, 4) +#endif +tracemalloc_frame { + /* filename cannot be NULL: "<unknown>" is used if the Python frame + filename is NULL */ + PyObject *filename; + unsigned int lineno; +}; +#ifdef _MSC_VER +#pragma pack(pop) +#endif + +struct tracemalloc_traceback { + Py_uhash_t hash; + /* Number of frames stored */ + uint16_t nframe; + /* Total number of frames the traceback had */ + uint16_t total_nframe; + struct tracemalloc_frame frames[1]; +}; + + +struct _tracemalloc_runtime_state { + struct _PyTraceMalloc_Config config; + + /* Protected by the GIL */ + struct { + PyMemAllocatorEx mem; + PyMemAllocatorEx raw; + PyMemAllocatorEx obj; + } allocators; + +#if defined(TRACE_RAW_MALLOC) + PyThread_type_lock tables_lock; +#endif + /* Size in bytes of currently traced memory. + Protected by TABLES_LOCK(). */ + size_t traced_memory; + /* Peak size in bytes of traced memory. + Protected by TABLES_LOCK(). */ + size_t peak_traced_memory; + /* Hash table used as a set to intern filenames: + PyObject* => PyObject*. + Protected by the GIL */ + _Py_hashtable_t *filenames; + /* Buffer to store a new traceback in traceback_new(). + Protected by the GIL. */ + struct tracemalloc_traceback *traceback; + /* Hash table used as a set to intern tracebacks: + traceback_t* => traceback_t* + Protected by the GIL */ + _Py_hashtable_t *tracebacks; + /* pointer (void*) => trace (trace_t*). + Protected by TABLES_LOCK(). */ + _Py_hashtable_t *traces; + /* domain (unsigned int) => traces (_Py_hashtable_t). + Protected by TABLES_LOCK(). */ + _Py_hashtable_t *domains; + + struct tracemalloc_traceback empty_traceback; + + Py_tss_t reentrant_key; +}; + +#define _tracemalloc_runtime_state_INIT \ + { \ + .config = { \ + .initialized = TRACEMALLOC_NOT_INITIALIZED, \ + .tracing = 0, \ + .max_nframe = 1, \ + }, \ + .reentrant_key = Py_tss_NEEDS_INIT, \ + } + + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_TRACEMALLOC_H diff --git a/Makefile.pre.in b/Makefile.pre.in index 815df69ad48c..9577f9dee6da 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1678,6 +1678,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ + $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tuple.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 0d70f0cf34c8..ac16626f2101 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -20,9 +20,6 @@ module _tracemalloc _Py_DECLARE_STR(anon_unknown, "<unknown>"); -/* Trace memory blocks allocated by PyMem_RawMalloc() */ -#define TRACE_RAW_MALLOC - /* Forward declaration */ static void tracemalloc_stop(void); static void* raw_malloc(size_t size); @@ -35,19 +32,14 @@ static void raw_free(void *ptr); #define TO_PTR(key) ((const void *)(uintptr_t)(key)) #define FROM_PTR(key) ((uintptr_t)(key)) -/* Protected by the GIL */ -static struct { - PyMemAllocatorEx mem; - PyMemAllocatorEx raw; - PyMemAllocatorEx obj; -} allocators; +#define allocators _PyRuntime.tracemalloc.allocators #if defined(TRACE_RAW_MALLOC) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ -static PyThread_type_lock tables_lock; +# define tables_lock _PyRuntime.tracemalloc.tables_lock # define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) # define TABLES_UNLOCK() PyThread_release_lock(tables_lock) #else @@ -59,33 +51,8 @@ static PyThread_type_lock tables_lock; #define DEFAULT_DOMAIN 0 -/* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. */ -typedef struct -#ifdef __GNUC__ -__attribute__((packed)) -#elif defined(_MSC_VER) -#pragma pack(push, 4) -#endif -{ - /* filename cannot be NULL: "<unknown>" is used if the Python frame - filename is NULL */ - PyObject *filename; - unsigned int lineno; -} frame_t; -#ifdef _MSC_VER -#pragma pack(pop) -#endif - - -typedef struct { - Py_uhash_t hash; - /* Number of frames stored */ - uint16_t nframe; - /* Total number of frames the traceback had */ - uint16_t total_nframe; - frame_t frames[1]; -} traceback_t; +typedef struct tracemalloc_frame frame_t; +typedef struct tracemalloc_traceback traceback_t; #define TRACEBACK_SIZE(NFRAME) \ (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1)) @@ -96,7 +63,8 @@ typedef struct { static const unsigned long MAX_NFRAME = Py_MIN(UINT16_MAX, ((SIZE_MAX - sizeof(traceback_t)) / sizeof(frame_t) + 1)); -static traceback_t tracemalloc_empty_traceback; +#define tracemalloc_empty_traceback _PyRuntime.tracemalloc.empty_traceback + /* Trace of a memory block */ typedef struct { @@ -108,35 +76,13 @@ typedef struct { } trace_t; -/* Size in bytes of currently traced memory. - Protected by TABLES_LOCK(). */ -static size_t tracemalloc_traced_memory = 0; - -/* Peak size in bytes of traced memory. - Protected by TABLES_LOCK(). */ -static size_t tracemalloc_peak_traced_memory = 0; - -/* Hash table used as a set to intern filenames: - PyObject* => PyObject*. - Protected by the GIL */ -static _Py_hashtable_t *tracemalloc_filenames = NULL; - -/* Buffer to store a new traceback in traceback_new(). - Protected by the GIL. */ -static traceback_t *tracemalloc_traceback = NULL; - -/* Hash table used as a set to intern tracebacks: - traceback_t* => traceback_t* - Protected by the GIL */ -static _Py_hashtable_t *tracemalloc_tracebacks = NULL; - -/* pointer (void*) => trace (trace_t*). - Protected by TABLES_LOCK(). */ -static _Py_hashtable_t *tracemalloc_traces = NULL; - -/* domain (unsigned int) => traces (_Py_hashtable_t). - Protected by TABLES_LOCK(). */ -static _Py_hashtable_t *tracemalloc_domains = NULL; +#define tracemalloc_traced_memory _PyRuntime.tracemalloc.traced_memory +#define tracemalloc_peak_traced_memory _PyRuntime.tracemalloc.peak_traced_memory +#define tracemalloc_filenames _PyRuntime.tracemalloc.filenames +#define tracemalloc_traceback _PyRuntime.tracemalloc.traceback +#define tracemalloc_tracebacks _PyRuntime.tracemalloc.tracebacks +#define tracemalloc_traces _PyRuntime.tracemalloc.traces +#define tracemalloc_domains _PyRuntime.tracemalloc.domains #ifdef TRACE_DEBUG @@ -157,7 +103,7 @@ tracemalloc_error(const char *format, ...) #if defined(TRACE_RAW_MALLOC) #define REENTRANT_THREADLOCAL -static Py_tss_t tracemalloc_reentrant_key = Py_tss_NEEDS_INIT; +#define tracemalloc_reentrant_key _PyRuntime.tracemalloc.reentrant_key /* Any non-NULL pointer can be used */ #define REENTRANT Py_True diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 35fbff320f46..25572d6ebcd5 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -259,6 +259,7 @@ <ClInclude Include="..\Include\internal\pycore_time.h" /> <ClInclude Include="..\Include\internal\pycore_token.h" /> <ClInclude Include="..\Include\internal\pycore_traceback.h" /> + <ClInclude Include="..\Include\internal\pycore_tracemalloc.h" /> <ClInclude Include="..\Include\internal\pycore_tuple.h" /> <ClInclude Include="..\Include\internal\pycore_typeobject.h" /> <ClInclude Include="..\Include\internal\pycore_ucnhash.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 19cb5cf1c807..d45b50c5d62d 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -678,6 +678,9 @@ <ClInclude Include="..\Include\internal\pycore_traceback.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_tracemalloc.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_tuple.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 94e9831db1fd..5dcd396c5487 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -377,17 +377,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## state -Modules/_tracemalloc.c - allocators - -Modules/_tracemalloc.c - tables_lock - -Modules/_tracemalloc.c - tracemalloc_empty_traceback - -Modules/_tracemalloc.c - tracemalloc_traced_memory - -Modules/_tracemalloc.c - tracemalloc_peak_traced_memory - -Modules/_tracemalloc.c - tracemalloc_filenames - -Modules/_tracemalloc.c - tracemalloc_traceback - -Modules/_tracemalloc.c - tracemalloc_tracebacks - -Modules/_tracemalloc.c - tracemalloc_traces - -Modules/_tracemalloc.c - tracemalloc_domains - -Modules/_tracemalloc.c - tracemalloc_reentrant_key - Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/signalmodule.c - is_tripped - Modules/signalmodule.c - signal_global_state - From webhook-mailer at python.org Mon Dec 12 11:59:26 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 16:59:26 -0000 Subject: [Python-checkins] gh-81057: Move faulthandler Globals to _PyRuntimeState (gh-100152) Message-ID: <mailman.2914.1670864368.3313.python-checkins@python.org> https://github.com/python/cpython/commit/53d9cd95cd91f1a291a3923acb95e0e86942291a commit: 53d9cd95cd91f1a291a3923acb95e0e86942291a branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T09:58:46-07:00 summary: gh-81057: Move faulthandler Globals to _PyRuntimeState (gh-100152) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_faulthandler.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M Modules/faulthandler.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h new file mode 100644 index 000000000000..e6aec7745a64 --- /dev/null +++ b/Include/internal/pycore_faulthandler.h @@ -0,0 +1,99 @@ +#ifndef Py_INTERNAL_FAULTHANDLER_H +#define Py_INTERNAL_FAULTHANDLER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef HAVE_SIGACTION +# include <signal.h> +#endif + + +#ifndef MS_WINDOWS + /* register() is useless on Windows, because only SIGSEGV, SIGABRT and + SIGILL can be handled by the process, and these signals can only be used + with enable(), not using register() */ +# define FAULTHANDLER_USER +#endif + + +#ifdef HAVE_SIGACTION +/* Using an alternative stack requires sigaltstack() + and sigaction() SA_ONSTACK */ +# ifdef HAVE_SIGALTSTACK +# define FAULTHANDLER_USE_ALT_STACK +# endif +typedef struct sigaction _Py_sighandler_t; +#else +typedef PyOS_sighandler_t _Py_sighandler_t; +#endif // HAVE_SIGACTION + + +#ifdef FAULTHANDLER_USER +struct faulthandler_user_signal { + int enabled; + PyObject *file; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; + PyInterpreterState *interp; +}; +#endif /* FAULTHANDLER_USER */ + + +struct _faulthandler_runtime_state { + struct { + int enabled; + PyObject *file; + int fd; + int all_threads; + PyInterpreterState *interp; +#ifdef MS_WINDOWS + void *exc_handler; +#endif + } fatal_error; + + struct { + PyObject *file; + int fd; + PY_TIMEOUT_T timeout_us; /* timeout in microseconds */ + int repeat; + PyInterpreterState *interp; + int exit; + char *header; + size_t header_len; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + PyThread_type_lock cancel_event; + /* released by child thread when joined */ + PyThread_type_lock running; + } thread; + +#ifdef FAULTHANDLER_USER + struct faulthandler_user_signal *user_signals; +#endif + +#ifdef FAULTHANDLER_USE_ALT_STACK + stack_t stack; + stack_t old_stack; +#endif +}; + +#define _faulthandler_runtime_state_INIT \ + { \ + .fatal_error = { \ + .fd = -1, \ + }, \ + } + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FAULTHANDLER_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 99ec6fc8862b..b9ed8f593156 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -13,6 +13,7 @@ extern "C" { #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_dtoa.h" // struct _dtoa_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state +#include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_function.h" // struct _func_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state @@ -140,6 +141,7 @@ typedef struct pyruntimestate { struct _getargs_runtime_state getargs; struct _dtoa_runtime_state dtoa; struct _fileutils_state fileutils; + struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; PyPreConfig preconfig; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 029357dedf3e..9677a727c446 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -54,6 +54,7 @@ extern "C" { .fileutils = { \ .force_ascii = -1, \ }, \ + .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ .float_state = { \ .float_format = _py_float_format_unknown, \ diff --git a/Makefile.pre.in b/Makefile.pre.in index 9577f9dee6da..dd6c3fbd1c64 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1634,6 +1634,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dtoa.h \ $(srcdir)/Include/internal/pycore_exceptions.h \ + $(srcdir)/Include/internal/pycore_faulthandler.h \ $(srcdir)/Include/internal/pycore_fileutils.h \ $(srcdir)/Include/internal/pycore_floatobject.h \ $(srcdir)/Include/internal/pycore_format.h \ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 341a03a244cd..5309a3728c5e 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -18,12 +18,6 @@ # include <sys/resource.h> #endif -/* Using an alternative stack requires sigaltstack() - and sigaction() SA_ONSTACK */ -#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) -# define FAULTHANDLER_USE_ALT_STACK -#endif - #if defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_LINUX_AUXVEC_H) && defined(HAVE_SYS_AUXV_H) # include <linux/auxvec.h> // AT_MINSIGSTKSZ # include <sys/auxv.h> // getauxval() @@ -32,13 +26,6 @@ /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) -#ifndef MS_WINDOWS - /* register() is useless on Windows, because only SIGSEGV, SIGABRT and - SIGILL can be handled by the process, and these signals can only be used - with enable(), not using register() */ -# define FAULTHANDLER_USER -#endif - #define PUTS(fd, str) _Py_write_noraise(fd, str, strlen(str)) @@ -58,12 +45,6 @@ #endif -#ifdef HAVE_SIGACTION -typedef struct sigaction _Py_sighandler_t; -#else -typedef PyOS_sighandler_t _Py_sighandler_t; -#endif - typedef struct { int signum; int enabled; @@ -72,47 +53,12 @@ typedef struct { int all_threads; } fault_handler_t; -static struct { - int enabled; - PyObject *file; - int fd; - int all_threads; - PyInterpreterState *interp; -#ifdef MS_WINDOWS - void *exc_handler; -#endif -} fatal_error = {0, NULL, -1, 0}; - -static struct { - PyObject *file; - int fd; - PY_TIMEOUT_T timeout_us; /* timeout in microseconds */ - int repeat; - PyInterpreterState *interp; - int exit; - char *header; - size_t header_len; - /* The main thread always holds this lock. It is only released when - faulthandler_thread() is interrupted before this thread exits, or at - Python exit. */ - PyThread_type_lock cancel_event; - /* released by child thread when joined */ - PyThread_type_lock running; -} thread; +#define fatal_error _PyRuntime.faulthandler.fatal_error +#define thread _PyRuntime.faulthandler.thread #ifdef FAULTHANDLER_USER -typedef struct { - int enabled; - PyObject *file; - int fd; - int all_threads; - int chain; - _Py_sighandler_t previous; - PyInterpreterState *interp; -} user_signal_t; - -static user_signal_t *user_signals; - +#define user_signals _PyRuntime.faulthandler.user_signals +typedef struct faulthandler_user_signal user_signal_t; static void faulthandler_user(int signum); #endif /* FAULTHANDLER_USER */ @@ -134,8 +80,8 @@ static const size_t faulthandler_nsignals = \ Py_ARRAY_LENGTH(faulthandler_handlers); #ifdef FAULTHANDLER_USE_ALT_STACK -static stack_t stack; -static stack_t old_stack; +# define stack _PyRuntime.faulthandler.stack +# define old_stack _PyRuntime.faulthandler.old_stack #endif @@ -1094,7 +1040,7 @@ faulthandler_fatal_error_thread(void *plock) static PyObject * faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) { - long thread; + long tid; PyThread_type_lock lock; faulthandler_suppress_crash_report(); @@ -1105,8 +1051,8 @@ faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) PyThread_acquire_lock(lock, WAIT_LOCK); - thread = PyThread_start_new_thread(faulthandler_fatal_error_thread, lock); - if (thread == -1) { + tid = PyThread_start_new_thread(faulthandler_fatal_error_thread, lock); + if (tid == -1) { PyThread_free_lock(lock); PyErr_SetString(PyExc_RuntimeError, "unable to start the thread"); return NULL; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 25572d6ebcd5..bb2aaae3317b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -214,6 +214,7 @@ <ClInclude Include="..\Include\internal\pycore_dict_state.h" /> <ClInclude Include="..\Include\internal\pycore_dtoa.h" /> <ClInclude Include="..\Include\internal\pycore_exceptions.h" /> + <ClInclude Include="..\Include\internal\pycore_faulthandler.h" /> <ClInclude Include="..\Include\internal\pycore_fileutils.h" /> <ClInclude Include="..\Include\internal\pycore_floatobject.h" /> <ClInclude Include="..\Include\internal\pycore_format.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d45b50c5d62d..339e7cc4937a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -549,6 +549,9 @@ <ClInclude Include="..\Include\internal\pycore_exceptions.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_faulthandler.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_fileutils.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -558,6 +561,9 @@ <ClInclude Include="..\Include\internal\pycore_format.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_function.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_gc.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 5dcd396c5487..eb57f95abd17 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -361,15 +361,6 @@ Modules/itertoolsmodule.c - tee_type - Modules/itertoolsmodule.c - teedataobject_type - Modules/itertoolsmodule.c - ziplongest_type - -##----------------------- -## state - -Modules/faulthandler.c - fatal_error - -Modules/faulthandler.c - thread - -Modules/faulthandler.c - user_signals - -Modules/faulthandler.c - stack - -Modules/faulthandler.c - old_stack - - ################################## ## global non-objects to fix in builtin modules @@ -377,7 +368,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## state -Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/signalmodule.c - is_tripped - Modules/signalmodule.c - signal_global_state - Modules/signalmodule.c - wakeup - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7d6ff0ba6d64..b03198627f42 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -151,6 +151,7 @@ Python/sysmodule.c - _preinit_xoptions - # thread-safety # XXX need race protection? +Modules/faulthandler.c faulthandler_dump_traceback reentrant - Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - From webhook-mailer at python.org Mon Dec 12 18:50:36 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 23:50:36 -0000 Subject: [Python-checkins] gh-81057: Move Signal-Related Globals to _PyRuntimeState (gh-100085) Message-ID: <mailman.2915.1670889037.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5eb28bca9fc963607189e3b3e1403f341dbf640a commit: 5eb28bca9fc963607189e3b3e1403f341dbf640a branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T16:50:19-07:00 summary: gh-81057: Move Signal-Related Globals to _PyRuntimeState (gh-100085) https://github.com/python/cpython/issues/81057 files: M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Include/internal/pycore_signal.h M Modules/signalmodule.c M Python/pylifecycle.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index b9ed8f593156..92ed45956c99 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -23,6 +23,7 @@ extern "C" { #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_signal.h" // struct _signals_runtime_state #include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids @@ -93,13 +94,9 @@ typedef struct pyruntimestate { struct _pymem_allocators allocators; struct _obmalloc_state obmalloc; struct pyhash_runtime_state pyhash_state; - struct { - /* True if the main interpreter thread exited due to an unhandled - * KeyboardInterrupt exception, suggesting the user pressed ^C. */ - int unhandled_keyboard_interrupt; - } signals; struct _time_runtime_state time; struct _pythread_runtime_state threads; + struct _signals_runtime_state signals; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 9677a727c446..1431096e2d24 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -26,6 +26,7 @@ extern "C" { }, \ .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ .pyhash_state = pyhash_state_INIT, \ + .signals = _signals_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ diff --git a/Include/internal/pycore_signal.h b/Include/internal/pycore_signal.h index b921dd170e9f..ca3f69d09fc0 100644 --- a/Include/internal/pycore_signal.h +++ b/Include/internal/pycore_signal.h @@ -10,8 +10,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_atomic.h" // _Py_atomic_address + #include <signal.h> // NSIG + #ifdef _SIG_MAXSIG // gh-91145: On FreeBSD, <signal.h> defines NSIG as 32: it doesn't include // realtime signals: [SIGRTMIN,SIGRTMAX]. Use _SIG_MAXSIG instead. For @@ -29,6 +32,66 @@ extern "C" { # define Py_NSIG 64 // Use a reasonable default value #endif +#define INVALID_FD (-1) + +struct _signals_runtime_state { + volatile struct { + _Py_atomic_int tripped; + /* func is atomic to ensure that PyErr_SetInterrupt is async-signal-safe + * (even though it would probably be otherwise, anyway). + */ + _Py_atomic_address func; + } handlers[Py_NSIG]; + + volatile struct { +#ifdef MS_WINDOWS + /* This would be "SOCKET fd" if <winsock2.h> were always included. + It isn't so we must cast to SOCKET where appropriate. */ + volatile int fd; +#elif defined(__VXWORKS__) + int fd; +#else + sig_atomic_t fd; +#endif + + int warn_on_full_buffer; +#ifdef MS_WINDOWS + int use_send; +#endif + } wakeup; + + /* Speed up sigcheck() when none tripped */ + _Py_atomic_int is_tripped; + + /* These objects necessarily belong to the main interpreter. */ + PyObject *default_handler; + PyObject *ignore_handler; + +#ifdef MS_WINDOWS + /* This would be "HANDLE sigint_event" if <windows.h> were always included. + It isn't so we must cast to HANDLE everywhere "sigint_event" is used. */ + void *sigint_event; +#endif + + /* True if the main interpreter thread exited due to an unhandled + * KeyboardInterrupt exception, suggesting the user pressed ^C. */ + int unhandled_keyboard_interrupt; +}; + +#ifdef MS_WINDOWS +# define _signals_WAKEUP_INIT \ + {.fd = INVALID_FD, .warn_on_full_buffer = 1, .use_send = 0} +#else +# define _signals_WAKEUP_INIT \ + {.fd = INVALID_FD, .warn_on_full_buffer = 1} +#endif + +#define _signals_RUNTIME_INIT \ + { \ + .wakeup = _signals_WAKEUP_INIT, \ + } + + #ifdef __cplusplus } #endif diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index c539787e5829..538a7e85bc95 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,7 +13,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_signal.h" // Py_NSIG +#include "pycore_signal.h" #ifndef MS_WINDOWS # include "posixmodule.h" @@ -23,12 +23,13 @@ #endif #ifdef MS_WINDOWS -# include <windows.h> # ifdef HAVE_PROCESS_H # include <process.h> # endif #endif +#include "pycore_signal.h" // Py_NSIG + #ifdef HAVE_SIGNAL_H # include <signal.h> #endif @@ -100,47 +101,13 @@ class sigset_t_converter(CConverter): may not be the thread that received the signal. */ -static volatile struct { - _Py_atomic_int tripped; - /* func is atomic to ensure that PyErr_SetInterrupt is async-signal-safe - * (even though it would probably be otherwise, anyway). - */ - _Py_atomic_address func; -} Handlers[Py_NSIG]; - -#ifdef MS_WINDOWS -#define INVALID_FD ((SOCKET_T)-1) - -static volatile struct { - SOCKET_T fd; - int warn_on_full_buffer; - int use_send; -} wakeup = {.fd = INVALID_FD, .warn_on_full_buffer = 1, .use_send = 0}; -#else -#define INVALID_FD (-1) -static volatile struct { -#ifdef __VXWORKS__ - int fd; -#else - sig_atomic_t fd; -#endif - int warn_on_full_buffer; -} wakeup = {.fd = INVALID_FD, .warn_on_full_buffer = 1}; -#endif - -/* Speed up sigcheck() when none tripped */ -static _Py_atomic_int is_tripped; - -typedef struct { - PyObject *default_handler; - PyObject *ignore_handler; -#ifdef MS_WINDOWS - HANDLE sigint_event; -#endif -} signal_state_t; +#define Handlers _PyRuntime.signals.handlers +#define wakeup _PyRuntime.signals.wakeup +#define is_tripped _PyRuntime.signals.is_tripped // State shared by all Python interpreters -static signal_state_t signal_global_state = {0}; +typedef struct _signals_runtime_state signal_state_t; +#define signal_global_state _PyRuntime.signals #if defined(HAVE_GETITIMER) || defined(HAVE_SETITIMER) # define PYHAVE_ITIMER_ERROR @@ -331,13 +298,7 @@ trip_signal(int sig_num) See bpo-30038 for more details. */ - int fd; -#ifdef MS_WINDOWS - fd = Py_SAFE_DOWNCAST(wakeup.fd, SOCKET_T, int); -#else - fd = wakeup.fd; -#endif - + int fd = wakeup.fd; if (fd != INVALID_FD) { unsigned char byte = (unsigned char)sig_num; #ifdef MS_WINDOWS @@ -407,7 +368,7 @@ signal_handler(int sig_num) #ifdef MS_WINDOWS if (sig_num == SIGINT) { signal_state_t *state = &signal_global_state; - SetEvent(state->sigint_event); + SetEvent((HANDLE)state->sigint_event); } #endif } @@ -822,7 +783,7 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) } old_sockfd = wakeup.fd; - wakeup.fd = sockfd; + wakeup.fd = Py_SAFE_DOWNCAST(sockfd, SOCKET_T, int); wakeup.warn_on_full_buffer = warn_on_full_buffer; wakeup.use_send = is_socket; @@ -873,11 +834,7 @@ PySignal_SetWakeupFd(int fd) fd = -1; } -#ifdef MS_WINDOWS - int old_fd = Py_SAFE_DOWNCAST(wakeup.fd, SOCKET_T, int); -#else int old_fd = wakeup.fd; -#endif wakeup.fd = fd; wakeup.warn_on_full_buffer = 1; return old_fd; @@ -1654,6 +1611,8 @@ signal_module_exec(PyObject *m) signal_state_t *state = &signal_global_state; _signal_module_state *modstate = get_signal_state(m); + // XXX For proper isolation, these values must be guaranteed + // to be effectively const (e.g. immortal). modstate->default_handler = state->default_handler; // borrowed ref modstate->ignore_handler = state->ignore_handler; // borrowed ref @@ -1783,7 +1742,7 @@ _PySignal_Fini(void) #ifdef MS_WINDOWS if (state->sigint_event != NULL) { - CloseHandle(state->sigint_event); + CloseHandle((HANDLE)state->sigint_event); state->sigint_event = NULL; } #endif @@ -2009,7 +1968,7 @@ _PySignal_Init(int install_signal_handlers) #ifdef MS_WINDOWS /* Create manual-reset event, initially unset */ - state->sigint_event = CreateEvent(NULL, TRUE, FALSE, FALSE); + state->sigint_event = (void *)CreateEvent(NULL, TRUE, FALSE, FALSE); if (state->sigint_event == NULL) { PyErr_SetFromWindowsErr(0); return -1; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e13660f51138..1cb0e4d747e1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -54,7 +54,6 @@ extern void _PyIO_Fini(void); #ifdef MS_WINDOWS # undef BYTE -# include "windows.h" extern PyTypeObject PyWindowsConsoleIO_Type; # define PyWindowsConsoleIO_Check(op) \ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index eb57f95abd17..fcec95bbb2f7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -365,13 +365,7 @@ Modules/itertoolsmodule.c - ziplongest_type - ################################## ## global non-objects to fix in builtin modules -##----------------------- -## state - -Modules/signalmodule.c - is_tripped - -Modules/signalmodule.c - signal_global_state - -Modules/signalmodule.c - wakeup - -Modules/signalmodule.c - Handlers - +# <none> ################################## From webhook-mailer at python.org Mon Dec 12 18:51:33 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 23:51:33 -0000 Subject: [Python-checkins] gh-81057: Move _Py_RefTotal to the "Ignored Globals" List (gh-100203) Message-ID: <mailman.2916.1670889095.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0e081a089ec969c9a34f5ff25886205616ef4dd3 commit: 0e081a089ec969c9a34f5ff25886205616ef4dd3 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T16:51:27-07:00 summary: gh-81057: Move _Py_RefTotal to the "Ignored Globals" List (gh-100203) We can't move it to _PyRuntimeState because the symbol is exposed in the stable ABI. We'll have to sort that out before a per-interpreter GIL, but it shouldn't be too hard. https://github.com/python/cpython/issues/81057 files: M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index fcec95bbb2f7..479221cbd4b6 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -302,10 +302,7 @@ Objects/sliceobject.c - _Py_EllipsisObject - ################################## ## global non-objects to fix in core code -##----------------------- -## state - -Objects/object.c - _Py_RefTotal - +# <none> ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index b03198627f42..c71fc0d95821 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -142,6 +142,12 @@ Python/pylifecycle.c - runtime_initialized - Modules/syslogmodule.c - S_ident_o - Modules/syslogmodule.c - S_log_open - +##----------------------- +## kept for stable ABI compatibility + +# XXX should be per-interpreter, without impacting stable ABI extensions +Objects/object.c - _Py_RefTotal - + ##----------------------- ## one-off temporary state From webhook-mailer at python.org Tue Dec 13 05:54:27 2022 From: webhook-mailer at python.org (hugovk) Date: Tue, 13 Dec 2022 10:54:27 -0000 Subject: [Python-checkins] GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (#100207) Message-ID: <mailman.2917.1670928868.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d3ea82aaf940167482df1e08d6482de8f2dd8526 commit: d3ea82aaf940167482df1e08d6482de8f2dd8526 branch: main author: Filipe La?ns <lains at riseup.net> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-13T12:54:07+02:00 summary: GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (#100207) files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 024988777030..839c2c015b49 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -121,7 +121,7 @@ identifier. Python currently uses eight paths: Return the default scheme name for the current platform. - .. versionchanged:: 3.10 + .. versionadded:: 3.10 This function was previously named ``_get_default_scheme()`` and considered an implementation detail. From webhook-mailer at python.org Tue Dec 13 06:02:52 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 13 Dec 2022 11:02:52 -0000 Subject: [Python-checkins] GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) Message-ID: <mailman.2918.1670929373.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cd4166fec2e72438a1057820a98d078644c3a173 commit: cd4166fec2e72438a1057820a98d078644c3a173 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-13T03:02:41-08:00 summary: GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) (cherry picked from commit d3ea82aaf940167482df1e08d6482de8f2dd8526) Co-authored-by: Filipe La?ns <lains at riseup.net> files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index fa18d62d22af..0049b663973f 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -121,7 +121,7 @@ identifier. Python currently uses eight paths: Return the default scheme name for the current platform. - .. versionchanged:: 3.10 + .. versionadded:: 3.10 This function was previously named ``_get_default_scheme()`` and considered an implementation detail. From webhook-mailer at python.org Tue Dec 13 06:03:28 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 13 Dec 2022 11:03:28 -0000 Subject: [Python-checkins] GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) Message-ID: <mailman.2919.1670929409.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f5f68ff1924b5588a7f4b2dcc6b17ccf419859f6 commit: f5f68ff1924b5588a7f4b2dcc6b17ccf419859f6 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-13T03:03:22-08:00 summary: GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) (cherry picked from commit d3ea82aaf940167482df1e08d6482de8f2dd8526) Co-authored-by: Filipe La?ns <lains at riseup.net> files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 7ef3b2489673..f42a5345357c 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -113,7 +113,7 @@ identifier. Python currently uses eight paths: Return the default scheme name for the current platform. - .. versionchanged:: 3.10 + .. versionadded:: 3.10 This function was previously named ``_get_default_scheme()`` and considered an implementation detail. From webhook-mailer at python.org Tue Dec 13 08:55:16 2022 From: webhook-mailer at python.org (iritkatriel) Date: Tue, 13 Dec 2022 13:55:16 -0000 Subject: [Python-checkins] gh-99955: undef ERROR and SUCCESS before redefining (fixes sanitizer warning) (#100215) Message-ID: <mailman.2920.1670939717.3313.python-checkins@python.org> https://github.com/python/cpython/commit/985a71032bf055240e61259285a0b4925c925383 commit: 985a71032bf055240e61259285a0b4925c925383 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-13T13:55:10Z summary: gh-99955: undef ERROR and SUCCESS before redefining (fixes sanitizer warning) (#100215) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 17b164a4d06e..813e0d5503b4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -55,6 +55,8 @@ */ #define STACK_USE_GUIDELINE 30 +#undef SUCCESS +#undef ERROR #define SUCCESS 0 #define ERROR -1 From webhook-mailer at python.org Wed Dec 14 06:12:59 2022 From: webhook-mailer at python.org (markshannon) Date: Wed, 14 Dec 2022 11:12:59 -0000 Subject: [Python-checkins] GH-100222: Redefine _Py_CODEUNIT as a union to clarify structure of code unit. (GH-100223) Message-ID: <mailman.2921.1671016380.3313.python-checkins@python.org> https://github.com/python/cpython/commit/6997e77bdf2297375962aaf82876da4e7ecdd61a commit: 6997e77bdf2297375962aaf82876da4e7ecdd61a branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-14T11:12:53Z summary: GH-100222: Redefine _Py_CODEUNIT as a union to clarify structure of code unit. (GH-100223) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst M Include/cpython/code.h M Include/internal/pycore_code.h M Objects/codeobject.c M Python/ceval.c M Python/compile.c M Python/generated_cases.c.h M Python/specialize.c M Tools/cases_generator/generate_cases.py diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fc7c5ed70243..6a13ff2dfd9f 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -16,21 +16,24 @@ extern "C" { * 2**32 - 1, rather than INT_MAX. */ -typedef uint16_t _Py_CODEUNIT; - -#ifdef WORDS_BIGENDIAN -# define _Py_OPCODE(word) ((word) >> 8) -# define _Py_OPARG(word) ((word) & 255) -# define _Py_MAKECODEUNIT(opcode, oparg) (((opcode)<<8)|(oparg)) -#else -# define _Py_OPCODE(word) ((word) & 255) -# define _Py_OPARG(word) ((word) >> 8) -# define _Py_MAKECODEUNIT(opcode, oparg) ((opcode)|((oparg)<<8)) -#endif +typedef union { + uint16_t cache; + struct { + uint8_t opcode; + uint8_t oparg; + }; +} _Py_CODEUNIT; + +#define _Py_OPCODE(word) ((word).opcode) +#define _Py_OPARG(word) ((word).oparg) + +static inline void +_py_set_opocde(_Py_CODEUNIT *word, uint8_t opcode) +{ + word->opcode = opcode; +} -// Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing: -#define _Py_SET_OPCODE(word, opcode) \ - do { ((unsigned char *)&(word))[0] = (opcode); } while (0) +#define _Py_SET_OPCODE(word, opcode) _py_set_opocde(&(word), opcode) typedef struct { PyObject *_co_code; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index f22fd45f8319..9e59fc98bf3d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -18,53 +18,53 @@ extern "C" { #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT index; - _Py_CODEUNIT module_keys_version[2]; - _Py_CODEUNIT builtin_keys_version; + uint16_t counter; + uint16_t index; + uint16_t module_keys_version[2]; + uint16_t builtin_keys_version; } _PyLoadGlobalCache; #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyUnpackSequenceCache; #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ CACHE_ENTRIES(_PyUnpackSequenceCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT mask; + uint16_t counter; + uint16_t mask; } _PyCompareOpCache; #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT type_version[2]; - _Py_CODEUNIT func_version; + uint16_t counter; + uint16_t type_version[2]; + uint16_t func_version; } _PyBinarySubscrCache; #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT version[2]; - _Py_CODEUNIT index; + uint16_t counter; + uint16_t version[2]; + uint16_t index; } _PyAttrCache; typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT type_version[2]; - _Py_CODEUNIT keys_version[2]; - _Py_CODEUNIT descr[4]; + uint16_t counter; + uint16_t type_version[2]; + uint16_t keys_version[2]; + uint16_t descr[4]; } _PyLoadMethodCache; @@ -74,21 +74,21 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT func_version[2]; - _Py_CODEUNIT min_args; + uint16_t counter; + uint16_t func_version[2]; + uint16_t min_args; } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyForIterCache; #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) @@ -409,7 +409,7 @@ write_location_entry_start(uint8_t *ptr, int code, int length) static inline uint16_t adaptive_counter_bits(int value, int backoff) { return (value << ADAPTIVE_BACKOFF_BITS) | - (backoff & ((1<<ADAPTIVE_BACKOFF_BITS)-1)); + (backoff & ((1<<ADAPTIVE_BACKOFF_BITS)-1)); } static inline uint16_t diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst new file mode 100644 index 000000000000..032b49420d8b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst @@ -0,0 +1,2 @@ +Redefine the ``_Py_CODEUNIT`` typedef as a union to describe its layout to +the C compiler, avoiding type punning and improving clarity. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index c92c7deaf808..f455cc603aae 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1520,9 +1520,10 @@ deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len) _Py_CODEUNIT instruction = instructions[i]; int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; int caches = _PyOpcode_Caches[opcode]; - instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); + instructions[i].opcode = opcode; while (caches--) { - instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + instructions[++i].opcode = CACHE; + instructions[i].oparg = 0; } } } @@ -1775,9 +1776,9 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - _Py_SET_OPCODE(co_instr, _PyOpcode_Deopt[_Py_OPCODE(co_instr)]); - _Py_SET_OPCODE(cp_instr, _PyOpcode_Deopt[_Py_OPCODE(cp_instr)]); - eq = co_instr == cp_instr; + co_instr.opcode = _PyOpcode_Deopt[_Py_OPCODE(co_instr)]; + cp_instr.opcode =_PyOpcode_Deopt[_Py_OPCODE(cp_instr)]; + eq = co_instr.cache == cp_instr.cache; if (!eq) { goto unequal; } diff --git a/Python/ceval.c b/Python/ceval.c index 9e4179e56071..45f42800d7ce 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -864,7 +864,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { STAT_INC(opcode, miss); \ STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ - if (ADAPTIVE_COUNTER_IS_ZERO(*next_instr)) { \ + if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ else { \ @@ -1289,7 +1289,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } opcode = _PyOpcode_Deopt[opcode]; if (_PyOpcode_Caches[opcode]) { - _Py_CODEUNIT *counter = &next_instr[1]; + uint16_t *counter = &next_instr[1].cache; // The instruction is going to decrement the counter, so we need to // increment it here to make sure it doesn't try to specialize: if (!ADAPTIVE_COUNTER_IS_MAX(*counter)) { diff --git a/Python/compile.c b/Python/compile.c index 813e0d5503b4..09eb4016940d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -263,22 +263,32 @@ write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: - *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF); + codestr->opcode = EXTENDED_ARG; + codestr->oparg = (oparg >> 24) & 0xFF; + codestr++; /* fall through */ case 3: - *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 16) & 0xFF); + codestr->opcode = EXTENDED_ARG; + codestr->oparg = (oparg >> 16) & 0xFF; + codestr++; /* fall through */ case 2: - *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 8) & 0xFF); + codestr->opcode = EXTENDED_ARG; + codestr->oparg = (oparg >> 8) & 0xFF; + codestr++; /* fall through */ case 1: - *codestr++ = _Py_MAKECODEUNIT(opcode, oparg & 0xFF); + codestr->opcode = opcode; + codestr->oparg = oparg & 0xFF; + codestr++; break; default: Py_UNREACHABLE(); } while (caches--) { - *codestr++ = _Py_MAKECODEUNIT(CACHE, 0); + codestr->opcode = CACHE; + codestr->oparg = 0; + codestr++; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 45382a466b1c..63635fbfc2f4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -562,8 +562,8 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = PEEK(1); PyObject *container = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t func_version = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t func_version = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -612,7 +612,7 @@ PyObject *sub = PEEK(1); PyObject *container = PEEK(2); PyObject *v = PEEK(3); - uint16_t counter = read_u16(next_instr + 0); + uint16_t counter = read_u16(&next_instr[0].cache); if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); next_instr--; @@ -1249,7 +1249,7 @@ PREDICTED(STORE_ATTR); PyObject *owner = PEEK(1); PyObject *v = PEEK(2); - uint16_t counter = read_u16(next_instr + 0); + uint16_t counter = read_u16(&next_instr[0].cache); if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg); @@ -2083,8 +2083,8 @@ TARGET(STORE_ATTR_INSTANCE_VALUE) { PyObject *owner = PEEK(1); PyObject *value = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t index = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2111,8 +2111,8 @@ TARGET(STORE_ATTR_WITH_HINT) { PyObject *owner = PEEK(1); PyObject *value = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t hint = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t hint = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2160,8 +2160,8 @@ TARGET(STORE_ATTR_SLOT) { PyObject *owner = PEEK(1); PyObject *value = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t index = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2209,7 +2209,7 @@ PyObject *right = _tmp_1; PyObject *left = _tmp_2; size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); + uint16_t when_to_jump_mask = read_u16(&next_instr[1].cache); assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); @@ -2247,7 +2247,7 @@ PyObject *right = _tmp_1; PyObject *left = _tmp_2; size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); + uint16_t when_to_jump_mask = read_u16(&next_instr[1].cache); assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); @@ -2286,7 +2286,7 @@ PyObject *right = _tmp_1; PyObject *left = _tmp_2; size_t jump; - uint16_t invert = read_u16(next_instr + 1); + uint16_t invert = read_u16(&next_instr[1].cache); assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); diff --git a/Python/specialize.c b/Python/specialize.c index 785088eac8c5..678c5d66b660 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -267,26 +267,26 @@ _PyCode_Quicken(PyCodeObject *code) int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; int caches = _PyOpcode_Caches[opcode]; if (caches) { - instructions[i + 1] = adaptive_counter_warmup(); + instructions[i + 1].cache = adaptive_counter_warmup(); previous_opcode = 0; i += caches; continue; } switch (previous_opcode << 8 | opcode) { case LOAD_CONST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_CONST__LOAD_FAST); + instructions[i - 1].opcode = LOAD_CONST__LOAD_FAST; break; case LOAD_FAST << 8 | LOAD_CONST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_CONST); + instructions[i - 1].opcode = LOAD_FAST__LOAD_CONST; break; case LOAD_FAST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_FAST); + instructions[i - 1].opcode = LOAD_FAST__LOAD_FAST; break; case STORE_FAST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__LOAD_FAST); + instructions[i - 1].opcode = STORE_FAST__LOAD_FAST; break; case STORE_FAST << 8 | STORE_FAST: - _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__STORE_FAST); + instructions[i - 1].opcode = STORE_FAST__STORE_FAST; break; } previous_opcode = opcode; @@ -482,7 +482,7 @@ specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - _Py_SET_OPCODE(*instr, opcode_module); + _py_set_opocde(instr, opcode_module); return 0; } @@ -634,7 +634,7 @@ specialize_dict_access( } write_u32(cache->version, type->tp_version_tag); cache->index = (uint16_t)index; - _Py_SET_OPCODE(*instr, values_op); + _py_set_opocde(instr, values_op); } else { PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); @@ -651,7 +651,7 @@ specialize_dict_access( } cache->index = (uint16_t)index; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, hint_op); + _py_set_opocde(instr, hint_op); } return 1; } @@ -730,7 +730,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ write_obj(lm_cache->descr, fget); - _Py_SET_OPCODE(*instr, LOAD_ATTR_PROPERTY); + _py_set_opocde(instr, LOAD_ATTR_PROPERTY); goto success; } case OBJECT_SLOT: @@ -754,7 +754,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT); + _py_set_opocde(instr, LOAD_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -763,7 +763,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT); + _py_set_opocde(instr, LOAD_ATTR_SLOT); goto success; } case OTHER_SLOT: @@ -791,7 +791,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) /* borrowed */ write_obj(lm_cache->descr, descr); write_u32(lm_cache->type_version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + _py_set_opocde(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); goto success; } case BUILTIN_CLASSMETHOD: @@ -809,7 +809,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, LOAD_ATTR); + _py_set_opocde(instr, LOAD_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -868,7 +868,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, STORE_ATTR_SLOT); + _py_set_opocde(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -897,7 +897,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, STORE_ATTR); + _py_set_opocde(instr, STORE_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -961,7 +961,7 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, case NON_DESCRIPTOR: write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); write_obj(cache->descr, descr); - _Py_SET_OPCODE(*instr, LOAD_ATTR_CLASS); + _py_set_opocde(instr, LOAD_ATTR_CLASS); return 0; #ifdef Py_STATS case ABSENT: @@ -1043,21 +1043,21 @@ PyObject *descr, DescriptorClassification kind) } switch(dictkind) { case NO_DICT: - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_NO_DICT); + _py_set_opocde(instr, LOAD_ATTR_METHOD_NO_DICT); break; case MANAGED_VALUES: - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_VALUES); + _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); goto fail; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_DICT); + _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_DICT); break; case LAZY_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_LAZY_DICT); + _py_set_opocde(instr, LOAD_ATTR_METHOD_LAZY_DICT); break; } /* `descr` is borrowed. This is safe for methods (even inherited ones from @@ -1114,7 +1114,7 @@ _Py_Specialize_LoadGlobal( } cache->index = (uint16_t)index; write_u32(cache->module_keys_version, keys_version); - _Py_SET_OPCODE(*instr, LOAD_GLOBAL_MODULE); + _py_set_opocde(instr, LOAD_GLOBAL_MODULE); goto success; } if (!PyDict_CheckExact(builtins)) { @@ -1150,12 +1150,12 @@ _Py_Specialize_LoadGlobal( cache->index = (uint16_t)index; write_u32(cache->module_keys_version, globals_version); cache->builtin_keys_version = (uint16_t)builtins_version; - _Py_SET_OPCODE(*instr, LOAD_GLOBAL_BUILTIN); + _py_set_opocde(instr, LOAD_GLOBAL_BUILTIN); goto success; fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, LOAD_GLOBAL); + _py_set_opocde(instr, LOAD_GLOBAL); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1259,7 +1259,7 @@ _Py_Specialize_BinarySubscr( PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_LIST_INT); + _py_set_opocde(instr, BINARY_SUBSCR_LIST_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1268,7 +1268,7 @@ _Py_Specialize_BinarySubscr( } if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_TUPLE_INT); + _py_set_opocde(instr, BINARY_SUBSCR_TUPLE_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1276,7 +1276,7 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_DICT); + _py_set_opocde(instr, BINARY_SUBSCR_DICT); goto success; } PyTypeObject *cls = Py_TYPE(container); @@ -1307,7 +1307,7 @@ _Py_Specialize_BinarySubscr( } cache->func_version = version; ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor; - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_GETITEM); + _py_set_opocde(instr, BINARY_SUBSCR_GETITEM); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1315,7 +1315,7 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, BINARY_SUBSCR); + _py_set_opocde(instr, BINARY_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1334,7 +1334,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) && ((PyLongObject *)sub)->ob_digit[0] < (size_t)PyList_GET_SIZE(container)) { - _Py_SET_OPCODE(*instr, STORE_SUBSCR_LIST_INT); + _py_set_opocde(instr, STORE_SUBSCR_LIST_INT); goto success; } else { @@ -1352,7 +1352,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins } } if (container_type == &PyDict_Type) { - _Py_SET_OPCODE(*instr, STORE_SUBSCR_DICT); + _py_set_opocde(instr, STORE_SUBSCR_DICT); goto success; } #ifdef Py_STATS @@ -1419,7 +1419,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, STORE_SUBSCR); + _py_set_opocde(instr, STORE_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1441,20 +1441,20 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, int oparg = _Py_OPARG(*instr); if (nargs == 1 && kwnames == NULL && oparg == 1) { if (tp == &PyUnicode_Type) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_STR_1); + _py_set_opocde(instr, CALL_NO_KW_STR_1); return 0; } else if (tp == &PyType_Type) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_TYPE_1); + _py_set_opocde(instr, CALL_NO_KW_TYPE_1); return 0; } else if (tp == &PyTuple_Type) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_TUPLE_1); + _py_set_opocde(instr, CALL_NO_KW_TUPLE_1); return 0; } } if (tp->tp_vectorcall != NULL) { - _Py_SET_OPCODE(*instr, CALL_BUILTIN_CLASS); + _py_set_opocde(instr, CALL_BUILTIN_CLASS); return 0; } SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? @@ -1506,7 +1506,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); + _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); return 0; } case METH_O: { @@ -1520,18 +1520,18 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (_Py_OPCODE(next) == POP_TOP); int oparg = _Py_OPARG(*instr); if ((PyObject *)descr == list_append && oparg == 1 && pop) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_LIST_APPEND); + _py_set_opocde(instr, CALL_NO_KW_LIST_APPEND); return 0; } - _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); + _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); return 0; } case METH_FASTCALL: { - _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); + _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); return 0; } case METH_FASTCALL|METH_KEYWORDS: { - _Py_SET_OPCODE(*instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + _py_set_opocde(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } @@ -1582,14 +1582,14 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, write_u32(cache->func_version, version); cache->min_args = min_args; if (argcount == nargs) { - _Py_SET_OPCODE(*instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); + _py_set_opocde(instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); } else if (bound_method) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); return -1; } else { - _Py_SET_OPCODE(*instr, CALL_PY_WITH_DEFAULTS); + _py_set_opocde(instr, CALL_PY_WITH_DEFAULTS); } return 0; } @@ -1616,10 +1616,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_LEN); + _py_set_opocde(instr, CALL_NO_KW_LEN); return 0; } - _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_O); + _py_set_opocde(instr, CALL_NO_KW_BUILTIN_O); return 0; } case METH_FASTCALL: { @@ -1631,15 +1631,15 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_ISINSTANCE); + _py_set_opocde(instr, CALL_NO_KW_ISINSTANCE); return 0; } } - _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_FAST); + _py_set_opocde(instr, CALL_NO_KW_BUILTIN_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - _Py_SET_OPCODE(*instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); + _py_set_opocde(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); return 0; } default: @@ -1732,7 +1732,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, CALL); + _py_set_opocde(instr, CALL); cache->counter = adaptive_counter_backoff(cache->counter); } else { @@ -1829,18 +1829,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, bool to_store = (_Py_OPCODE(next) == STORE_FAST || _Py_OPCODE(next) == STORE_FAST__LOAD_FAST); if (to_store && locals[_Py_OPARG(next)] == lhs) { - _Py_SET_OPCODE(*instr, BINARY_OP_INPLACE_ADD_UNICODE); + _py_set_opocde(instr, BINARY_OP_INPLACE_ADD_UNICODE); goto success; } - _Py_SET_OPCODE(*instr, BINARY_OP_ADD_UNICODE); + _py_set_opocde(instr, BINARY_OP_ADD_UNICODE); goto success; } if (PyLong_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_ADD_INT); + _py_set_opocde(instr, BINARY_OP_ADD_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_ADD_FLOAT); + _py_set_opocde(instr, BINARY_OP_ADD_FLOAT); goto success; } break; @@ -1850,11 +1850,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_INT); + _py_set_opocde(instr, BINARY_OP_MULTIPLY_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_FLOAT); + _py_set_opocde(instr, BINARY_OP_MULTIPLY_FLOAT); goto success; } break; @@ -1864,18 +1864,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_INT); + _py_set_opocde(instr, BINARY_OP_SUBTRACT_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_FLOAT); + _py_set_opocde(instr, BINARY_OP_SUBTRACT_FLOAT); goto success; } break; } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - _Py_SET_OPCODE(*instr, BINARY_OP); + _py_set_opocde(instr, BINARY_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1957,13 +1957,13 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, COMPARE_OP_FLOAT_JUMP); + _py_set_opocde(instr, COMPARE_OP_FLOAT_JUMP); cache->mask = when_to_jump_mask; goto success; } if (PyLong_CheckExact(lhs)) { if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) { - _Py_SET_OPCODE(*instr, COMPARE_OP_INT_JUMP); + _py_set_opocde(instr, COMPARE_OP_INT_JUMP); cache->mask = when_to_jump_mask; goto success; } @@ -1978,7 +1978,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } else { - _Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP); + _py_set_opocde(instr, COMPARE_OP_STR_JUMP); cache->mask = (when_to_jump_mask & 2) == 0; goto success; } @@ -1986,7 +1986,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); - _Py_SET_OPCODE(*instr, COMPARE_OP); + _py_set_opocde(instr, COMPARE_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2020,10 +2020,10 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) goto failure; } if (PyTuple_GET_SIZE(seq) == 2) { - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TWO_TUPLE); + _py_set_opocde(instr, UNPACK_SEQUENCE_TWO_TUPLE); goto success; } - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TUPLE); + _py_set_opocde(instr, UNPACK_SEQUENCE_TUPLE); goto success; } if (PyList_CheckExact(seq)) { @@ -2031,13 +2031,13 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); goto failure; } - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_LIST); + _py_set_opocde(instr, UNPACK_SEQUENCE_LIST); goto success; } SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE); + _py_set_opocde(instr, UNPACK_SEQUENCE); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2126,26 +2126,26 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; int next_op = _PyOpcode_Deopt[_Py_OPCODE(next)]; if (tp == &PyListIter_Type) { - _Py_SET_OPCODE(*instr, FOR_ITER_LIST); + _py_set_opocde(instr, FOR_ITER_LIST); goto success; } else if (tp == &PyTupleIter_Type) { - _Py_SET_OPCODE(*instr, FOR_ITER_TUPLE); + _py_set_opocde(instr, FOR_ITER_TUPLE); goto success; } else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { - _Py_SET_OPCODE(*instr, FOR_ITER_RANGE); + _py_set_opocde(instr, FOR_ITER_RANGE); goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { assert(_Py_OPCODE(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1]) == END_FOR); - _Py_SET_OPCODE(*instr, FOR_ITER_GEN); + _py_set_opocde(instr, FOR_ITER_GEN); goto success; } SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); - _Py_SET_OPCODE(*instr, FOR_ITER); + _py_set_opocde(instr, FOR_ITER); cache->counter = adaptive_counter_backoff(cache->counter); return; success: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 5930c797b8a4..2dfc76f2560e 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -205,7 +205,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None else: typ = f"uint{bits}_t " func = f"read_u{bits}" - out.emit(f"{typ}{ceffect.name} = {func}(next_instr + {cache_offset});") + out.emit(f"{typ}{ceffect.name} = {func}(&next_instr[{cache_offset}].cache);") cache_offset += ceffect.size assert cache_offset == self.cache_offset + cache_adjust From webhook-mailer at python.org Wed Dec 14 06:37:26 2022 From: webhook-mailer at python.org (vsajip) Date: Wed, 14 Dec 2022 11:37:26 -0000 Subject: [Python-checkins] gh-100176: venv: Remove redundant compat code for Python <= 3.2 (#100177) Message-ID: <mailman.2922.1671017848.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3192c00a3cf136e06592d9a14d4d7b82412da4de commit: 3192c00a3cf136e06592d9a14d4d7b82412da4de branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: vsajip <vinay_sajip at yahoo.co.uk> date: 2022-12-14T11:37:11Z summary: gh-100176: venv: Remove redundant compat code for Python <= 3.2 (#100177) gh-100176: Remove redundant compat code for Python 3.2 and older Python 3.2 has been EOL since 2016-02-20 and 2.7 since 2020-01-01, so we can remove this old compatibility check and unindent the old else-block. Also, in the unindented block, replace a .format() call with an f-string. Plus similar changes in the documentation. files: M Doc/library/venv.rst M Lib/venv/__init__.py diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index adc6cd339ac1..2a41096de006 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -497,76 +497,68 @@ subclass which installs setuptools and pip into a created virtual environment:: url = 'https://bootstrap.pypa.io/get-pip.py' self.install_script(context, 'pip', url) + def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory in which to create the ' + 'virtual environment.') + parser.add_argument('--no-setuptools', default=False, + action='store_true', dest='nodist', + help="Don't install setuptools or pip in the " + "virtual environment.") + parser.add_argument('--no-pip', default=False, + action='store_true', dest='nopip', + help="Don't install pip in the virtual " + "environment.") + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the ' - 'virtual environment.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'virtual environment ' - 'directory if it already ' - 'exists, before virtual ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the virtual ' - 'environment directory to ' - 'use this version of ' - 'Python, assuming Python ' - 'has been upgraded ' - 'in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) + use_symlinks = True + parser.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'virtual environment ' + 'directory if it already ' + 'exists, before virtual ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the virtual ' + 'environment directory to ' + 'use this version of ' + 'Python, assuming Python ' + 'has been upgraded ' + 'in-place.') + parser.add_argument('--verbose', default=False, action='store_true', + dest='verbose', help='Display the output ' + 'from the scripts which ' + 'install setuptools and pip.') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = ExtendedEnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + nodist=options.nodist, + nopip=options.nopip, + verbose=options.verbose) + for d in options.dirs: + builder.create(d) if __name__ == '__main__': rc = 1 diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 7bfc2d1b6fe0..978c98336f2b 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -223,7 +223,7 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): force_copy = not self.symlinks if not force_copy: try: - if not os.path.islink(dst): # can't link to itself! + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) @@ -418,11 +418,11 @@ def install_scripts(self, context, path): binpath = context.bin_path plen = len(path) for root, dirs, files in os.walk(path): - if root == path: # at top-level, remove irrelevant dirs + if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: if d not in ('common', os.name): dirs.remove(d) - continue # ignore files in top level + continue # ignore files in top level for f in files: if (os.name == 'nt' and f.startswith('python') and f.endswith(('.exe', '.pdb'))): @@ -468,83 +468,76 @@ def create(env_dir, system_site_packages=False, clear=False, prompt=prompt, upgrade_deps=upgrade_deps) builder.create(env_dir) + def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with Python >= 3.3') + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.', + epilog='Once an environment has been ' + 'created, you may wish to ' + 'activate it, e.g. by ' + 'sourcing an activate script ' + 'in its bin directory.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory to create the environment in.') + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.', - epilog='Once an environment has been ' - 'created, you may wish to ' - 'activate it, e.g. by ' - 'sourcing an activate script ' - 'in its bin directory.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory to create the environment in.') - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - group = parser.add_mutually_exclusive_group() - group.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - group.add_argument('--copies', default=not use_symlinks, - action='store_false', dest='symlinks', - help='Try to use copies rather than symlinks, ' - 'even when symlinks are the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'environment directory if it ' - 'already exists, before ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the environment ' - 'directory to use this version ' - 'of Python, assuming Python ' - 'has been upgraded in-place.') - parser.add_argument('--without-pip', dest='with_pip', - default=True, action='store_false', - help='Skips installing or upgrading pip in the ' - 'virtual environment (pip is bootstrapped ' - 'by default)') - parser.add_argument('--prompt', - help='Provides an alternative prompt prefix for ' - 'this environment.') - parser.add_argument('--upgrade-deps', default=False, action='store_true', - dest='upgrade_deps', - help='Upgrade core dependencies: {} to the latest ' - 'version in PyPI'.format( - ' '.join(CORE_VENV_DEPS))) - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = EnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - with_pip=options.with_pip, - prompt=options.prompt, - upgrade_deps=options.upgrade_deps) - for d in options.dirs: - builder.create(d) + use_symlinks = True + group = parser.add_mutually_exclusive_group() + group.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + group.add_argument('--copies', default=not use_symlinks, + action='store_false', dest='symlinks', + help='Try to use copies rather than symlinks, ' + 'even when symlinks are the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'environment directory if it ' + 'already exists, before ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the environment ' + 'directory to use this version ' + 'of Python, assuming Python ' + 'has been upgraded in-place.') + parser.add_argument('--without-pip', dest='with_pip', + default=True, action='store_false', + help='Skips installing or upgrading pip in the ' + 'virtual environment (pip is bootstrapped ' + 'by default)') + parser.add_argument('--prompt', + help='Provides an alternative prompt prefix for ' + 'this environment.') + parser.add_argument('--upgrade-deps', default=False, action='store_true', + dest='upgrade_deps', + help=f'Upgrade core dependencies: {" ".join(CORE_VENV_DEPS)} ' + 'to the latest version in PyPI') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = EnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + with_pip=options.with_pip, + prompt=options.prompt, + upgrade_deps=options.upgrade_deps) + for d in options.dirs: + builder.create(d) + if __name__ == '__main__': rc = 1 From webhook-mailer at python.org Wed Dec 14 10:50:42 2022 From: webhook-mailer at python.org (markshannon) Date: Wed, 14 Dec 2022 15:50:42 -0000 Subject: [Python-checkins] Assorted minor fixes for specialization stats. (GH-100219) Message-ID: <mailman.2923.1671033042.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5693f45b19de47f8cb2f08d3baf43e626d3fbdf3 commit: 5693f45b19de47f8cb2f08d3baf43e626d3fbdf3 branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-14T15:50:02Z summary: Assorted minor fixes for specialization stats. (GH-100219) files: M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Python/specialize.c b/Python/specialize.c index 678c5d66b660..4fa2c2ffc04b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -307,6 +307,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_NOT_PY_FUNCTION 7 +#define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17 #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18 /* Attributes */ @@ -332,6 +333,8 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 +#define SPEC_FAIL_ATTR_NOT_IN_KEYS 29 +#define SPEC_FAIL_ATTR_NOT_IN_DICT 30 /* Binary subscr and store subscr */ @@ -448,41 +451,44 @@ static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); static int -specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr, - PyObject *name, int opcode, int opcode_module) -{ +specialize_module_load_attr( + PyObject *owner, _Py_CODEUNIT *instr, PyObject *name +) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyModuleObject *m = (PyModuleObject *)owner; assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); PyDictObject *dict = (PyDictObject *)m->md_dict; if (dict == NULL) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT); return -1; } if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); return -1; } Py_ssize_t index = _PyDict_LookupIndex(dict, &_Py_ID(__getattr__)); assert(index != DKIX_ERROR); if (index != DKIX_EMPTY) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND); return -1; } index = _PyDict_LookupIndex(dict, name); assert (index != DKIX_ERROR); if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(LOAD_ATTR, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND : + SPEC_FAIL_OUT_OF_RANGE); return -1; } uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys); if (keys_version == 0) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - _py_set_opocde(instr, opcode_module); + _py_set_opocde(instr, LOAD_ATTR_MODULE); return 0; } @@ -629,7 +635,10 @@ specialize_dict_access( Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); assert (index != DKIX_ERROR); if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_KEYS : + SPEC_FAIL_OUT_OF_RANGE); return 0; } write_u32(cache->version, type->tp_version_tag); @@ -646,7 +655,10 @@ specialize_dict_access( Py_ssize_t index = _PyDict_LookupIndex(dict, name); if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_DICT : + SPEC_FAIL_OUT_OF_RANGE); return 0; } cache->index = (uint16_t)index; @@ -674,8 +686,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) goto fail; } if (PyModule_CheckExact(owner)) { - if (specialize_module_load_attr(owner, instr, name, LOAD_ATTR, - LOAD_ATTR_MODULE)) + if (specialize_module_load_attr(owner, instr, name)) { goto fail; } @@ -702,7 +713,9 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) goto success; } } - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + else { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + } goto fail; } case PROPERTY: @@ -1076,7 +1089,6 @@ PyObject *descr, DescriptorClassification kind) */ write_u32(cache->type_version, owner_cls->tp_version_tag); write_obj(cache->descr, descr); - // Fall through. return 1; fail: return 0; @@ -1092,6 +1104,7 @@ _Py_Specialize_LoadGlobal( _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1); assert(PyUnicode_CheckExact(name)); if (!PyDict_CheckExact(globals)) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT); goto fail; } PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys; @@ -1101,15 +1114,17 @@ _Py_Specialize_LoadGlobal( } Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name); if (index == DKIX_ERROR) { - SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } if (index != DKIX_EMPTY) { if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); if (keys_version == 0) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } cache->index = (uint16_t)index; @@ -1118,6 +1133,7 @@ _Py_Specialize_LoadGlobal( goto success; } if (!PyDict_CheckExact(builtins)) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT); goto fail; } PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys; @@ -1127,10 +1143,11 @@ _Py_Specialize_LoadGlobal( } index = _PyDictKeys_StringLookup(builtin_keys, name); if (index == DKIX_ERROR) { - SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index c30a60e9514b..81b06f9f7469 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -32,7 +32,7 @@ opmap = {name: i for i, name in enumerate(opname)} opmap = dict(sorted(opmap.items())) -TOTAL = "specialization.deferred", "specialization.hit", "specialization.miss", "execution_count" +TOTAL = "specialization.hit", "specialization.miss", "execution_count" def format_ratio(num, den): """ @@ -90,7 +90,7 @@ def calculate_specialization_stats(family_stats, total): if key in ("specialization.hit", "specialization.miss"): label = key[len("specialization."):] elif key == "execution_count": - label = "unquickened" + continue elif key in ("specialization.success", "specialization.failure", "specializable"): continue elif key.startswith("pair"): @@ -115,7 +115,7 @@ def calculate_specialization_success_failure(family_stats): def calculate_specialization_failure_kinds(name, family_stats, defines): total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [ 0 ] * 30 + failure_kinds = [ 0 ] * 40 for key in family_stats: if not key.startswith("specialization.failure_kind"): continue @@ -224,7 +224,7 @@ def pretty(defname): return defname.replace("_", " ").lower() def kind_to_text(kind, defines, opname): - if kind < 7: + if kind <= 7: return pretty(defines[kind][0]) if opname.endswith("ATTR"): opname = "ATTR" @@ -241,10 +241,7 @@ def categorized_counts(opcode_stats): not_specialized = 0 specialized_instructions = { op for op in opcode._specialized_instructions - if "__" not in op and "ADAPTIVE" not in op} - adaptive_instructions = { - op for op in opcode._specialized_instructions - if "ADAPTIVE" in op} + if "__" not in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: continue @@ -252,8 +249,6 @@ def categorized_counts(opcode_stats): name = opname[i] if "specializable" in opcode_stat: not_specialized += count - elif name in adaptive_instructions: - not_specialized += count elif name in specialized_instructions: miss = opcode_stat.get("specialization.miss", 0) not_specialized += miss From webhook-mailer at python.org Wed Dec 14 11:57:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 14 Dec 2022 16:57:32 -0000 Subject: [Python-checkins] gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (#100249) Message-ID: <mailman.2924.1671037053.3313.python-checkins@python.org> https://github.com/python/cpython/commit/96638538002fc6e209755c06f613b0a59eb91166 commit: 96638538002fc6e209755c06f613b0a59eb91166 branch: main author: busywhitespace <busywhitespace at tuta.io> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-14T22:27:02+05:30 summary: gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (#100249) files: M Doc/library/asyncio-stream.rst diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index d87e3c042c99..c1ae8abb9abc 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -52,6 +52,7 @@ and work with streams: limit=None, ssl=None, family=0, proto=0, \ flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ happy_eyeballs_delay=None, interleave=None) Establish a network connection and return a pair of @@ -82,6 +83,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_server(client_connected_cb, host=None, \ port=None, *, limit=None, \ @@ -89,7 +93,7 @@ and work with streams: flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ reuse_port=None, ssl_handshake_timeout=None, \ - start_serving=True) + ssl_shutdown_timeout=None, start_serving=True) Start a socket server. @@ -121,12 +125,15 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. rubric:: Unix Sockets .. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \ ssl=None, sock=None, server_hostname=None, \ - ssl_handshake_timeout=None) + ssl_handshake_timeout=None, ssl_shutdown_timeout=None) Establish a Unix socket connection and return a pair of ``(reader, writer)``. @@ -150,10 +157,14 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ *, limit=None, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, start_serving=True) + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, start_serving=True) Start a Unix socket server. @@ -176,6 +187,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + StreamReader ============ From webhook-mailer at python.org Wed Dec 14 12:07:42 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 14 Dec 2022 17:07:42 -0000 Subject: [Python-checkins] gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (GH-100249) Message-ID: <mailman.2925.1671037663.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cb1001ce23205eeed69395cdbfdb99e99a37aca1 commit: cb1001ce23205eeed69395cdbfdb99e99a37aca1 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-14T09:07:36-08:00 summary: gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (GH-100249) (cherry picked from commit 96638538002fc6e209755c06f613b0a59eb91166) Co-authored-by: busywhitespace <busywhitespace at tuta.io> files: M Doc/library/asyncio-stream.rst diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index d87e3c042c99..c1ae8abb9abc 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -52,6 +52,7 @@ and work with streams: limit=None, ssl=None, family=0, proto=0, \ flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ happy_eyeballs_delay=None, interleave=None) Establish a network connection and return a pair of @@ -82,6 +83,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_server(client_connected_cb, host=None, \ port=None, *, limit=None, \ @@ -89,7 +93,7 @@ and work with streams: flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ reuse_port=None, ssl_handshake_timeout=None, \ - start_serving=True) + ssl_shutdown_timeout=None, start_serving=True) Start a socket server. @@ -121,12 +125,15 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. rubric:: Unix Sockets .. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \ ssl=None, sock=None, server_hostname=None, \ - ssl_handshake_timeout=None) + ssl_handshake_timeout=None, ssl_shutdown_timeout=None) Establish a Unix socket connection and return a pair of ``(reader, writer)``. @@ -150,10 +157,14 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ *, limit=None, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, start_serving=True) + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, start_serving=True) Start a Unix socket server. @@ -176,6 +187,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + StreamReader ============ From webhook-mailer at python.org Wed Dec 14 13:54:23 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Wed, 14 Dec 2022 18:54:23 -0000 Subject: [Python-checkins] gh-90111: Minor Cleanup for Runtime-Global Objects (gh-100254) Message-ID: <mailman.2926.1671044064.3313.python-checkins@python.org> https://github.com/python/cpython/commit/aa8591e9ca9a3e55e96a03c17fa9d3902f0674dc commit: aa8591e9ca9a3e55e96a03c17fa9d3902f0674dc branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-14T11:53:57-07:00 summary: gh-90111: Minor Cleanup for Runtime-Global Objects (gh-100254) * move _PyRuntime.global_objects.interned to _PyRuntime.cached_objects.interned_strings (and use _Py_CACHED_OBJECT()) * rename _PyRuntime.global_objects to _PyRuntime.static_objects (This also relates to gh-96075.) https://github.com/python/cpython/issues/90111 files: M Include/internal/pycore_global_objects.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Objects/unicodeobject.c diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 80f6bb2c8a7c..d0461fa7e82e 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -28,14 +28,16 @@ extern "C" { struct _Py_cached_objects { PyObject *str_replace_inf; + + PyObject *interned_strings; }; #define _Py_GLOBAL_OBJECT(NAME) \ - _PyRuntime.global_objects.NAME + _PyRuntime.static_objects.NAME #define _Py_SINGLETON(NAME) \ _Py_GLOBAL_OBJECT(singletons.NAME) -struct _Py_global_objects { +struct _Py_static_objects { struct { /* Small integers are preallocated in this array so that they * can be shared. @@ -59,8 +61,6 @@ struct _Py_global_objects { PyHamtNode_Bitmap hamt_bitmap_node_empty; _PyContextTokenMissing context_token_missing; } singletons; - - PyObject *interned; }; #define _Py_INTERP_CACHED_OBJECT(interp, NAME) \ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 92ed45956c99..d100e836c7d1 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -163,7 +163,7 @@ typedef struct pyruntimestate { /* All the objects that are shared by the runtime's interpreters. */ struct _Py_cached_objects cached_objects; - struct _Py_global_objects global_objects; + struct _Py_static_objects static_objects; /* The following fields are here to avoid allocation during init. The data is exposed through _PyRuntimeState pointer fields. diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 1431096e2d24..6342a28f4df9 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -70,7 +70,7 @@ extern "C" { .types = { \ .next_version_tag = 1, \ }, \ - .global_objects = { \ + .static_objects = { \ .singletons = { \ .small_ints = _Py_small_ints_INIT, \ .bytes_empty = _PyBytes_SIMPLE_INIT(0, 0), \ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index b721ccd805ed..f0c7aa7707fd 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -233,12 +233,12 @@ static inline PyObject* unicode_new_empty(void) */ static inline PyObject *get_interned_dict(void) { - return _PyRuntime.global_objects.interned; + return _Py_CACHED_OBJECT(interned_strings); } static inline void set_interned_dict(PyObject *dict) { - _PyRuntime.global_objects.interned = dict; + _Py_CACHED_OBJECT(interned_strings) = dict; } #define _Py_RETURN_UNICODE_EMPTY() \ From webhook-mailer at python.org Wed Dec 14 14:14:26 2022 From: webhook-mailer at python.org (markshannon) Date: Wed, 14 Dec 2022 19:14:26 -0000 Subject: [Python-checkins] GH-100000: Cleanup and polish various watchers code (GH-99998) Message-ID: <mailman.2927.1671045267.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ae83c782155ffe86830c3255e338f366e331ad30 commit: ae83c782155ffe86830c3255e338f366e331ad30 branch: main author: Itamar Ostricher <itamarost at gmail.com> committer: markshannon <mark at hotpy.org> date: 2022-12-14T19:14:16Z summary: GH-100000: Cleanup and polish various watchers code (GH-99998) * Initialize `type_watchers` array to `NULL`s * Optimize code watchers notification * Optimize func watchers notification files: M Include/internal/pycore_interp.h M Modules/_testcapi/watchers.c M Objects/codeobject.c M Objects/funcobject.c M Objects/typeobject.c M Python/pystate.c diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ffda1351952d..0e3d46852f2e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -142,7 +142,6 @@ struct _is { // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; - PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS]; PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS]; // One bit is set for each non-NULL entry in func_watchers uint8_t active_func_watchers; diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 1d91c206f630..2e8fe1dbf786 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -630,14 +630,16 @@ static PyMethodDef test_methods[] = { {"clear_dict_watcher", clear_dict_watcher, METH_O, NULL}, {"watch_dict", watch_dict, METH_VARARGS, NULL}, {"unwatch_dict", unwatch_dict, METH_VARARGS, NULL}, - {"get_dict_watcher_events", get_dict_watcher_events, METH_NOARGS, NULL}, + {"get_dict_watcher_events", + (PyCFunction) get_dict_watcher_events, METH_NOARGS, NULL}, // Type watchers. {"add_type_watcher", add_type_watcher, METH_O, NULL}, {"clear_type_watcher", clear_type_watcher, METH_O, NULL}, {"watch_type", watch_type, METH_VARARGS, NULL}, {"unwatch_type", unwatch_type, METH_VARARGS, NULL}, - {"get_type_modified_events", get_type_modified_events, METH_NOARGS, NULL}, + {"get_type_modified_events", + (PyCFunction) get_type_modified_events, METH_NOARGS, NULL}, // Code object watchers. {"add_code_watcher", add_code_watcher, METH_O, NULL}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index f455cc603aae..1e5a92270be8 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -15,14 +15,21 @@ static void notify_code_watchers(PyCodeEvent event, PyCodeObject *co) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->active_code_watchers) { - assert(interp->_initialized); - for (int i = 0; i < CODE_MAX_WATCHERS; i++) { + assert(interp->_initialized); + uint8_t bits = interp->active_code_watchers; + int i = 0; + while (bits) { + assert(i < CODE_MAX_WATCHERS); + if (bits & 1) { PyCode_WatchCallback cb = interp->code_watchers[i]; - if ((cb != NULL) && (cb(event, co) < 0)) { + // callback must be non-null if the watcher bit is set + assert(cb != NULL); + if (cb(event, co) < 0) { PyErr_WriteUnraisable((PyObject *) co); } } + i++; + bits >>= 1; } } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 9df06520586a..d5cf5b9277b3 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -12,11 +12,20 @@ static void notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) { - for (int i = 0; i < FUNC_MAX_WATCHERS; i++) { - PyFunction_WatchCallback cb = interp->func_watchers[i]; - if ((cb != NULL) && (cb(event, func, new_value) < 0)) { - PyErr_WriteUnraisable((PyObject *) func); + uint8_t bits = interp->active_func_watchers; + int i = 0; + while (bits) { + assert(i < FUNC_MAX_WATCHERS); + if (bits & 1) { + PyFunction_WatchCallback cb = interp->func_watchers[i]; + // callback must be non-null if the watcher bit is set + assert(cb != NULL); + if (cb(event, func, new_value) < 0) { + PyErr_WriteUnraisable((PyObject *) func); + } } + i++; + bits >>= 1; } } @@ -25,6 +34,7 @@ handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) { PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); if (interp->active_func_watchers) { notify_func_watchers(interp, event, func, new_value); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ae80f5a8fd88..2c3b39521a6d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -485,23 +485,24 @@ PyType_Modified(PyTypeObject *type) } } + // Notify registered type watchers, if any if (type->tp_watched) { PyInterpreterState *interp = _PyInterpreterState_GET(); int bits = type->tp_watched; int i = 0; - while(bits && i < TYPE_MAX_WATCHERS) { + while (bits) { + assert(i < TYPE_MAX_WATCHERS); if (bits & 1) { PyType_WatchCallback cb = interp->type_watchers[i]; if (cb && (cb(type) < 0)) { PyErr_WriteUnraisable((PyObject *)type); } } - i += 1; + i++; bits >>= 1; } } - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; /* 0 is not a valid version tag */ } diff --git a/Python/pystate.c b/Python/pystate.c index ea3c22c5d71a..f52fc38b3586 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -461,6 +461,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->dict_state.watchers[i] = NULL; } + for (int i=0; i < TYPE_MAX_WATCHERS; i++) { + interp->type_watchers[i] = NULL; + } + for (int i=0; i < FUNC_MAX_WATCHERS; i++) { interp->func_watchers[i] = NULL; } From webhook-mailer at python.org Wed Dec 14 19:39:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 15 Dec 2022 00:39:11 -0000 Subject: [Python-checkins] GH-100222: fix typo _py_set_opocde -> _py_set_opcode (GH-100259) Message-ID: <mailman.2928.1671064752.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bdd86741bebd3efb51e540d5148e658cb34fd3ce commit: bdd86741bebd3efb51e540d5148e658cb34fd3ce branch: main author: Carl Meyer <carl at oddbird.net> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-14T16:39:00-08:00 summary: GH-100222: fix typo _py_set_opocde -> _py_set_opcode (GH-100259) Typo introduced in #100223. Automerge-Triggered-By: GH:brandtbucher files: M Include/cpython/code.h M Python/specialize.c diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 6a13ff2dfd9f..1c619322926e 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -28,7 +28,7 @@ typedef union { #define _Py_OPARG(word) ((word).oparg) static inline void -_py_set_opocde(_Py_CODEUNIT *word, uint8_t opcode) +_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) { word->opcode = opcode; } diff --git a/Python/specialize.c b/Python/specialize.c index 4fa2c2ffc04b..e88fdde79a67 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -488,7 +488,7 @@ specialize_module_load_attr( } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - _py_set_opocde(instr, LOAD_ATTR_MODULE); + _py_set_opcode(instr, LOAD_ATTR_MODULE); return 0; } @@ -643,7 +643,7 @@ specialize_dict_access( } write_u32(cache->version, type->tp_version_tag); cache->index = (uint16_t)index; - _py_set_opocde(instr, values_op); + _py_set_opcode(instr, values_op); } else { PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); @@ -663,7 +663,7 @@ specialize_dict_access( } cache->index = (uint16_t)index; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, hint_op); + _py_set_opcode(instr, hint_op); } return 1; } @@ -743,7 +743,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ write_obj(lm_cache->descr, fget); - _py_set_opocde(instr, LOAD_ATTR_PROPERTY); + _py_set_opcode(instr, LOAD_ATTR_PROPERTY); goto success; } case OBJECT_SLOT: @@ -767,7 +767,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, LOAD_ATTR_SLOT); + _py_set_opcode(instr, LOAD_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -776,7 +776,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, LOAD_ATTR_SLOT); + _py_set_opcode(instr, LOAD_ATTR_SLOT); goto success; } case OTHER_SLOT: @@ -804,7 +804,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) /* borrowed */ write_obj(lm_cache->descr, descr); write_u32(lm_cache->type_version, type->tp_version_tag); - _py_set_opocde(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + _py_set_opcode(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); goto success; } case BUILTIN_CLASSMETHOD: @@ -822,7 +822,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, LOAD_ATTR); + _py_set_opcode(instr, LOAD_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -881,7 +881,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, STORE_ATTR_SLOT); + _py_set_opcode(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -910,7 +910,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, STORE_ATTR); + _py_set_opcode(instr, STORE_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -974,7 +974,7 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, case NON_DESCRIPTOR: write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); write_obj(cache->descr, descr); - _py_set_opocde(instr, LOAD_ATTR_CLASS); + _py_set_opcode(instr, LOAD_ATTR_CLASS); return 0; #ifdef Py_STATS case ABSENT: @@ -1056,21 +1056,21 @@ PyObject *descr, DescriptorClassification kind) } switch(dictkind) { case NO_DICT: - _py_set_opocde(instr, LOAD_ATTR_METHOD_NO_DICT); + _py_set_opcode(instr, LOAD_ATTR_METHOD_NO_DICT); break; case MANAGED_VALUES: - _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_VALUES); + _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); goto fail; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_DICT); + _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_DICT); break; case LAZY_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _py_set_opocde(instr, LOAD_ATTR_METHOD_LAZY_DICT); + _py_set_opcode(instr, LOAD_ATTR_METHOD_LAZY_DICT); break; } /* `descr` is borrowed. This is safe for methods (even inherited ones from @@ -1129,7 +1129,7 @@ _Py_Specialize_LoadGlobal( } cache->index = (uint16_t)index; write_u32(cache->module_keys_version, keys_version); - _py_set_opocde(instr, LOAD_GLOBAL_MODULE); + _py_set_opcode(instr, LOAD_GLOBAL_MODULE); goto success; } if (!PyDict_CheckExact(builtins)) { @@ -1167,12 +1167,12 @@ _Py_Specialize_LoadGlobal( cache->index = (uint16_t)index; write_u32(cache->module_keys_version, globals_version); cache->builtin_keys_version = (uint16_t)builtins_version; - _py_set_opocde(instr, LOAD_GLOBAL_BUILTIN); + _py_set_opcode(instr, LOAD_GLOBAL_BUILTIN); goto success; fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, LOAD_GLOBAL); + _py_set_opcode(instr, LOAD_GLOBAL); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1276,7 +1276,7 @@ _Py_Specialize_BinarySubscr( PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opocde(instr, BINARY_SUBSCR_LIST_INT); + _py_set_opcode(instr, BINARY_SUBSCR_LIST_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1285,7 +1285,7 @@ _Py_Specialize_BinarySubscr( } if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opocde(instr, BINARY_SUBSCR_TUPLE_INT); + _py_set_opcode(instr, BINARY_SUBSCR_TUPLE_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1293,7 +1293,7 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - _py_set_opocde(instr, BINARY_SUBSCR_DICT); + _py_set_opcode(instr, BINARY_SUBSCR_DICT); goto success; } PyTypeObject *cls = Py_TYPE(container); @@ -1324,7 +1324,7 @@ _Py_Specialize_BinarySubscr( } cache->func_version = version; ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor; - _py_set_opocde(instr, BINARY_SUBSCR_GETITEM); + _py_set_opcode(instr, BINARY_SUBSCR_GETITEM); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1332,7 +1332,7 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, BINARY_SUBSCR); + _py_set_opcode(instr, BINARY_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1351,7 +1351,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) && ((PyLongObject *)sub)->ob_digit[0] < (size_t)PyList_GET_SIZE(container)) { - _py_set_opocde(instr, STORE_SUBSCR_LIST_INT); + _py_set_opcode(instr, STORE_SUBSCR_LIST_INT); goto success; } else { @@ -1369,7 +1369,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins } } if (container_type == &PyDict_Type) { - _py_set_opocde(instr, STORE_SUBSCR_DICT); + _py_set_opcode(instr, STORE_SUBSCR_DICT); goto success; } #ifdef Py_STATS @@ -1436,7 +1436,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, STORE_SUBSCR); + _py_set_opcode(instr, STORE_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1458,20 +1458,20 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, int oparg = _Py_OPARG(*instr); if (nargs == 1 && kwnames == NULL && oparg == 1) { if (tp == &PyUnicode_Type) { - _py_set_opocde(instr, CALL_NO_KW_STR_1); + _py_set_opcode(instr, CALL_NO_KW_STR_1); return 0; } else if (tp == &PyType_Type) { - _py_set_opocde(instr, CALL_NO_KW_TYPE_1); + _py_set_opcode(instr, CALL_NO_KW_TYPE_1); return 0; } else if (tp == &PyTuple_Type) { - _py_set_opocde(instr, CALL_NO_KW_TUPLE_1); + _py_set_opcode(instr, CALL_NO_KW_TUPLE_1); return 0; } } if (tp->tp_vectorcall != NULL) { - _py_set_opocde(instr, CALL_BUILTIN_CLASS); + _py_set_opcode(instr, CALL_BUILTIN_CLASS); return 0; } SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? @@ -1523,7 +1523,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); + _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); return 0; } case METH_O: { @@ -1537,18 +1537,18 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (_Py_OPCODE(next) == POP_TOP); int oparg = _Py_OPARG(*instr); if ((PyObject *)descr == list_append && oparg == 1 && pop) { - _py_set_opocde(instr, CALL_NO_KW_LIST_APPEND); + _py_set_opcode(instr, CALL_NO_KW_LIST_APPEND); return 0; } - _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); + _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); return 0; } case METH_FASTCALL: { - _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); + _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); return 0; } case METH_FASTCALL|METH_KEYWORDS: { - _py_set_opocde(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + _py_set_opcode(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } @@ -1599,14 +1599,14 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, write_u32(cache->func_version, version); cache->min_args = min_args; if (argcount == nargs) { - _py_set_opocde(instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); + _py_set_opcode(instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); } else if (bound_method) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); return -1; } else { - _py_set_opocde(instr, CALL_PY_WITH_DEFAULTS); + _py_set_opcode(instr, CALL_PY_WITH_DEFAULTS); } return 0; } @@ -1633,10 +1633,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - _py_set_opocde(instr, CALL_NO_KW_LEN); + _py_set_opcode(instr, CALL_NO_KW_LEN); return 0; } - _py_set_opocde(instr, CALL_NO_KW_BUILTIN_O); + _py_set_opcode(instr, CALL_NO_KW_BUILTIN_O); return 0; } case METH_FASTCALL: { @@ -1648,15 +1648,15 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - _py_set_opocde(instr, CALL_NO_KW_ISINSTANCE); + _py_set_opcode(instr, CALL_NO_KW_ISINSTANCE); return 0; } } - _py_set_opocde(instr, CALL_NO_KW_BUILTIN_FAST); + _py_set_opcode(instr, CALL_NO_KW_BUILTIN_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - _py_set_opocde(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); + _py_set_opcode(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); return 0; } default: @@ -1749,7 +1749,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, CALL); + _py_set_opcode(instr, CALL); cache->counter = adaptive_counter_backoff(cache->counter); } else { @@ -1846,18 +1846,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, bool to_store = (_Py_OPCODE(next) == STORE_FAST || _Py_OPCODE(next) == STORE_FAST__LOAD_FAST); if (to_store && locals[_Py_OPARG(next)] == lhs) { - _py_set_opocde(instr, BINARY_OP_INPLACE_ADD_UNICODE); + _py_set_opcode(instr, BINARY_OP_INPLACE_ADD_UNICODE); goto success; } - _py_set_opocde(instr, BINARY_OP_ADD_UNICODE); + _py_set_opcode(instr, BINARY_OP_ADD_UNICODE); goto success; } if (PyLong_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_ADD_INT); + _py_set_opcode(instr, BINARY_OP_ADD_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_ADD_FLOAT); + _py_set_opcode(instr, BINARY_OP_ADD_FLOAT); goto success; } break; @@ -1867,11 +1867,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_MULTIPLY_INT); + _py_set_opcode(instr, BINARY_OP_MULTIPLY_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_MULTIPLY_FLOAT); + _py_set_opcode(instr, BINARY_OP_MULTIPLY_FLOAT); goto success; } break; @@ -1881,18 +1881,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_SUBTRACT_INT); + _py_set_opcode(instr, BINARY_OP_SUBTRACT_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_SUBTRACT_FLOAT); + _py_set_opcode(instr, BINARY_OP_SUBTRACT_FLOAT); goto success; } break; } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - _py_set_opocde(instr, BINARY_OP); + _py_set_opcode(instr, BINARY_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1974,13 +1974,13 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, COMPARE_OP_FLOAT_JUMP); + _py_set_opcode(instr, COMPARE_OP_FLOAT_JUMP); cache->mask = when_to_jump_mask; goto success; } if (PyLong_CheckExact(lhs)) { if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) { - _py_set_opocde(instr, COMPARE_OP_INT_JUMP); + _py_set_opcode(instr, COMPARE_OP_INT_JUMP); cache->mask = when_to_jump_mask; goto success; } @@ -1995,7 +1995,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } else { - _py_set_opocde(instr, COMPARE_OP_STR_JUMP); + _py_set_opcode(instr, COMPARE_OP_STR_JUMP); cache->mask = (when_to_jump_mask & 2) == 0; goto success; } @@ -2003,7 +2003,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); - _py_set_opocde(instr, COMPARE_OP); + _py_set_opcode(instr, COMPARE_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2037,10 +2037,10 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) goto failure; } if (PyTuple_GET_SIZE(seq) == 2) { - _py_set_opocde(instr, UNPACK_SEQUENCE_TWO_TUPLE); + _py_set_opcode(instr, UNPACK_SEQUENCE_TWO_TUPLE); goto success; } - _py_set_opocde(instr, UNPACK_SEQUENCE_TUPLE); + _py_set_opcode(instr, UNPACK_SEQUENCE_TUPLE); goto success; } if (PyList_CheckExact(seq)) { @@ -2048,13 +2048,13 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); goto failure; } - _py_set_opocde(instr, UNPACK_SEQUENCE_LIST); + _py_set_opcode(instr, UNPACK_SEQUENCE_LIST); goto success; } SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - _py_set_opocde(instr, UNPACK_SEQUENCE); + _py_set_opcode(instr, UNPACK_SEQUENCE); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2143,26 +2143,26 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; int next_op = _PyOpcode_Deopt[_Py_OPCODE(next)]; if (tp == &PyListIter_Type) { - _py_set_opocde(instr, FOR_ITER_LIST); + _py_set_opcode(instr, FOR_ITER_LIST); goto success; } else if (tp == &PyTupleIter_Type) { - _py_set_opocde(instr, FOR_ITER_TUPLE); + _py_set_opcode(instr, FOR_ITER_TUPLE); goto success; } else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { - _py_set_opocde(instr, FOR_ITER_RANGE); + _py_set_opcode(instr, FOR_ITER_RANGE); goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { assert(_Py_OPCODE(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1]) == END_FOR); - _py_set_opocde(instr, FOR_ITER_GEN); + _py_set_opcode(instr, FOR_ITER_GEN); goto success; } SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); - _py_set_opocde(instr, FOR_ITER); + _py_set_opcode(instr, FOR_ITER); cache->counter = adaptive_counter_backoff(cache->counter); return; success: From webhook-mailer at python.org Thu Dec 15 04:45:09 2022 From: webhook-mailer at python.org (markshannon) Date: Thu, 15 Dec 2022 09:45:09 -0000 Subject: [Python-checkins] Move stats for the method cache into the `Py_STAT` machinery (GH-100255) Message-ID: <mailman.2929.1671097510.3313.python-checkins@python.org> https://github.com/python/cpython/commit/48e352a2410b6e962d40359939a0d43aaba5ece9 commit: 48e352a2410b6e962d40359939a0d43aaba5ece9 branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-15T09:45:03Z summary: Move stats for the method cache into the `Py_STAT` machinery (GH-100255) files: M Include/internal/pycore_typeobject.h M Include/pystats.h M Objects/typeobject.c M Python/specialize.c diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 71f3068900da..c207ce615c6f 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -36,15 +36,9 @@ struct type_cache_entry { }; #define MCACHE_SIZE_EXP 12 -#define MCACHE_STATS 0 struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; -#if MCACHE_STATS - size_t hits; - size_t misses; - size_t collisions; -#endif }; /* For now we hard-code this to a value for which we are confident diff --git a/Include/pystats.h b/Include/pystats.h index 87e92aa4f05f..04630c9e0f92 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -65,8 +65,15 @@ typedef struct _object_stats { uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; + uint64_t type_cache_hits; + uint64_t type_cache_misses; + uint64_t type_cache_dunder_hits; + uint64_t type_cache_dunder_misses; + uint64_t type_cache_collisions; } ObjectStats; +# + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2c3b39521a6d..a96f993e99dd 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -327,18 +327,6 @@ static unsigned int _PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->types.type_cache; -#if MCACHE_STATS - size_t total = cache->hits + cache->collisions + cache->misses; - fprintf(stderr, "-- Method cache hits = %zd (%d%%)\n", - cache->hits, (int) (100.0 * cache->hits / total)); - fprintf(stderr, "-- Method cache true misses = %zd (%d%%)\n", - cache->misses, (int) (100.0 * cache->misses / total)); - fprintf(stderr, "-- Method cache collisions = %zd (%d%%)\n", - cache->collisions, (int) (100.0 * cache->collisions / total)); - fprintf(stderr, "-- Method cache size = %zd KiB\n", - sizeof(cache->hashtable) / 1024); -#endif - // Set to None, rather than NULL, so _PyType_Lookup() can // use Py_SETREF() rather than using slower Py_XSETREF(). type_cache_clear(cache, Py_None); @@ -4148,6 +4136,24 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) return res; } +/* Check if the "readied" PyUnicode name + is a double-underscore special name. */ +static int +is_dunder_name(PyObject *name) +{ + Py_ssize_t length = PyUnicode_GET_LENGTH(name); + int kind = PyUnicode_KIND(name); + /* Special names contain at least "__x__" and are always ASCII. */ + if (length > 4 && kind == PyUnicode_1BYTE_KIND) { + const Py_UCS1 *characters = PyUnicode_1BYTE_DATA(name); + return ( + ((characters[length-2] == '_') && (characters[length-1] == '_')) && + ((characters[0] == '_') && (characters[1] == '_')) + ); + } + return 0; +} + /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ PyObject * @@ -4161,12 +4167,13 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) struct type_cache_entry *entry = &cache->hashtable[h]; if (entry->version == type->tp_version_tag && entry->name == name) { -#if MCACHE_STATS - cache->hits++; -#endif assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); + OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); return entry->value; } + OBJECT_STAT_INC_COND(type_cache_misses, !is_dunder_name(name)); + OBJECT_STAT_INC_COND(type_cache_dunder_misses, is_dunder_name(name)); /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); @@ -4194,14 +4201,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) entry->version = type->tp_version_tag; entry->value = res; /* borrowed */ assert(_PyASCIIObject_CAST(name)->hash != -1); -#if MCACHE_STATS - if (entry->name != Py_None && entry->name != name) { - cache->collisions++; - } - else { - cache->misses++; - } -#endif + OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); Py_SETREF(entry->name, Py_NewRef(name)); } @@ -4218,24 +4218,6 @@ _PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) return _PyType_Lookup(type, oname); } -/* Check if the "readied" PyUnicode name - is a double-underscore special name. */ -static int -is_dunder_name(PyObject *name) -{ - Py_ssize_t length = PyUnicode_GET_LENGTH(name); - int kind = PyUnicode_KIND(name); - /* Special names contain at least "__x__" and are always ASCII. */ - if (length > 4 && kind == PyUnicode_1BYTE_KIND) { - const Py_UCS1 *characters = PyUnicode_1BYTE_DATA(name); - return ( - ((characters[length-2] == '_') && (characters[length-1] == '_')) && - ((characters[0] == '_') && (characters[1] == '_')) - ); - } - return 0; -} - /* This is similar to PyObject_GenericGetAttr(), but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ static PyObject * diff --git a/Python/specialize.c b/Python/specialize.c index e88fdde79a67..d1a38450fff0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -187,6 +187,11 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); + fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); + fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); + fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); + fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); + fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); } static void From webhook-mailer at python.org Thu Dec 15 06:26:14 2022 From: webhook-mailer at python.org (markshannon) Date: Thu, 15 Dec 2022 11:26:14 -0000 Subject: [Python-checkins] GH-99767: update PyTypeObject docs for type watchers (GH-99928) Message-ID: <mailman.2930.1671103575.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b7e4f1d97c6e784d2dee182d2b81541ddcff5751 commit: b7e4f1d97c6e784d2dee182d2b81541ddcff5751 branch: main author: Carl Meyer <carl at oddbird.net> committer: markshannon <mark at hotpy.org> date: 2022-12-15T11:26:08Z summary: GH-99767: update PyTypeObject docs for type watchers (GH-99928) files: M Doc/c-api/typeobj.rst M Doc/includes/typestruct.h diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 8f8869ec668a..c7b318b21853 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,6 +147,8 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_watched` | char | | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] @@ -2090,6 +2092,13 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) +.. c:member:: char PyTypeObject.tp_watched + + Internal. Do not use. + + .. versionadded:: 3.12 + + .. _static-types: Static Types diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 02f8ccfe4438..f0ad1e47cb0d 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -80,4 +80,7 @@ typedef struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + + /* bitset of which type-watchers care about this type */ + char tp_watched; } PyTypeObject; From webhook-mailer at python.org Thu Dec 15 13:39:37 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 15 Dec 2022 18:39:37 -0000 Subject: [Python-checkins] Remove uninformative itertools recipe (GH-100253) Message-ID: <mailman.2931.1671129578.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8356c14b4f81f4d0010afb61610edacf4068b804 commit: 8356c14b4f81f4d0010afb61610edacf4068b804 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-15T12:39:01-06:00 summary: Remove uninformative itertools recipe (GH-100253) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 624d2430ac20..9146ed1bfb62 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -829,10 +829,6 @@ which incur interpreter overhead. "Count how many times the predicate is true" return sum(map(pred, iterable)) - def pad_none(iterable): - "Returns the sequence elements and then returns None indefinitely." - return chain(iterable, repeat(None)) - def ncycles(iterable, n): "Returns the sequence elements n times" return chain.from_iterable(repeat(tuple(iterable), n)) @@ -1193,9 +1189,6 @@ which incur interpreter overhead. >>> take(5, map(int, repeatfunc(random.random))) [0, 0, 0, 0, 0] - >>> list(islice(pad_none('abc'), 0, 6)) - ['a', 'b', 'c', None, None, None] - >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] From webhook-mailer at python.org Thu Dec 15 13:40:51 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 15 Dec 2022 18:40:51 -0000 Subject: [Python-checkins] GH-100234: Set a default value for random.expovariate() (GH-100235) Message-ID: <mailman.2932.1671129653.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b430399d41fa88e9040cd055e55cf9211bf63c61 commit: b430399d41fa88e9040cd055e55cf9211bf63c61 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-15T12:40:45-06:00 summary: GH-100234: Set a default value for random.expovariate() (GH-100235) files: A Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst M Doc/library/random.rst M Lib/random.py M Lib/test/test_random.py diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 669204ba65b7..d944518a7909 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -320,7 +320,7 @@ be found in any statistics text. ``beta > 0``. Returned values range between 0 and 1. -.. function:: expovariate(lambd) +.. function:: expovariate(lambd = 1.0) Exponential distribution. *lambd* is 1.0 divided by the desired mean. It should be nonzero. (The parameter would be called @@ -328,6 +328,9 @@ be found in any statistics text. range from 0 to positive infinity if *lambd* is positive, and from negative infinity to 0 if *lambd* is negative. + .. versionchanged:: 3.12 + Added the default value for ``lambd``. + .. function:: gammavariate(alpha, beta) diff --git a/Lib/random.py b/Lib/random.py index c70294ee0cbf..e60b7294b6dd 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -577,7 +577,7 @@ def lognormvariate(self, mu, sigma): """ return _exp(self.normalvariate(mu, sigma)) - def expovariate(self, lambd): + def expovariate(self, lambd=1.0): """Exponential distribution. lambd is 1.0 divided by the desired mean. It should be diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 1e825c3572d2..67de54c7db8b 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -988,6 +988,7 @@ def test_zeroinputs(self): g.random = x[:].pop; g.uniform(1,10) g.random = x[:].pop; g.paretovariate(1.0) g.random = x[:].pop; g.expovariate(1.0) + g.random = x[:].pop; g.expovariate() g.random = x[:].pop; g.weibullvariate(1.0, 1.0) g.random = x[:].pop; g.vonmisesvariate(1.0, 1.0) g.random = x[:].pop; g.normalvariate(0.0, 1.0) diff --git a/Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst b/Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst new file mode 100644 index 000000000000..6d9b909d874b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst @@ -0,0 +1,2 @@ +Set a default value of 1.0 for the ``lambd`` parameter in +random.expovariate(). From webhook-mailer at python.org Thu Dec 15 13:48:04 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 15 Dec 2022 18:48:04 -0000 Subject: [Python-checkins] [3.11] Remove uninformative itertools recipe (GH-100253) (GH-100275) Message-ID: <mailman.2933.1671130085.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9d2dcbbccdf4207249665b58c8bd28430c4f7afd commit: 9d2dcbbccdf4207249665b58c8bd28430c4f7afd branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-15T12:47:58-06:00 summary: [3.11] Remove uninformative itertools recipe (GH-100253) (GH-100275) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 3d93c2c0573e..ee70b16f42fb 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -790,10 +790,6 @@ which incur interpreter overhead. "Count how many times the predicate is true" return sum(map(pred, iterable)) - def pad_none(iterable): - "Returns the sequence elements and then returns None indefinitely." - return chain(iterable, repeat(None)) - def ncycles(iterable, n): "Returns the sequence elements n times" return chain.from_iterable(repeat(tuple(iterable), n)) @@ -1163,9 +1159,6 @@ which incur interpreter overhead. >>> take(5, map(int, repeatfunc(random.random))) [0, 0, 0, 0, 0] - >>> list(islice(pad_none('abc'), 0, 6)) - ['a', 'b', 'c', None, None, None] - >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] From webhook-mailer at python.org Thu Dec 15 17:59:43 2022 From: webhook-mailer at python.org (FFY00) Date: Thu, 15 Dec 2022 22:59:43 -0000 Subject: [Python-checkins] gh-78997: AttributeError if loading fails in LibraryLoader.__getattr__ Message-ID: <mailman.2934.1671145184.3313.python-checkins@python.org> https://github.com/python/cpython/commit/101cfe679fe10b9b662e815999dadf81fcc32c9c commit: 101cfe679fe10b9b662e815999dadf81fcc32c9c branch: main author: Ateeq Sharfuddin <ateeq.sharfuddin at gmail.com> committer: FFY00 <filipe.lains at gmail.com> date: 2022-12-15T22:59:37Z summary: gh-78997: AttributeError if loading fails in LibraryLoader.__getattr__ Co-authored-by: Zachary Ware <zachary.ware at gmail.com> Co-authored-by: Filipe La?ns <filipe.lains at gmail.com> Co-authored-by: Eric Wieser <wieser.eric at gmail.com> files: A Lib/ctypes/test/test_loading.py A Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst M Lib/ctypes/__init__.py diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index b94b337a3157..2e9d4c5e7238 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -444,7 +444,10 @@ def __init__(self, dlltype): def __getattr__(self, name): if name[0] == '_': raise AttributeError(name) - dll = self._dlltype(name) + try: + dll = self._dlltype(name) + except OSError: + raise AttributeError(name) setattr(self, name, dll) return dll diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py new file mode 100644 index 000000000000..b61d6fa2912a --- /dev/null +++ b/Lib/ctypes/test/test_loading.py @@ -0,0 +1,188 @@ +from ctypes import * +import os +import shutil +import subprocess +import sys +import unittest +import test.support +from test.support import import_helper +from test.support import os_helper +from ctypes.util import find_library + +libc_name = None + +def setUpModule(): + global libc_name + if os.name == "nt": + libc_name = find_library("c") + elif sys.platform == "cygwin": + libc_name = "cygwin1.dll" + else: + libc_name = find_library("c") + + if test.support.verbose: + print("libc_name is", libc_name) + +class LoaderTest(unittest.TestCase): + + unknowndll = "xxrandomnamexx" + + def test_load(self): + if libc_name is None: + self.skipTest('could not find libc') + CDLL(libc_name) + CDLL(os.path.basename(libc_name)) + self.assertRaises(OSError, CDLL, self.unknowndll) + + def test_load_version(self): + if libc_name is None: + self.skipTest('could not find libc') + if os.path.basename(libc_name) != 'libc.so.6': + self.skipTest('wrong libc path for test') + cdll.LoadLibrary("libc.so.6") + # linux uses version, libc 9 should not exist + self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") + self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) + + def test_find(self): + for name in ("c", "m"): + lib = find_library(name) + if lib: + cdll.LoadLibrary(lib) + CDLL(lib) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_library(self): + # CRT is no longer directly loadable. See issue23606 for the + # discussion about alternative approaches. + #self.assertIsNotNone(libc_name) + if test.support.verbose: + print(find_library("kernel32")) + print(find_library("user32")) + + if os.name == "nt": + windll.kernel32.GetModuleHandleW + windll["kernel32"].GetModuleHandleW + windll.LoadLibrary("kernel32").GetModuleHandleW + WinDLL("kernel32").GetModuleHandleW + # embedded null character + self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_ordinal_functions(self): + import _ctypes_test + dll = WinDLL(_ctypes_test.__file__) + # We load the same function both via ordinal and name + func_ord = dll[2] + func_name = dll.GetString + # addressof gets the address where the function pointer is stored + a_ord = addressof(func_ord) + a_name = addressof(func_name) + f_ord_addr = c_void_p.from_address(a_ord).value + f_name_addr = c_void_p.from_address(a_name).value + self.assertEqual(hex(f_ord_addr), hex(f_name_addr)) + + self.assertRaises(AttributeError, dll.__getitem__, 1234) + + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_1703286_A(self): + from _ctypes import LoadLibrary, FreeLibrary + # On winXP 64-bit, advapi32 loads at an address that does + # NOT fit into a 32-bit integer. FreeLibrary must be able + # to accept this address. + + # These are tests for https://www.python.org/sf/1703286 + handle = LoadLibrary("advapi32") + FreeLibrary(handle) + + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_1703286_B(self): + # Since on winXP 64-bit advapi32 loads like described + # above, the (arbitrarily selected) CloseEventLog function + # also has a high address. 'call_function' should accept + # addresses so large. + from _ctypes import call_function + advapi32 = windll.advapi32 + # Calling CloseEventLog with a NULL argument should fail, + # but the call should not segfault or so. + self.assertEqual(0, advapi32.CloseEventLog(None)) + windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p + windll.kernel32.GetProcAddress.restype = c_void_p + proc = windll.kernel32.GetProcAddress(advapi32._handle, + b"CloseEventLog") + self.assertTrue(proc) + # This is the real test: call the function via 'call_function' + self.assertEqual(0, call_function(proc, (None,))) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_hasattr(self): + # bpo-34816: shouldn't raise OSError + self.assertFalse(hasattr(windll, 'test')) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_dll_with_flags(self): + _sqlite3 = import_helper.import_module("_sqlite3") + src = _sqlite3.__file__ + if src.lower().endswith("_d.pyd"): + ext = "_d.dll" + else: + ext = ".dll" + + with os_helper.temp_dir() as tmp: + # We copy two files and load _sqlite3.dll (formerly .pyd), + # which has a dependency on sqlite3.dll. Then we test + # loading it in subprocesses to avoid it starting in memory + # for each test. + target = os.path.join(tmp, "_sqlite3.dll") + shutil.copy(src, target) + shutil.copy(os.path.join(os.path.dirname(src), "sqlite3" + ext), + os.path.join(tmp, "sqlite3" + ext)) + + def should_pass(command): + with self.subTest(command): + subprocess.check_output( + [sys.executable, "-c", + "from ctypes import *; import nt;" + command], + cwd=tmp + ) + + def should_fail(command): + with self.subTest(command): + with self.assertRaises(subprocess.CalledProcessError): + subprocess.check_output( + [sys.executable, "-c", + "from ctypes import *; import nt;" + command], + cwd=tmp, stderr=subprocess.STDOUT, + ) + + # Default load should not find this in CWD + should_fail("WinDLL('_sqlite3.dll')") + + # Relative path (but not just filename) should succeed + should_pass("WinDLL('./_sqlite3.dll')") + + # Insecure load flags should succeed + # Clear the DLL directory to avoid safe search settings propagating + should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)") + + # Full path load without DLL_LOAD_DIR shouldn't find dependency + should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + + "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32)") + + # Full path load with DLL_LOAD_DIR should succeed + should_pass("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + + "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32|" + + "nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)") + + # User-specified directory should succeed + should_pass("import os; p = os.add_dll_directory(os.getcwd());" + + "WinDLL('_sqlite3.dll'); p.close()") + + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst b/Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst new file mode 100644 index 000000000000..6fe5f12cc9d9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst @@ -0,0 +1,3 @@ +``hasattr(ctypes.windll, 'nonexistant')`` now returns ``False`` instead of raising :exc:`OSError`. + + From webhook-mailer at python.org Fri Dec 16 05:26:40 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 16 Dec 2022 10:26:40 -0000 Subject: [Python-checkins] Fix typo in introduction.rst (#100266) Message-ID: <mailman.2935.1671186401.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2cd5146f273f8e6e7cfeccb19676907294ba1ace commit: 2cd5146f273f8e6e7cfeccb19676907294ba1ace branch: main author: Mikhail Berkov <uhasker at protonmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-16T15:56:30+05:30 summary: Fix typo in introduction.rst (#100266) files: M Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 558b1c3eec60..6532a4e2cc83 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -269,7 +269,7 @@ Indices may also be negative numbers, to start counting from the right:: Note that since -0 is the same as 0, negative indices start from -1. In addition to indexing, *slicing* is also supported. While indexing is used -to obtain individual characters, *slicing* allows you to obtain substring:: +to obtain individual characters, *slicing* allows you to obtain a substring:: >>> word[0:2] # characters from position 0 (included) to 2 (excluded) 'Py' From webhook-mailer at python.org Fri Dec 16 05:32:15 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 16 Dec 2022 10:32:15 -0000 Subject: [Python-checkins] gh-99767: mark `PyTypeObject.tp_watched` as internal use only in table (#100271) Message-ID: <mailman.2936.1671186737.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e00266f39800af9cfb15cb3ad6b795c77044d798 commit: e00266f39800af9cfb15cb3ad6b795c77044d798 branch: main author: Carl Meyer <carl at oddbird.net> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-16T16:02:09+05:30 summary: gh-99767: mark `PyTypeObject.tp_watched` as internal use only in table (#100271) files: M Doc/c-api/typeobj.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c7b318b21853..016a92f11dbd 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,7 +147,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_watched` | char | | | | | | + | [:c:member:`~PyTypeObject.tp_watched`] | char | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] From webhook-mailer at python.org Fri Dec 16 05:40:16 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 16 Dec 2022 10:40:16 -0000 Subject: [Python-checkins] Document that zipfile's pwd parameter is a `bytes` object (#100209) Message-ID: <mailman.2937.1671187217.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5ee7eb9debb12914f36c5ccee92460a681516fd6 commit: 5ee7eb9debb12914f36c5ccee92460a681516fd6 branch: main author: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-16T16:10:10+05:30 summary: Document that zipfile's pwd parameter is a `bytes` object (#100209) files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 82709dbc9249..ef6934483bcd 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -273,7 +273,8 @@ ZipFile Objects Access a member of the archive as a binary file-like object. *name* can be either the name of a file within the archive or a :class:`ZipInfo` object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files. + or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files as a + :class:`bytes` object. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -325,7 +326,7 @@ ZipFile Objects must be its full name or a :class:`ZipInfo` object. Its file information is extracted as accurately as possible. *path* specifies a different directory to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files. + *pwd* is the password used for encrypted files as a :class:`bytes` object. Returns the normalized path created (a directory or new file). @@ -352,7 +353,7 @@ ZipFile Objects Extract all members from the archive to the current working directory. *path* specifies a different directory to extract to. *members* is optional and must be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files. + used for encrypted files as a :class:`bytes` object. .. warning:: @@ -377,16 +378,16 @@ ZipFile Objects .. method:: ZipFile.setpassword(pwd) - Set *pwd* as default password to extract encrypted files. + Set *pwd* (a :class:`bytes` object) as default password to extract encrypted files. .. method:: ZipFile.read(name, pwd=None) Return the bytes of the file *name* in the archive. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files and, if specified, - it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a ZipFile that uses a compression method other than + read or append. *pwd* is the password used for encrypted files as a :class:`bytes` + object and, if specified, overrides the default password set with :meth:`setpassword`. + Calling :meth:`read` on a ZipFile that uses a compression method other than :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also be raised if the corresponding compression module is not available. From webhook-mailer at python.org Fri Dec 16 08:48:09 2022 From: webhook-mailer at python.org (pablogsal) Date: Fri, 16 Dec 2022 13:48:09 -0000 Subject: [Python-checkins] gh-81057: Move the Cached Parser Dummy Name to _PyRuntimeState (#100277) Message-ID: <mailman.2938.1671198491.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0415cf895f96ae3f896f1f25f0c030a820845e13 commit: 0415cf895f96ae3f896f1f25f0c030a820845e13 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-16T13:48:03Z summary: gh-81057: Move the Cached Parser Dummy Name to _PyRuntimeState (#100277) files: M Include/internal/pycore_parser.h M Include/internal/pycore_runtime_init.h M Parser/action_helpers.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2d2b56bd824c..dd51b92801ae 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -9,6 +9,8 @@ extern "C" { #endif +#include "pycore_ast.h" // struct _expr +#include "pycore_global_strings.h" // _Py_DECLARE_STR() #include "pycore_pyarena.h" // PyArena @@ -22,9 +24,22 @@ struct _parser_runtime_state { #else int _not_used; #endif + struct _expr dummy_name; }; - +_Py_DECLARE_STR(empty, "") +#define _parser_runtime_state_INIT \ + { \ + .dummy_name = { \ + .kind = Name_kind, \ + .v.Name.id = &_Py_STR(empty), \ + .v.Name.ctx = Load, \ + .lineno = 1, \ + .col_offset = 0, \ + .end_lineno = 1, \ + .end_col_offset = 0, \ + }, \ + } extern struct _mod* _PyParser_ASTFromString( const char *str, diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 6342a28f4df9..cb3fce3732c7 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_object.h" +#include "pycore_parser.h" #include "pycore_pymem_init.h" #include "pycore_obmalloc_init.h" @@ -32,6 +33,7 @@ extern "C" { until _PyInterpreterState_Enable() is called. */ \ .next_id = -1, \ }, \ + .parser = _parser_runtime_state_INIT, \ .imports = { \ .lock = { \ .mutex = NULL, \ diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index f12dad095aca..46390966892d 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -2,30 +2,12 @@ #include "pegen.h" #include "string_parser.h" - -static PyObject * -_create_dummy_identifier(Parser *p) -{ - return _PyPegen_new_identifier(p, ""); -} +#include "pycore_runtime.h" // _PyRuntime void * _PyPegen_dummy_name(Parser *p, ...) { - // XXX This leaks memory from the initial arena. - // Use a statically allocated variable instead of a pointer? - static void *cache = NULL; - - if (cache != NULL) { - return cache; - } - - PyObject *id = _create_dummy_identifier(p); - if (!id) { - return NULL; - } - cache = _PyAST_Name(id, Load, 1, 0, 1, 0, p->arena); - return cache; + return &_PyRuntime.parser.dummy_name; } /* Creates a single-element asdl_seq* that contains a */ diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c71fc0d95821..02531692a884 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -50,10 +50,6 @@ Python/getversion.c - version - Python/bootstrap_hash.c - _Py_HashSecret_Initialized - Python/pyhash.c - _Py_HashSecret - -## internal state - set lazily (*after* first init) -# XXX Move to _PyRuntimeState (i.e. tie to init/fini cycle)? -Parser/action_helpers.c _PyPegen_dummy_name cache - - ################################## ## state tied to Py_Main() From webhook-mailer at python.org Fri Dec 16 10:41:29 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 16 Dec 2022 15:41:29 -0000 Subject: [Python-checkins] Better stats for `LOAD_ATTR` and `STORE_ATTR` (GH-100295) Message-ID: <mailman.2939.1671205290.3313.python-checkins@python.org> https://github.com/python/cpython/commit/289c1126dd59e5daae59a7ee8a0aca547c1e351e commit: 289c1126dd59e5daae59a7ee8a0aca547c1e351e branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-16T15:41:23Z summary: Better stats for `LOAD_ATTR` and `STORE_ATTR` (GH-100295) * Don't attempt to specialize for LOAD_ATTR on instance if class has attribute * Improvement to LOAD_ATTR and STORE_ATTR specialization stats. files: M Include/pystats.h M Python/specialize.c diff --git a/Include/pystats.h b/Include/pystats.h index 04630c9e0f92..25ed4bddc724 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -8,7 +8,7 @@ extern "C" { #ifdef Py_STATS -#define SPECIALIZATION_FAILURE_KINDS 32 +#define SPECIALIZATION_FAILURE_KINDS 36 /* Stats for determining who is calling PyEval_EvalFrame */ #define EVAL_CALL_TOTAL 0 diff --git a/Python/specialize.c b/Python/specialize.c index d1a38450fff0..2e135be010ff 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -340,6 +340,9 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 #define SPEC_FAIL_ATTR_NOT_IN_KEYS 29 #define SPEC_FAIL_ATTR_NOT_IN_DICT 30 +#define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31 +#define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32 +#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33 /* Binary subscr and store subscr */ @@ -813,16 +816,29 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) goto success; } case BUILTIN_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); + goto fail; case PYTHON_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); + goto fail; case NON_OVERRIDING: + SPECIALIZATION_FAIL(LOAD_ATTR, + (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ? + SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR : + SPEC_FAIL_ATTR_NOT_MANAGED_DICT); + goto fail; case NON_DESCRIPTOR: + SPECIALIZATION_FAIL(LOAD_ATTR, + (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ? + SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE : + SPEC_FAIL_ATTR_NOT_MANAGED_DICT); + goto fail; case ABSENT: - break; - } - if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, - LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) - { - goto success; + if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, + LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) + { + goto success; + } } fail: STAT_INC(LOAD_ATTR, failure); @@ -901,16 +917,23 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; case BUILTIN_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); + goto fail; case PYTHON_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); + goto fail; case NON_OVERRIDING: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR); + goto fail; case NON_DESCRIPTOR: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); + goto fail; case ABSENT: - break; - } - if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, - STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) - { - goto success; + if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, + STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) + { + goto success; + } } fail: STAT_INC(STORE_ATTR, failure); From webhook-mailer at python.org Fri Dec 16 10:43:10 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 16 Dec 2022 15:43:10 -0000 Subject: [Python-checkins] Improve stats presentation for calls. (GH-100274) Message-ID: <mailman.2940.1671205391.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d4052d835bd6f1b648d533dab8c228b6e3944651 commit: d4052d835bd6f1b648d533dab8c228b6e3944651 branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-16T15:43:04Z summary: Improve stats presentation for calls. (GH-100274) files: M Python/specialize.c diff --git a/Python/specialize.c b/Python/specialize.c index 2e135be010ff..020127e9bf3c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -394,14 +394,14 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_INSTANCE_METHOD 11 #define SPEC_FAIL_CALL_CMETHOD 12 -#define SPEC_FAIL_CALL_PYCFUNCTION 13 -#define SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS 14 -#define SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS 15 -#define SPEC_FAIL_CALL_PYCFUNCTION_NOARGS 16 +#define SPEC_FAIL_CALL_CFUNC_VARARGS 13 +#define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14 +#define SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS 15 +#define SPEC_FAIL_CALL_CFUNC_NOARGS 16 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17 -#define SPEC_FAIL_CALL_CLASS 18 +#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 18 #define SPEC_FAIL_CALL_PYTHON_CLASS 19 -#define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20 +#define SPEC_FAIL_CALL_PEP_523 20 #define SPEC_FAIL_CALL_BOUND_METHOD 21 #define SPEC_FAIL_CALL_STR 22 #define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23 @@ -409,8 +409,6 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_KWNAMES 25 #define SPEC_FAIL_CALL_METHOD_WRAPPER 26 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27 -#define SPEC_FAIL_CALL_PYFUNCTION 28 -#define SPEC_FAIL_CALL_PEP_523 29 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -1517,17 +1515,19 @@ builtin_call_fail_kind(int ml_flags) switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { case METH_VARARGS: - return SPEC_FAIL_CALL_PYCFUNCTION; + return SPEC_FAIL_CALL_CFUNC_VARARGS; case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS; + return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS; + return SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS; case METH_NOARGS: - return SPEC_FAIL_CALL_PYCFUNCTION_NOARGS; - /* This case should never happen with PyCFunctionObject -- only - PyMethodObject. See zlib.compressobj()'s methods for an example. - */ + return SPEC_FAIL_CALL_CFUNC_NOARGS; case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + return SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS; + /* These cases should be optimized, but return "other" just in case */ + case METH_O: + case METH_FASTCALL: + return SPEC_FAIL_OTHER; default: return SPEC_FAIL_CALL_BAD_CALL_FLAGS; } @@ -1698,33 +1698,18 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, static int call_fail_kind(PyObject *callable) { - if (PyCFunction_CheckExact(callable)) { - return SPEC_FAIL_CALL_PYCFUNCTION; - } - else if (PyFunction_Check(callable)) { - return SPEC_FAIL_CALL_PYFUNCTION; - } - else if (PyInstanceMethod_Check(callable)) { + assert(!PyCFunction_CheckExact(callable)); + assert(!PyFunction_Check(callable)); + assert(!PyType_Check(callable)); + assert(!Py_IS_TYPE(callable, &PyMethodDescr_Type)); + assert(!PyMethod_Check(callable)); + if (PyInstanceMethod_Check(callable)) { return SPEC_FAIL_CALL_INSTANCE_METHOD; } - else if (PyMethod_Check(callable)) { - return SPEC_FAIL_CALL_BOUND_METHOD; - } // builtin method else if (PyCMethod_Check(callable)) { return SPEC_FAIL_CALL_CMETHOD; } - else if (PyType_Check(callable)) { - if (((PyTypeObject *)callable)->tp_new == PyBaseObject_Type.tp_new) { - return SPEC_FAIL_CALL_PYTHON_CLASS; - } - else { - return SPEC_FAIL_CALL_CLASS; - } - } - else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { - return SPEC_FAIL_CALL_METHOD_DESCRIPTOR; - } else if (Py_TYPE(callable) == &PyWrapperDescr_Type) { return SPEC_FAIL_CALL_OPERATOR_WRAPPER; } @@ -1760,7 +1745,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, fail = specialize_method_descriptor((PyMethodDescrObject *)callable, instr, nargs, kwnames); } - else if (Py_TYPE(callable) == &PyMethod_Type) { + else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { fail = specialize_py_call((PyFunctionObject *)func, From webhook-mailer at python.org Fri Dec 16 13:18:41 2022 From: webhook-mailer at python.org (brandtbucher) Date: Fri, 16 Dec 2022 18:18:41 -0000 Subject: [Python-checkins] GH-90043: Handle NaNs in COMPARE_OP_FLOAT_JUMP (GH-100278) Message-ID: <mailman.2941.1671214723.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9076455d1b8132928470c11df874ca7c8580b012 commit: 9076455d1b8132928470c11df874ca7c8580b012 branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: brandtbucher <brandtbucher at gmail.com> date: 2022-12-16T10:18:31-08:00 summary: GH-90043: Handle NaNs in COMPARE_OP_FLOAT_JUMP (GH-100278) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst M Python/bytecodes.c M Python/generated_cases.c.h M Python/specialize.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst new file mode 100644 index 000000000000..9a4f896e2cf5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst @@ -0,0 +1,2 @@ +Handle NaNs when specializing :opcode:`COMPARE_OP` for :class:`float` +values. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c56f1d3ef9f4..8e95b7370302 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2024,13 +2024,11 @@ dummy_func( // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (dleft >= dright) + (dleft <= dright)); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); jump = sign_ish & when_to_jump_mask; @@ -2057,8 +2055,8 @@ dummy_func( assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + // 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (ileft >= iright) + (ileft <= iright)); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); jump = sign_ish & when_to_jump_mask; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 63635fbfc2f4..6d84a643b457 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2214,13 +2214,11 @@ // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (dleft >= dright) + (dleft <= dright)); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); jump = sign_ish & when_to_jump_mask; @@ -2258,8 +2256,8 @@ assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + // 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (ileft >= iright) + (ileft <= iright)); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); jump = sign_ish & when_to_jump_mask; diff --git a/Python/specialize.c b/Python/specialize.c index 020127e9bf3c..a1666ccc9159 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1951,15 +1951,16 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) static int compare_masks[] = { - // 1-bit: jump if less than - // 2-bit: jump if equal + // 1-bit: jump if unordered + // 2-bit: jump if less // 4-bit: jump if greater - [Py_LT] = 1 | 0 | 0, - [Py_LE] = 1 | 2 | 0, - [Py_EQ] = 0 | 2 | 0, - [Py_NE] = 1 | 0 | 4, - [Py_GT] = 0 | 0 | 4, - [Py_GE] = 0 | 2 | 4, + // 8-bit: jump if equal + [Py_LT] = 0 | 2 | 0 | 0, + [Py_LE] = 0 | 2 | 0 | 8, + [Py_EQ] = 0 | 0 | 0 | 8, + [Py_NE] = 1 | 2 | 4 | 0, + [Py_GT] = 0 | 0 | 4 | 0, + [Py_GE] = 0 | 0 | 4 | 8, }; void @@ -1980,7 +1981,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, assert(oparg <= Py_GE); int when_to_jump_mask = compare_masks[oparg]; if (next_opcode == POP_JUMP_IF_FALSE) { - when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask; + when_to_jump_mask = (1 | 2 | 4 | 8) & ~when_to_jump_mask; } if (Py_TYPE(lhs) != Py_TYPE(rhs)) { SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); @@ -2009,7 +2010,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } else { _py_set_opcode(instr, COMPARE_OP_STR_JUMP); - cache->mask = (when_to_jump_mask & 2) == 0; + cache->mask = (when_to_jump_mask & 8) == 0; goto success; } } From webhook-mailer at python.org Fri Dec 16 13:43:48 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Fri, 16 Dec 2022 18:43:48 -0000 Subject: [Python-checkins] "Compound statement" docs: Fix with-statement step indexing (#100286) Message-ID: <mailman.2942.1671216229.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f23236a92d8796ae91772adaf27c3485fda963e8 commit: f23236a92d8796ae91772adaf27c3485fda963e8 branch: main author: Frank Dana <ferdnyc at gmail.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-16T18:43:43Z summary: "Compound statement" docs: Fix with-statement step indexing (#100286) Back in commit 226e6e7d4326cf91ef37e13528eb1f62de1bb832 an item was added to the list, renumbering all the rest of the items, but the forward-reference wasn't updated to match. files: M Doc/reference/compound_stmts.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 9e09515f50d1..fe9dda933c80 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -506,7 +506,7 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo method returns without an error, then :meth:`__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would - be. See step 6 below. + be. See step 7 below. #. The suite is executed. From webhook-mailer at python.org Fri Dec 16 14:01:30 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:01:30 -0000 Subject: [Python-checkins] [3.11] "Compound statement" docs: Fix with-statement step indexing (GH-100286) (GH-100300) Message-ID: <mailman.2943.1671217291.3313.python-checkins@python.org> https://github.com/python/cpython/commit/138c2e6f3fa13444db527a272dc8eda8565a54e5 commit: 138c2e6f3fa13444db527a272dc8eda8565a54e5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:01:24-08:00 summary: [3.11] "Compound statement" docs: Fix with-statement step indexing (GH-100286) (GH-100300) Back in commit 226e6e7d4326cf91ef37e13528eb1f62de1bb832 an item was added to the list, renumbering all the rest of the items, but the forward-reference wasn't updated to match. (cherry picked from commit f23236a92d8796ae91772adaf27c3485fda963e8) Co-authored-by: Frank Dana <ferdnyc at gmail.com> files: M Doc/reference/compound_stmts.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 9e09515f50d1..fe9dda933c80 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -506,7 +506,7 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo method returns without an error, then :meth:`__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would - be. See step 6 below. + be. See step 7 below. #. The suite is executed. From webhook-mailer at python.org Fri Dec 16 14:02:46 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:02:46 -0000 Subject: [Python-checkins] "Compound statement" docs: Fix with-statement step indexing (GH-100286) Message-ID: <mailman.2944.1671217367.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0f610202553744deef4a2d639ed53a9793a87e72 commit: 0f610202553744deef4a2d639ed53a9793a87e72 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:02:41-08:00 summary: "Compound statement" docs: Fix with-statement step indexing (GH-100286) Back in commit 226e6e7d4326cf91ef37e13528eb1f62de1bb832 an item was added to the list, renumbering all the rest of the items, but the forward-reference wasn't updated to match. (cherry picked from commit f23236a92d8796ae91772adaf27c3485fda963e8) Co-authored-by: Frank Dana <ferdnyc at gmail.com> files: M Doc/reference/compound_stmts.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 93f5682f3f43..58b88c4d37f1 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -416,7 +416,7 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo method returns without an error, then :meth:`__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would - be. See step 6 below. + be. See step 7 below. #. The suite is executed. From webhook-mailer at python.org Fri Dec 16 14:05:00 2022 From: webhook-mailer at python.org (gvanrossum) Date: Fri, 16 Dec 2022 19:05:00 -0000 Subject: [Python-checkins] gh-99830: asyncio: Document returns of remove_{reader, writer} (#100302) Message-ID: <mailman.2945.1671217501.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5234e1cbea686e38392f113707db322ad8405048 commit: 5234e1cbea686e38392f113707db322ad8405048 branch: main author: Ben Darnell <ben at bendarnell.com> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-16T11:04:55-08:00 summary: gh-99830: asyncio: Document returns of remove_{reader,writer} (#100302) files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index fd47b0c24d8a..470d1aa130e4 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -932,7 +932,8 @@ Watching file descriptors .. method:: loop.remove_reader(fd) - Stop monitoring the *fd* file descriptor for read availability. + Stop monitoring the *fd* file descriptor for read availability. Returns + ``True`` if *fd* was previously being monitored for reads. .. method:: loop.add_writer(fd, callback, *args) @@ -945,7 +946,8 @@ Watching file descriptors .. method:: loop.remove_writer(fd) - Stop monitoring the *fd* file descriptor for write availability. + Stop monitoring the *fd* file descriptor for write availability. Returns + ``True`` if *fd* was previously being monitored for writes. See also :ref:`Platform Support <asyncio-platform-support>` section for some limitations of these methods. From webhook-mailer at python.org Fri Dec 16 14:14:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:14:34 -0000 Subject: [Python-checkins] gh-99830: asyncio: Document returns of remove_{reader, writer} (GH-100302) Message-ID: <mailman.2946.1671218075.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4c810f92baed7562e7f2da0c70e2edac2e89bf75 commit: 4c810f92baed7562e7f2da0c70e2edac2e89bf75 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:14:28-08:00 summary: gh-99830: asyncio: Document returns of remove_{reader,writer} (GH-100302) (cherry picked from commit 5234e1cbea686e38392f113707db322ad8405048) Co-authored-by: Ben Darnell <ben at bendarnell.com> files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 28b7a900583b..d32e848a8413 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -915,7 +915,8 @@ Watching file descriptors .. method:: loop.remove_reader(fd) - Stop monitoring the *fd* file descriptor for read availability. + Stop monitoring the *fd* file descriptor for read availability. Returns + ``True`` if *fd* was previously being monitored for reads. .. method:: loop.add_writer(fd, callback, *args) @@ -928,7 +929,8 @@ Watching file descriptors .. method:: loop.remove_writer(fd) - Stop monitoring the *fd* file descriptor for write availability. + Stop monitoring the *fd* file descriptor for write availability. Returns + ``True`` if *fd* was previously being monitored for writes. See also :ref:`Platform Support <asyncio-platform-support>` section for some limitations of these methods. From webhook-mailer at python.org Fri Dec 16 14:15:18 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:15:18 -0000 Subject: [Python-checkins] gh-99830: asyncio: Document returns of remove_{reader, writer} (GH-100302) Message-ID: <mailman.2947.1671218119.3313.python-checkins@python.org> https://github.com/python/cpython/commit/958be792a7fd50971a048662a377f4ae101652bd commit: 958be792a7fd50971a048662a377f4ae101652bd branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:15:12-08:00 summary: gh-99830: asyncio: Document returns of remove_{reader,writer} (GH-100302) (cherry picked from commit 5234e1cbea686e38392f113707db322ad8405048) Co-authored-by: Ben Darnell <ben at bendarnell.com> files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index a23be64ec39a..7b4c3c7682fe 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -858,7 +858,8 @@ Watching file descriptors .. method:: loop.remove_reader(fd) - Stop monitoring the *fd* file descriptor for read availability. + Stop monitoring the *fd* file descriptor for read availability. Returns + ``True`` if *fd* was previously being monitored for reads. .. method:: loop.add_writer(fd, callback, *args) @@ -871,7 +872,8 @@ Watching file descriptors .. method:: loop.remove_writer(fd) - Stop monitoring the *fd* file descriptor for write availability. + Stop monitoring the *fd* file descriptor for write availability. Returns + ``True`` if *fd* was previously being monitored for writes. See also :ref:`Platform Support <asyncio-platform-support>` section for some limitations of these methods. From webhook-mailer at python.org Fri Dec 16 15:30:53 2022 From: webhook-mailer at python.org (ethanfurman) Date: Fri, 16 Dec 2022 20:30:53 -0000 Subject: [Python-checkins] gh-100039: enhance __signature__ to work with str and callables (GH-100168) Message-ID: <mailman.2948.1671222654.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a5a7cea202d34ca699d9594d428ba527ef7ff2ce commit: a5a7cea202d34ca699d9594d428ba527ef7ff2ce branch: main author: Ethan Furman <ethan at stoneleaf.us> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-16T12:30:47-08:00 summary: gh-100039: enhance __signature__ to work with str and callables (GH-100168) Callables should be either class- or static-methods. Enum now uses the classmethod version to greatly improve the help given for enums and flags. files: A Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst M Lib/enum.py M Lib/inspect.py M Lib/test/test_enum.py M Lib/test/test_inspect.py diff --git a/Lib/enum.py b/Lib/enum.py index a0cad066dc23..21f63881d78d 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -685,7 +685,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k 'member order does not match _order_:\n %r\n %r' % (enum_class._member_names_, _order_) ) - + # return enum_class def __bool__(cls): @@ -1083,6 +1083,13 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ + @classmethod + def __signature__(cls): + if cls._member_names_: + return '(*values)' + else: + return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' + def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' diff --git a/Lib/inspect.py b/Lib/inspect.py index e165937e448a..e92c355220fd 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2443,10 +2443,18 @@ def _signature_from_callable(obj, *, pass else: if sig is not None: + # since __text_signature__ is not writable on classes, __signature__ + # may contain text (or be a callable that returns text); + # if so, convert it + o_sig = sig + if not isinstance(sig, (Signature, str)) and callable(sig): + sig = sig() + if isinstance(sig, str): + sig = _signature_fromstr(sigcls, obj, sig) if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' - 'attribute'.format(sig)) + 'attribute'.format(o_sig)) return sig try: diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index d876c8e5fb77..1e653e94f6b5 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4259,7 +4259,7 @@ class TestEnumTypeSubclassing(unittest.TestCase): Help on class Color in module %s: class Color(enum.Enum) - | Color(value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None) + | Color(*values) | | Method resolution order: | Color @@ -4315,7 +4315,7 @@ class Color(enum.Enum) Help on class Color in module %s: class Color(enum.Enum) - | Color(value, names=None, *values, module=None, qualname=None, type=None, start=1) + | Color(*values) | | Method resolution order: | Color @@ -4462,6 +4462,27 @@ def test_inspect_classify_class_attrs(self): if failed: self.fail("result does not equal expected, see print above") + def test_inspect_signatures(self): + from inspect import signature, Signature, Parameter + self.assertEqual( + signature(Enum), + Signature([ + Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None), + ]), + ) + self.assertEqual( + signature(enum.FlagBoundary), + Signature([ + Parameter('values', Parameter.VAR_POSITIONAL), + ]), + ) + def test_test_simple_enum(self): @_simple_enum(Enum) class SimpleColor: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 2b7977b1648f..0f8217ba3b75 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3614,6 +3614,38 @@ def foo(): pass self.assertEqual(signature_func(foo), inspect.Signature()) self.assertEqual(inspect.get_annotations(foo), {}) + def test_signature_as_str(self): + self.maxDiff = None + class S: + __signature__ = '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + + def test_signature_as_callable(self): + # __signature__ should be either a staticmethod or a bound classmethod + class S: + @classmethod + def __signature__(cls): + return '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + + class S: + @staticmethod + def __signature__(): + return '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): diff --git a/Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst b/Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst new file mode 100644 index 000000000000..d56643f7e871 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst @@ -0,0 +1 @@ +Improve signatures for enums and flags. From webhook-mailer at python.org Fri Dec 16 15:36:19 2022 From: webhook-mailer at python.org (ethanfurman) Date: Fri, 16 Dec 2022 20:36:19 -0000 Subject: [Python-checkins] gh-99540: Constant hash for _PyNone_Type to aid reproducibility (GH-99541) Message-ID: <mailman.2949.1671222980.3313.python-checkins@python.org> https://github.com/python/cpython/commit/432117cd1f59c76d97da2eaff55a7d758301dbc7 commit: 432117cd1f59c76d97da2eaff55a7d758301dbc7 branch: main author: yonillasky <yonillasky at users.noreply.github.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-16T12:36:13-08:00 summary: gh-99540: Constant hash for _PyNone_Type to aid reproducibility (GH-99541) Needed for ASLR builds of Python. files: A Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst M Objects/object.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst new file mode 100644 index 000000000000..ae043f3aba55 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst @@ -0,0 +1 @@ +``None`` now hashes to a constant value. This is not a requirements change. diff --git a/Objects/object.c b/Objects/object.c index 687bd36d2b4a..028b0edc9111 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1641,6 +1641,11 @@ none_bool(PyObject *v) return 0; } +static Py_hash_t none_hash(PyObject *v) +{ + return 0xFCA86420; +} + static PyNumberMethods none_as_number = { 0, /* nb_add */ 0, /* nb_subtract */ @@ -1692,7 +1697,7 @@ PyTypeObject _PyNone_Type = { &none_as_number, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ - 0, /*tp_hash */ + (hashfunc)none_hash,/*tp_hash */ 0, /*tp_call */ 0, /*tp_str */ 0, /*tp_getattro */ From webhook-mailer at python.org Fri Dec 16 19:14:33 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 00:14:33 -0000 Subject: [Python-checkins] gh-78707: deprecate passing >1 argument to `PurePath.[is_]relative_to()` (GH-94469) Message-ID: <mailman.2950.1671236074.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5a991da32961ef5780996d58b8816d5f2085f540 commit: 5a991da32961ef5780996d58b8816d5f2085f540 branch: main author: Barney Gale <barney.gale at gmail.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T16:14:27-08:00 summary: gh-78707: deprecate passing >1 argument to `PurePath.[is_]relative_to()` (GH-94469) This brings `relative_to()` and `is_relative_to()` more in line with other pathlib methods like `rename()` and `symlink_to()`. Resolves #78707. files: A Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst M Doc/library/pathlib.rst M Lib/pathlib.py M Lib/test/test_pathlib.py diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 6537637f33c7..47687400c14e 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -490,7 +490,7 @@ Pure paths provide the following methods and properties: True -.. method:: PurePath.is_relative_to(*other) +.. method:: PurePath.is_relative_to(other) Return whether or not this path is relative to the *other* path. @@ -502,6 +502,10 @@ Pure paths provide the following methods and properties: .. versionadded:: 3.9 + .. deprecated-removed:: 3.12 3.14 + + Passing additional arguments is deprecated; if supplied, they are joined + with *other*. .. method:: PurePath.is_reserved() @@ -564,7 +568,7 @@ Pure paths provide the following methods and properties: True -.. method:: PurePath.relative_to(*other, walk_up=False) +.. method:: PurePath.relative_to(other, walk_up=False) Compute a version of this path relative to the path represented by *other*. If it's impossible, :exc:`ValueError` is raised:: @@ -581,7 +585,7 @@ Pure paths provide the following methods and properties: raise ValueError(error_message.format(str(self), str(formatted))) ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other is absolute. -When *walk_up* is False (the default), the path must start with *other*. + When *walk_up* is False (the default), the path must start with *other*. When the argument is True, ``..`` entries may be added to form the relative path. In all other cases, such as the paths referencing different drives, :exc:`ValueError` is raised.:: @@ -605,6 +609,10 @@ When *walk_up* is False (the default), the path must start with *other*. .. versionadded:: 3.12 The *walk_up* argument (old behavior is the same as ``walk_up=False``). + .. deprecated-removed:: 3.12 3.14 + + Passing additional positional arguments is deprecated; if supplied, + they are joined with *other*. .. method:: PurePath.with_name(name) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f31eb3010368..7890fdade420 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -624,7 +624,7 @@ def with_suffix(self, suffix): return self._from_parsed_parts(self._drv, self._root, self._parts[:-1] + [name]) - def relative_to(self, *other, walk_up=False): + def relative_to(self, other, /, *_deprecated, walk_up=False): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not related to the other path), raise ValueError. @@ -632,10 +632,14 @@ def relative_to(self, *other, walk_up=False): The *walk_up* parameter controls whether `..` may be used to resolve the path. """ - if not other: - raise TypeError("need at least one argument") + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, + remove=(3, 14)) path_cls = type(self) - other = path_cls(*other) + other = path_cls(other, *_deprecated) for step, path in enumerate([other] + list(other.parents)): if self.is_relative_to(path): break @@ -646,12 +650,16 @@ def relative_to(self, *other, walk_up=False): parts = ('..',) * step + self.parts[len(path.parts):] return path_cls(*parts) - def is_relative_to(self, *other): + def is_relative_to(self, other, /, *_deprecated): """Return True if the path is relative to another path or False. """ - if not other: - raise TypeError("need at least one argument") - other = type(self)(*other) + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", + msg, remove=(3, 14)) + other = type(self)(other, *_deprecated) return other == self or other in self.parents @property diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 1d01d3cbd91d..fa6ea0ac63d8 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -654,8 +654,9 @@ def test_relative_to_common(self): self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) # With several args. - self.assertEqual(p.relative_to('a', 'b'), P()) - self.assertEqual(p.relative_to('a', 'b', walk_up=True), P()) + with self.assertWarns(DeprecationWarning): + p.relative_to('a', 'b') + p.relative_to('a', 'b', walk_up=True) # Unrelated paths. self.assertRaises(ValueError, p.relative_to, P('c')) self.assertRaises(ValueError, p.relative_to, P('a/b/c')) @@ -706,7 +707,8 @@ def test_is_relative_to_common(self): self.assertTrue(p.is_relative_to(P('a/b'))) self.assertTrue(p.is_relative_to('a/b')) # With several args. - self.assertTrue(p.is_relative_to('a', 'b')) + with self.assertWarns(DeprecationWarning): + p.is_relative_to('a', 'b') # Unrelated paths. self.assertFalse(p.is_relative_to(P('c'))) self.assertFalse(p.is_relative_to(P('a/b/c'))) diff --git a/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst b/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst new file mode 100644 index 000000000000..c490a3c1bd7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst @@ -0,0 +1,3 @@ +Deprecate passing more than one positional argument to +:meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. From webhook-mailer at python.org Fri Dec 16 23:46:04 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 17 Dec 2022 04:46:04 -0000 Subject: [Python-checkins] GH-98831: Add DECREF_INPUTS(), expanding to DECREF() each stack input (#100205) Message-ID: <mailman.2951.1671252365.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9cdd2fa63b7549d00830bccf19a34e9d69d0b15e commit: 9cdd2fa63b7549d00830bccf19a34e9d69d0b15e branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-16T20:45:55-08:00 summary: GH-98831: Add DECREF_INPUTS(), expanding to DECREF() each stack input (#100205) The presence of this macro indicates that a particular instruction may be considered for conversion to a register-based format (see https://github.com/faster-cpython/ideas/issues/485). An invariant (currently unchecked) is that `DEOPT_IF()` may only occur *before* `DECREF_INPUTS()`, and `ERROR_IF()` may only occur *after* it. One reason not to check this is that there are a few places where we insert *two* `DECREF_INPUTS()` calls, in different branches of the code. The invariant checking would have to be able to do some flow control analysis to understand this. Note that many instructions, especially specialized ones, can't be converted to use this macro straightforwardly. This is because the generator currently only generates plain `Py_DECREF(variable)` statements, and cannot generate things like `_Py_DECREF_SPECIALIZED()` let alone deal with `_PyList_AppendTakeRef()`. files: M Python/bytecodes.c M Tools/cases_generator/generate_cases.py diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8e95b7370302..b29e16e080e9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -161,7 +161,7 @@ dummy_func( super(LOAD_CONST__LOAD_FAST) = LOAD_CONST + LOAD_FAST; inst(POP_TOP, (value --)) { - Py_DECREF(value); + DECREF_INPUTS(); } inst(PUSH_NULL, (-- res)) { @@ -172,19 +172,19 @@ dummy_func( inst(UNARY_POSITIVE, (value -- res)) { res = PyNumber_Positive(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(UNARY_NOT, (value -- res)) { int err = PyObject_IsTrue(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(err < 0, error); if (err == 0) { res = Py_True; @@ -197,7 +197,7 @@ dummy_func( inst(UNARY_INVERT, (value -- res)) { res = PyNumber_Invert(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -351,8 +351,7 @@ dummy_func( STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -438,8 +437,7 @@ dummy_func( ERROR_IF(true, error); } Py_INCREF(res); // Do this before DECREF'ing dict, sub - Py_DECREF(dict); - Py_DECREF(sub); + DECREF_INPUTS(); } inst(BINARY_SUBSCR_GETITEM, (unused/1, type_version/2, func_version/1, container, sub -- unused)) { @@ -500,9 +498,7 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -538,8 +534,7 @@ dummy_func( inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -550,11 +545,11 @@ dummy_func( if (hook == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(true, error); } res = PyObject_CallOneArg(hook, value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); Py_DECREF(res); } @@ -625,12 +620,12 @@ dummy_func( "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + DECREF_INPUTS(); ERROR_IF(true, error); } iter = (*getter)(obj); - Py_DECREF(obj); + DECREF_INPUTS(); ERROR_IF(iter == NULL, error); if (Py_TYPE(iter)->tp_as_async == NULL || diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 2dfc76f2560e..85a7c6098e0b 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -209,7 +209,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None cache_offset += ceffect.size assert cache_offset == self.cache_offset + cache_adjust - # Write the body, substituting a goto for ERROR_IF() + # Write the body, substituting a goto for ERROR_IF() and other stuff assert dedent <= 0 extra = " " * -dedent for line in self.block_text: @@ -232,6 +232,10 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None ) else: out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") + elif m := re.match(r"(\s*)DECREF_INPUTS;\s*$", line): + space = m.group(1) + for ieff in self.input_effects: + out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") else: out.write_raw(extra + line) From webhook-mailer at python.org Sat Dec 17 01:35:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 06:35:24 -0000 Subject: [Python-checkins] [3.11] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100230) Message-ID: <mailman.2952.1671258926.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dd323afea81523482e14c57eaebd6d3c7d61c212 commit: dd323afea81523482e14c57eaebd6d3c7d61c212 branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:04:54+05:30 summary: [3.11] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100230) (cherry picked from commit c450c8c9ed6e420025f39d0e4850a79f8160cdcd) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst A Modules/_testclinic.c A Modules/clinic/_testclinic.c.h M Lib/test/test_clinic.py M Modules/Setup.stdlib.in M Tools/scripts/generate_stdlib_module_names.py M configure M configure.ac M setup.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4aa9691a4829..86c4f94ac028 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3,7 +3,7 @@ # Licensed to the PSF under a contributor agreement. from test import support, test_tools -from test.support import os_helper +from test.support import import_helper, os_helper from unittest import TestCase import collections import inspect @@ -820,5 +820,397 @@ def test_external(self): self.assertEqual(new_mtime_ns, old_mtime_ns) +ac_tester = import_helper.import_module('_testclinic') + + +class ClinicFunctionalTest(unittest.TestCase): + locals().update((name, getattr(ac_tester, name)) + for name in dir(ac_tester) if name.startswith('test_')) + + def test_objects_converter(self): + with self.assertRaises(TypeError): + ac_tester.objects_converter() + self.assertEqual(ac_tester.objects_converter(1, 2), (1, 2)) + self.assertEqual(ac_tester.objects_converter([], 'whatever class'), ([], 'whatever class')) + self.assertEqual(ac_tester.objects_converter(1), (1, None)) + + def test_bytes_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.bytes_object_converter(1) + self.assertEqual(ac_tester.bytes_object_converter(b'BytesObject'), (b'BytesObject',)) + + def test_byte_array_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.byte_array_object_converter(1) + byte_arr = bytearray(b'ByteArrayObject') + self.assertEqual(ac_tester.byte_array_object_converter(byte_arr), (byte_arr,)) + + def test_unicode_converter(self): + with self.assertRaises(TypeError): + ac_tester.unicode_converter(1) + self.assertEqual(ac_tester.unicode_converter('unicode'), ('unicode',)) + + def test_bool_converter(self): + with self.assertRaises(TypeError): + ac_tester.bool_converter(False, False, 'not a int') + self.assertEqual(ac_tester.bool_converter(), (True, True, True)) + self.assertEqual(ac_tester.bool_converter('', [], 5), (False, False, True)) + self.assertEqual(ac_tester.bool_converter(('not empty',), {1: 2}, 0), (True, True, False)) + + def test_char_converter(self): + with self.assertRaises(TypeError): + ac_tester.char_converter(1) + with self.assertRaises(TypeError): + ac_tester.char_converter(b'ab') + chars = [b'A', b'\a', b'\b', b'\t', b'\n', b'\v', b'\f', b'\r', b'"', b"'", b'?', b'\\', b'\000', b'\377'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(), expected) + chars = [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'a', b'b', b'c', b'd'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(*chars), expected) + + def test_unsigned_char_converter(self): + from _testcapi import UCHAR_MAX + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(UCHAR_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(0, UCHAR_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_char_converter([]) + self.assertEqual(ac_tester.unsigned_char_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_short_converter(self): + from _testcapi import SHRT_MIN, SHRT_MAX + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.short_converter([]) + self.assertEqual(ac_tester.short_converter(-1234), (-1234,)) + self.assertEqual(ac_tester.short_converter(4321), (4321,)) + + def test_unsigned_short_converter(self): + from _testcapi import USHRT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_short_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(USHRT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(0, USHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_short_converter([]) + self.assertEqual(ac_tester.unsigned_short_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_int_converter(self): + from _testcapi import INT_MIN, INT_MAX + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.int_converter(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.int_converter([]) + self.assertEqual(ac_tester.int_converter(), (12, 34, 45)) + self.assertEqual(ac_tester.int_converter(1, 2, '3'), (1, 2, ord('3'))) + + def test_unsigned_int_converter(self): + from _testcapi import UINT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_int_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(UINT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(0, UINT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_int_converter([]) + self.assertEqual(ac_tester.unsigned_int_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_converter(self): + from _testcapi import LONG_MIN, LONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_converter([]) + self.assertEqual(ac_tester.long_converter(), (12,)) + self.assertEqual(ac_tester.long_converter(-1234), (-1234,)) + + def test_unsigned_long_converter(self): + from _testcapi import ULONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(ULONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(0, ULONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_long_converter(self): + from _testcapi import LLONG_MIN, LLONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_long_converter([]) + self.assertEqual(ac_tester.long_long_converter(), (12,)) + self.assertEqual(ac_tester.long_long_converter(-1234), (-1234,)) + + def test_unsigned_long_long_converter(self): + from _testcapi import ULLONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(ULLONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(0, ULLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_py_ssize_t_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.py_ssize_t_converter([]) + self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None), (1, 2, 56)) + + def test_slice_index_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(TypeError): + ac_tester.slice_index_converter([]) + self.assertEqual(ac_tester.slice_index_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.slice_index_converter(1, 2, None), (1, 2, 56)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MAX, PY_SSIZE_T_MAX + 1, PY_SSIZE_T_MAX + 1234), + (PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MIN, PY_SSIZE_T_MIN - 1, PY_SSIZE_T_MIN - 1234), + (PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, PY_SSIZE_T_MIN)) + + def test_size_t_converter(self): + with self.assertRaises(ValueError): + ac_tester.size_t_converter(-1) + with self.assertRaises(TypeError): + ac_tester.size_t_converter([]) + self.assertEqual(ac_tester.size_t_converter(), (12,)) + + def test_float_converter(self): + with self.assertRaises(TypeError): + ac_tester.float_converter([]) + self.assertEqual(ac_tester.float_converter(), (12.5,)) + self.assertEqual(ac_tester.float_converter(-0.5), (-0.5,)) + + def test_double_converter(self): + with self.assertRaises(TypeError): + ac_tester.double_converter([]) + self.assertEqual(ac_tester.double_converter(), (12.5,)) + self.assertEqual(ac_tester.double_converter(-0.5), (-0.5,)) + + def test_py_complex_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_complex_converter([]) + self.assertEqual(ac_tester.py_complex_converter(complex(1, 2)), (complex(1, 2),)) + self.assertEqual(ac_tester.py_complex_converter(complex('-1-2j')), (complex('-1-2j'),)) + self.assertEqual(ac_tester.py_complex_converter(-0.5), (-0.5,)) + self.assertEqual(ac_tester.py_complex_converter(10), (10,)) + + def test_str_converter(self): + with self.assertRaises(TypeError): + ac_tester.str_converter(1) + with self.assertRaises(TypeError): + ac_tester.str_converter('a', 'b', 'c') + with self.assertRaises(ValueError): + ac_tester.str_converter('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter('a', b'b', 'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + + def test_py_buffer_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_buffer_converter('a', 'b') + self.assertEqual(ac_tester.py_buffer_converter('abc', bytearray([1, 2, 3])), (b'abc', b'\x01\x02\x03')) + + def test_keywords(self): + self.assertEqual(ac_tester.keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.keywords(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords(a=1, b=2), (1, 2)) + + def test_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly(1, 2) + self.assertEqual(ac_tester.keywords_kwonly(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords_kwonly(a=1, b=2), (1, 2)) + + def test_keywords_opt(self): + self.assertEqual(ac_tester.keywords_opt(1), (1, None, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, 3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_keywords_opt_kwonly(self): + self.assertEqual(ac_tester.keywords_opt_kwonly(1), (1, None, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_opt_kwonly(1, 2, 3) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, c=3), (1, None, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_keywords_kwonly_opt(self): + self.assertEqual(ac_tester.keywords_kwonly_opt(1), (1, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly_opt(1, 2) + self.assertEqual(ac_tester.keywords_kwonly_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.posonly_keywords(1, b=2), (1, 2)) + + def test_posonly_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(a=1, b=2) + self.assertEqual(ac_tester.posonly_kwonly(1, b=2), (1, 2)) + + def test_posonly_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(a=1, b=2, c=3, d=4) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_opt_keywords_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(a=1, b=2, c=3, d=4) + + def test_posonly_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1, 2) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(a=1, b=2, c=3, d=4) + + def test_posonly_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_keywords_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, b=2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_keywords_opt_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2), (1, 2, None, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, 4) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4, e=5), (1, 2, 3, 4, 5)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_opt_keywords_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, 4) + + def test_keyword_only_parameter(self): + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter() + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter(1) + self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst new file mode 100644 index 000000000000..dc86e1d70f12 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst @@ -0,0 +1 @@ +Add functional test for Argument Clinic. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 2730030a1565..2009a4d2ea66 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,6 +169,7 @@ @MODULE__TESTBUFFER_TRUE at _testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE at _testinternalcapi _testinternalcapi.c @MODULE__TESTCAPI_TRUE at _testcapi _testcapimodule.c + at MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. *shared* diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c new file mode 100644 index 000000000000..c9858e964457 --- /dev/null +++ b/Modules/_testclinic.c @@ -0,0 +1,952 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +/* Always enable assertions */ +#undef NDEBUG + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" + +#include "clinic/_testclinic.c.h" + + +/* Pack arguments to a tuple, implicitly increase all the arguments' refcount. + * NULL arguments will be replaced to Py_None. */ +static PyObject * +pack_arguments_newref(int argc, ...) +{ + assert(!PyErr_Occurred()); + PyObject *tuple = PyTuple_New(argc); + if (!tuple) { + return NULL; + } + + va_list vargs; + va_start(vargs, argc); + for (int i = 0; i < argc; i++) { + PyObject *arg = va_arg(vargs, PyObject *); + if (arg) { + if (_PyObject_IsFreed(arg)) { + PyErr_Format(PyExc_AssertionError, + "argument %d at %p is freed or corrupted!", + i, arg); + va_end(vargs); + Py_DECREF(tuple); + return NULL; + } + } + else { + arg = Py_None; + } + PyTuple_SET_ITEM(tuple, i, Py_NewRef(arg)); + } + va_end(vargs); + return tuple; +} + +/* Pack arguments to a tuple. + * `wrapper` is function which converts primitive type to PyObject. + * `arg_type` is type that arguments should be converted to before wrapped. */ +#define RETURN_PACKED_ARGS(argc, wrapper, arg_type, ...) do { \ + assert(!PyErr_Occurred()); \ + arg_type in[argc] = {__VA_ARGS__}; \ + PyObject *out[argc] = {NULL,}; \ + for (int _i = 0; _i < argc; _i++) { \ + out[_i] = wrapper(in[_i]); \ + assert(out[_i] || PyErr_Occurred()); \ + if (!out[_i]) { \ + for (int _j = 0; _j < _i; _j++) { \ + Py_DECREF(out[_j]); \ + } \ + return NULL; \ + } \ + } \ + PyObject *tuple = PyTuple_New(argc); \ + if (!tuple) { \ + for (int _i = 0; _i < argc; _i++) { \ + Py_DECREF(out[_i]); \ + } \ + return NULL; \ + } \ + for (int _i = 0; _i < argc; _i++) { \ + PyTuple_SET_ITEM(tuple, _i, out[_i]); \ + } \ + return tuple; \ + } while (0) + + +/*[clinic input] +module _testclinic +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d4981b80d6efdb12]*/ + + +/*[clinic input] +test_empty_function + +[clinic start generated code]*/ + +static PyObject * +test_empty_function_impl(PyObject *module) +/*[clinic end generated code: output=0f8aeb3ddced55cb input=0dd7048651ad4ae4]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +objects_converter + + a: object + b: object = NULL + / + +[clinic start generated code]*/ + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=3f9c9415ec86c695 input=1533b1bd94187de4]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +bytes_object_converter + + a: PyBytesObject + / + +[clinic start generated code]*/ + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a) +/*[clinic end generated code: output=7732da869d74b784 input=94211751e7996236]*/ +{ + if (!PyBytes_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyBytesObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +byte_array_object_converter + + a: PyByteArrayObject + / + +[clinic start generated code]*/ + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a) +/*[clinic end generated code: output=51f15c76f302b1f7 input=b04d253db51c6f56]*/ +{ + if (!PyByteArray_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyByteArrayObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +unicode_converter + + a: unicode + / + +[clinic start generated code]*/ + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=1b4a4adbb6ac6e34 input=de7b5adbf07435ba]*/ +{ + if (!PyUnicode_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a unicode object"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +bool_converter + + a: bool = True + b: bool(accept={object}) = True + c: bool(accept={int}) = True + / + +[clinic start generated code]*/ + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=17005b0c29afd590 input=7f6537705b2f32f4]*/ +{ + PyObject *obj_a = a ? Py_True : Py_False; + PyObject *obj_b = b ? Py_True : Py_False; + PyObject *obj_c = c ? Py_True : Py_False; + return pack_arguments_newref(3, obj_a, obj_b, obj_c); +} + + +/*[clinic input] +char_converter + + a: char = b'A' + b: char = b'\a' + c: char = b'\b' + d: char = b'\t' + e: char = b'\n' + f: char = b'\v' + g: char = b'\f' + h: char = b'\r' + i: char = b'"' + j: char = b"'" + k: char = b'?' + l: char = b'\\' + m: char = b'\000' + n: char = b'\377' + / + +[clinic start generated code]*/ + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n) +/*[clinic end generated code: output=f929dbd2e55a9871 input=b601bc5bc7fe85e3]*/ +{ + RETURN_PACKED_ARGS(14, PyLong_FromUnsignedLong, unsigned char, + a, b, c, d, e, f, g, h, i, j, k, l, m, n); +} + + +/*[clinic input] +unsigned_char_converter + + a: unsigned_char = 12 + b: unsigned_char(bitwise=False) = 34 + c: unsigned_char(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c) +/*[clinic end generated code: output=490af3b39ce0b199 input=e859502fbe0b3185]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned char, a, b, c); +} + + +/*[clinic input] +short_converter + + a: short = 12 + / + +[clinic start generated code]*/ + +static PyObject * +short_converter_impl(PyObject *module, short a) +/*[clinic end generated code: output=1ebb7ddb64248988 input=b4e2309a66f650ae]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_short_converter + + a: unsigned_short = 12 + b: unsigned_short(bitwise=False) = 34 + c: unsigned_short(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c) +/*[clinic end generated code: output=5f92cc72fc8707a7 input=9d15cd11e741d0c6]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +int_converter + + a: int = 12 + b: int(accept={int}) = 34 + c: int(accept={str}) = 45 + / + +[clinic start generated code]*/ + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=8e56b59be7d0c306 input=a1dbc6344853db7a]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromLong, long, a, b, c); +} + + +/*[clinic input] +unsigned_int_converter + + a: unsigned_int = 12 + b: unsigned_int(bitwise=False) = 34 + c: unsigned_int(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c) +/*[clinic end generated code: output=399a57a05c494cc7 input=8427ed9a3f96272d]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_converter + + a: long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_converter_impl(PyObject *module, long a) +/*[clinic end generated code: output=9663d936a652707a input=84ad0ef28f24bd85]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_long_converter + + a: unsigned_long = 12 + b: unsigned_long(bitwise=False) = 34 + c: unsigned_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c) +/*[clinic end generated code: output=120b82ea9ebd93a8 input=440dd6f1817f5d91]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_long_converter + + a: long_long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_long_converter_impl(PyObject *module, long long a) +/*[clinic end generated code: output=5fb5f2220770c3e1 input=730fcb3eecf4d993]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLongLong, long long, a); +} + + +/*[clinic input] +unsigned_long_long_converter + + a: unsigned_long_long = 12 + b: unsigned_long_long(bitwise=False) = 34 + c: unsigned_long_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c) +/*[clinic end generated code: output=65b7273e63501762 input=300737b0bdb230e9]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLongLong, unsigned long long, + a, b, c); +} + + +/*[clinic input] +py_ssize_t_converter + + a: Py_ssize_t = 12 + b: Py_ssize_t(accept={int}) = 34 + c: Py_ssize_t(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=ce252143e0ed0372 input=76d0f342e9317a1f]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +slice_index_converter + + a: slice_index = 12 + b: slice_index(accept={int}) = 34 + c: slice_index(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=923c6cac77666a6b input=64f99f3f83265e47]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +size_t_converter + + a: size_t = 12 + / + +[clinic start generated code]*/ + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a) +/*[clinic end generated code: output=412b5b7334ab444d input=83ae7d9171fbf208]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromSize_t, size_t, a); +} + + +/*[clinic input] +float_converter + + a: float = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +float_converter_impl(PyObject *module, float a) +/*[clinic end generated code: output=1c98f64f2cf1d55c input=a625b59ad68047d8]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +double_converter + + a: double = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +double_converter_impl(PyObject *module, double a) +/*[clinic end generated code: output=a4e8532d284d035d input=098df188f24e7c62]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +py_complex_converter + + a: Py_complex + / + +[clinic start generated code]*/ + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a) +/*[clinic end generated code: output=9e6ca2eb53b14846 input=e9148a8ca1dbf195]*/ +{ + RETURN_PACKED_ARGS(1, PyComplex_FromCComplex, Py_complex, a); +} + + +/*[clinic input] +str_converter + + a: str = "a" + b: str(accept={robuffer}) = "b" + c: str(accept={robuffer, str}, zeroes=True) = "c" + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_t c_length) +/*[clinic end generated code: output=475bea40548c8cd6 input=bff2656c92ee25de]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + +static PyObject * +bytes_from_buffer(Py_buffer *buf) +{ + PyObject *bytes_obj = PyBytes_FromStringAndSize(NULL, buf->len); + if (!bytes_obj) { + return NULL; + } + void *bytes_obj_buf = ((PyBytesObject *)bytes_obj)->ob_sval; + if (PyBuffer_ToContiguous(bytes_obj_buf, buf, buf->len, 'C') < 0) { + Py_DECREF(bytes_obj); + return NULL; + } + return bytes_obj; +} + +/*[clinic input] +py_buffer_converter + + a: Py_buffer(accept={str, buffer, NoneType}) + b: Py_buffer(accept={rwbuffer}) + / + +[clinic start generated code]*/ + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b) +/*[clinic end generated code: output=52fb13311e3d6d03 input=775de727de5c7421]*/ +{ + RETURN_PACKED_ARGS(2, bytes_from_buffer, Py_buffer *, a, b); +} + + +/*[clinic input] +keywords + + a: object + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=850aaed53e26729e input=f44b89e718c1a93b]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_kwonly + + a: object + * + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=a45c48241da584dc input=1f08e39c3312b015]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_opt + + a: object + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) +/*[clinic end generated code: output=25e4b67d91c76a66 input=b0ba0e4f04904556]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +keywords_opt_kwonly + + a: object + b: object = None + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=6aa5b655a6e9aeb0 input=f79da689d6c51076]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keywords_kwonly_opt + + a: object + * + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=707f78eb0f55c2b1 input=e0fa1a0e46dca791]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords + + a: object + / + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=6ac88f4a5f0bfc8d input=fde0a2f79fe82b06]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_kwonly + + a: object + / + * + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=483e6790d3482185 input=78b3712768da9a19]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_keywords_kwonly + + a: object + / + b: object + * + c: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=2fae573e8cc3fad8 input=a1ad5d2295eb803c]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords_opt + + a: object + / + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=f5eb66241bcf68fb input=51c10de2a120e279]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_keywords_opt + + a: object + b: object = None + / + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=d54a30e549296ffd input=f408a1de7dfaf31f]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_kwonly_opt + + a: object + / + * + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=a20503fe36b4fd62 input=3494253975272f52]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_kwonly_opt + + a: object + b: object = None + / + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=64f3204a3a0413b6 input=d17516581e478412]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_keywords_kwonly_opt + + a: object + / + b: object + * + c: object + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=dbd7e7ddd6257fa0 input=33529f29e97e5adb]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_keywords_opt_kwonly_opt + + a: object + / + b: object + c: object = None + * + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e) +/*[clinic end generated code: output=775d12ae44653045 input=4d4cc62f11441301]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_opt_keywords_opt_kwonly_opt + + a: object + b: object = None + / + c: object = None + * + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=40c6dc422591eade input=3964960a68622431]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keyword_only_parameter + + * + a: object + +[clinic start generated code]*/ + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=c454b6ce98232787 input=8d2868b8d0b27bdb]*/ +{ + return pack_arguments_newref(1, a); +} + + +static PyMethodDef tester_methods[] = { + TEST_EMPTY_FUNCTION_METHODDEF + OBJECTS_CONVERTER_METHODDEF + BYTES_OBJECT_CONVERTER_METHODDEF + BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF + UNICODE_CONVERTER_METHODDEF + BOOL_CONVERTER_METHODDEF + CHAR_CONVERTER_METHODDEF + UNSIGNED_CHAR_CONVERTER_METHODDEF + SHORT_CONVERTER_METHODDEF + UNSIGNED_SHORT_CONVERTER_METHODDEF + INT_CONVERTER_METHODDEF + UNSIGNED_INT_CONVERTER_METHODDEF + LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_CONVERTER_METHODDEF + LONG_LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_LONG_CONVERTER_METHODDEF + PY_SSIZE_T_CONVERTER_METHODDEF + SLICE_INDEX_CONVERTER_METHODDEF + SIZE_T_CONVERTER_METHODDEF + FLOAT_CONVERTER_METHODDEF + DOUBLE_CONVERTER_METHODDEF + PY_COMPLEX_CONVERTER_METHODDEF + STR_CONVERTER_METHODDEF + PY_BUFFER_CONVERTER_METHODDEF + KEYWORDS_METHODDEF + KEYWORDS_KWONLY_METHODDEF + KEYWORDS_OPT_METHODDEF + KEYWORDS_OPT_KWONLY_METHODDEF + KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_METHODDEF + POSONLY_KWONLY_METHODDEF + POSONLY_KEYWORDS_KWONLY_METHODDEF + POSONLY_KEYWORDS_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_METHODDEF + POSONLY_KWONLY_OPT_METHODDEF + POSONLY_OPT_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + KEYWORD_ONLY_PARAMETER_METHODDEF + {NULL, NULL} +}; + +static struct PyModuleDef _testclinic_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testclinic", + .m_size = 0, + .m_methods = tester_methods, +}; + +PyMODINIT_FUNC +PyInit__testclinic(void) +{ + return PyModule_Create(&_testclinic_module); +} + +#undef RETURN_PACKED_ARGS diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h new file mode 100644 index 000000000000..d363dd3f358e --- /dev/null +++ b/Modules/clinic/_testclinic.c.h @@ -0,0 +1,1917 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(test_empty_function__doc__, +"test_empty_function($module, /)\n" +"--\n" +"\n"); + +#define TEST_EMPTY_FUNCTION_METHODDEF \ + {"test_empty_function", (PyCFunction)test_empty_function, METH_NOARGS, test_empty_function__doc__}, + +static PyObject * +test_empty_function_impl(PyObject *module); + +static PyObject * +test_empty_function(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_empty_function_impl(module); +} + +PyDoc_STRVAR(objects_converter__doc__, +"objects_converter($module, a, b=<unrepresentable>, /)\n" +"--\n" +"\n"); + +#define OBJECTS_CONVERTER_METHODDEF \ + {"objects_converter", _PyCFunction_CAST(objects_converter), METH_FASTCALL, objects_converter__doc__}, + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +objects_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b = NULL; + + if (!_PyArg_CheckPositional("objects_converter", nargs, 1, 2)) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + return_value = objects_converter_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_object_converter__doc__, +"bytes_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTES_OBJECT_CONVERTER_METHODDEF \ + {"bytes_object_converter", (PyCFunction)bytes_object_converter, METH_O, bytes_object_converter__doc__}, + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a); + +static PyObject * +bytes_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyBytesObject *a; + + if (!PyBytes_Check(arg)) { + _PyArg_BadArgument("bytes_object_converter", "argument", "bytes", arg); + goto exit; + } + a = (PyBytesObject *)arg; + return_value = bytes_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(byte_array_object_converter__doc__, +"byte_array_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF \ + {"byte_array_object_converter", (PyCFunction)byte_array_object_converter, METH_O, byte_array_object_converter__doc__}, + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a); + +static PyObject * +byte_array_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyByteArrayObject *a; + + if (!PyByteArray_Check(arg)) { + _PyArg_BadArgument("byte_array_object_converter", "argument", "bytearray", arg); + goto exit; + } + a = (PyByteArrayObject *)arg; + return_value = byte_array_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_converter__doc__, +"unicode_converter($module, a, /)\n" +"--\n" +"\n"); + +#define UNICODE_CONVERTER_METHODDEF \ + {"unicode_converter", (PyCFunction)unicode_converter, METH_O, unicode_converter__doc__}, + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a); + +static PyObject * +unicode_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("unicode_converter", "argument", "str", arg); + goto exit; + } + if (PyUnicode_READY(arg) == -1) { + goto exit; + } + a = arg; + return_value = unicode_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(bool_converter__doc__, +"bool_converter($module, a=True, b=True, c=True, /)\n" +"--\n" +"\n"); + +#define BOOL_CONVERTER_METHODDEF \ + {"bool_converter", _PyCFunction_CAST(bool_converter), METH_FASTCALL, bool_converter__doc__}, + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +bool_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 1; + int b = 1; + int c = 1; + + if (!_PyArg_CheckPositional("bool_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyObject_IsTrue(args[0]); + if (a < 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = PyObject_IsTrue(args[1]); + if (b < 0) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = _PyLong_AsInt(args[2]); + if (c == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = bool_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(char_converter__doc__, +"char_converter($module, a=b\'A\', b=b\'\\x07\', c=b\'\\x08\', d=b\'\\t\', e=b\'\\n\',\n" +" f=b\'\\x0b\', g=b\'\\x0c\', h=b\'\\r\', i=b\'\"\', j=b\"\'\", k=b\'?\',\n" +" l=b\'\\\\\', m=b\'\\x00\', n=b\'\\xff\', /)\n" +"--\n" +"\n"); + +#define CHAR_CONVERTER_METHODDEF \ + {"char_converter", _PyCFunction_CAST(char_converter), METH_FASTCALL, char_converter__doc__}, + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n); + +static PyObject * +char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char a = 'A'; + char b = '\x07'; + char c = '\x08'; + char d = '\t'; + char e = '\n'; + char f = '\x0b'; + char g = '\x0c'; + char h = '\r'; + char i = '"'; + char j = '\''; + char k = '?'; + char l = '\\'; + char m = '\x00'; + char n = '\xff'; + + if (!_PyArg_CheckPositional("char_converter", nargs, 0, 14)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + a = PyBytes_AS_STRING(args[0])[0]; + } + else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + a = PyByteArray_AS_STRING(args[0])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 1", "a byte string of length 1", args[0]); + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + b = PyBytes_AS_STRING(args[1])[0]; + } + else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + b = PyByteArray_AS_STRING(args[1])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 2", "a byte string of length 1", args[1]); + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + c = PyBytes_AS_STRING(args[2])[0]; + } + else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + c = PyByteArray_AS_STRING(args[2])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 3", "a byte string of length 1", args[2]); + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + d = PyBytes_AS_STRING(args[3])[0]; + } + else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + d = PyByteArray_AS_STRING(args[3])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 4", "a byte string of length 1", args[3]); + goto exit; + } + if (nargs < 5) { + goto skip_optional; + } + if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + e = PyBytes_AS_STRING(args[4])[0]; + } + else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + e = PyByteArray_AS_STRING(args[4])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 5", "a byte string of length 1", args[4]); + goto exit; + } + if (nargs < 6) { + goto skip_optional; + } + if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + f = PyBytes_AS_STRING(args[5])[0]; + } + else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + f = PyByteArray_AS_STRING(args[5])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 6", "a byte string of length 1", args[5]); + goto exit; + } + if (nargs < 7) { + goto skip_optional; + } + if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + g = PyBytes_AS_STRING(args[6])[0]; + } + else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + g = PyByteArray_AS_STRING(args[6])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 7", "a byte string of length 1", args[6]); + goto exit; + } + if (nargs < 8) { + goto skip_optional; + } + if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + h = PyBytes_AS_STRING(args[7])[0]; + } + else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + h = PyByteArray_AS_STRING(args[7])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 8", "a byte string of length 1", args[7]); + goto exit; + } + if (nargs < 9) { + goto skip_optional; + } + if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + i = PyBytes_AS_STRING(args[8])[0]; + } + else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + i = PyByteArray_AS_STRING(args[8])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 9", "a byte string of length 1", args[8]); + goto exit; + } + if (nargs < 10) { + goto skip_optional; + } + if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + j = PyBytes_AS_STRING(args[9])[0]; + } + else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + j = PyByteArray_AS_STRING(args[9])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 10", "a byte string of length 1", args[9]); + goto exit; + } + if (nargs < 11) { + goto skip_optional; + } + if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + k = PyBytes_AS_STRING(args[10])[0]; + } + else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + k = PyByteArray_AS_STRING(args[10])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 11", "a byte string of length 1", args[10]); + goto exit; + } + if (nargs < 12) { + goto skip_optional; + } + if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + l = PyBytes_AS_STRING(args[11])[0]; + } + else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + l = PyByteArray_AS_STRING(args[11])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 12", "a byte string of length 1", args[11]); + goto exit; + } + if (nargs < 13) { + goto skip_optional; + } + if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + m = PyBytes_AS_STRING(args[12])[0]; + } + else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + m = PyByteArray_AS_STRING(args[12])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 13", "a byte string of length 1", args[12]); + goto exit; + } + if (nargs < 14) { + goto skip_optional; + } + if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + n = PyBytes_AS_STRING(args[13])[0]; + } + else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + n = PyByteArray_AS_STRING(args[13])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 14", "a byte string of length 1", args[13]); + goto exit; + } +skip_optional: + return_value = char_converter_impl(module, a, b, c, d, e, f, g, h, i, j, k, l, m, n); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_char_converter__doc__, +"unsigned_char_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_CHAR_CONVERTER_METHODDEF \ + {"unsigned_char_converter", _PyCFunction_CAST(unsigned_char_converter), METH_FASTCALL, unsigned_char_converter__doc__}, + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c); + +static PyObject * +unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned char a = 12; + unsigned char b = 34; + unsigned char c = 56; + + if (!_PyArg_CheckPositional("unsigned_char_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + a = (unsigned char) ival; + } + } + if (nargs < 2) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + b = (unsigned char) ival; + } + } + if (nargs < 3) { + goto skip_optional; + } + { + unsigned long ival = PyLong_AsUnsignedLongMask(args[2]); + if (ival == (unsigned long)-1 && PyErr_Occurred()) { + goto exit; + } + else { + c = (unsigned char) ival; + } + } +skip_optional: + return_value = unsigned_char_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(short_converter__doc__, +"short_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SHORT_CONVERTER_METHODDEF \ + {"short_converter", _PyCFunction_CAST(short_converter), METH_FASTCALL, short_converter__doc__}, + +static PyObject * +short_converter_impl(PyObject *module, short a); + +static PyObject * +short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + short a = 12; + + if (!_PyArg_CheckPositional("short_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + a = (short) ival; + } + } +skip_optional: + return_value = short_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_short_converter__doc__, +"unsigned_short_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_SHORT_CONVERTER_METHODDEF \ + {"unsigned_short_converter", _PyCFunction_CAST(unsigned_short_converter), METH_FASTCALL, unsigned_short_converter__doc__}, + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c); + +static PyObject * +unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned short a = 12; + unsigned short b = 34; + unsigned short c = 56; + + if (!_PyArg_CheckPositional("unsigned_short_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned short)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned short)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_short_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(int_converter__doc__, +"int_converter($module, a=12, b=34, c=45, /)\n" +"--\n" +"\n"); + +#define INT_CONVERTER_METHODDEF \ + {"int_converter", _PyCFunction_CAST(int_converter), METH_FASTCALL, int_converter__doc__}, + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 12; + int b = 34; + int c = 45; + + if (!_PyArg_CheckPositional("int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = _PyLong_AsInt(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = _PyLong_AsInt(args[1]); + if (b == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyUnicode_Check(args[2])) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + if (PyUnicode_READY(args[2])) { + goto exit; + } + if (PyUnicode_GET_LENGTH(args[2]) != 1) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + c = PyUnicode_READ_CHAR(args[2], 0); +skip_optional: + return_value = int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_int_converter__doc__, +"unsigned_int_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_INT_CONVERTER_METHODDEF \ + {"unsigned_int_converter", _PyCFunction_CAST(unsigned_int_converter), METH_FASTCALL, unsigned_int_converter__doc__}, + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c); + +static PyObject * +unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int a = 12; + unsigned int b = 34; + unsigned int c = 56; + + if (!_PyArg_CheckPositional("unsigned_int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned int)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned int)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_converter__doc__, +"long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_CONVERTER_METHODDEF \ + {"long_converter", _PyCFunction_CAST(long_converter), METH_FASTCALL, long_converter__doc__}, + +static PyObject * +long_converter_impl(PyObject *module, long a); + +static PyObject * +long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long a = 12; + + if (!_PyArg_CheckPositional("long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_converter__doc__, +"unsigned_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_converter", _PyCFunction_CAST(unsigned_long_converter), METH_FASTCALL, unsigned_long_converter__doc__}, + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c); + +static PyObject * +unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long a = 12; + unsigned long b = 34; + unsigned long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongMask(args[2]); +skip_optional: + return_value = unsigned_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_long_converter__doc__, +"long_long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_LONG_CONVERTER_METHODDEF \ + {"long_long_converter", _PyCFunction_CAST(long_long_converter), METH_FASTCALL, long_long_converter__doc__}, + +static PyObject * +long_long_converter_impl(PyObject *module, long long a); + +static PyObject * +long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long long a = 12; + + if (!_PyArg_CheckPositional("long_long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLongLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_long_converter__doc__, +"unsigned_long_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_long_converter", _PyCFunction_CAST(unsigned_long_long_converter), METH_FASTCALL, unsigned_long_long_converter__doc__}, + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c); + +static PyObject * +unsigned_long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long long a = 12; + unsigned long long b = 34; + unsigned long long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongLongMask(args[2]); +skip_optional: + return_value = unsigned_long_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_ssize_t_converter__doc__, +"py_ssize_t_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define PY_SSIZE_T_CONVERTER_METHODDEF \ + {"py_ssize_t_converter", _PyCFunction_CAST(py_ssize_t_converter), METH_FASTCALL, py_ssize_t_converter__doc__}, + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("py_ssize_t_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + a = ival; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + b = ival; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = py_ssize_t_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(slice_index_converter__doc__, +"slice_index_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define SLICE_INDEX_CONVERTER_METHODDEF \ + {"slice_index_converter", _PyCFunction_CAST(slice_index_converter), METH_FASTCALL, slice_index_converter__doc__}, + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +slice_index_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("slice_index_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = slice_index_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(size_t_converter__doc__, +"size_t_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SIZE_T_CONVERTER_METHODDEF \ + {"size_t_converter", _PyCFunction_CAST(size_t_converter), METH_FASTCALL, size_t_converter__doc__}, + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a); + +static PyObject * +size_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + size_t a = 12; + + if (!_PyArg_CheckPositional("size_t_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_Size_t_Converter(args[0], &a)) { + goto exit; + } +skip_optional: + return_value = size_t_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(float_converter__doc__, +"float_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define FLOAT_CONVERTER_METHODDEF \ + {"float_converter", _PyCFunction_CAST(float_converter), METH_FASTCALL, float_converter__doc__}, + +static PyObject * +float_converter_impl(PyObject *module, float a); + +static PyObject * +float_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float a = 12.5; + + if (!_PyArg_CheckPositional("float_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + a = (float) PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = float_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(double_converter__doc__, +"double_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define DOUBLE_CONVERTER_METHODDEF \ + {"double_converter", _PyCFunction_CAST(double_converter), METH_FASTCALL, double_converter__doc__}, + +static PyObject * +double_converter_impl(PyObject *module, double a); + +static PyObject * +double_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double a = 12.5; + + if (!_PyArg_CheckPositional("double_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = PyFloat_AS_DOUBLE(args[0]); + } + else + { + a = PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = double_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_complex_converter__doc__, +"py_complex_converter($module, a, /)\n" +"--\n" +"\n"); + +#define PY_COMPLEX_CONVERTER_METHODDEF \ + {"py_complex_converter", (PyCFunction)py_complex_converter, METH_O, py_complex_converter__doc__}, + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a); + +static PyObject * +py_complex_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_complex a; + + a = PyComplex_AsCComplex(arg); + if (PyErr_Occurred()) { + goto exit; + } + return_value = py_complex_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(str_converter__doc__, +"str_converter($module, a=\'a\', b=\'b\', c=\'c\', /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_METHODDEF \ + {"str_converter", _PyCFunction_CAST(str_converter), METH_FASTCALL, str_converter__doc__}, + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_t c_length); + +static PyObject * +str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + const char *a = "a"; + const char *b = "b"; + const char *c = "c"; + Py_ssize_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "|sys#:str_converter", + &a, &b, &c, &c_length)) { + goto exit; + } + return_value = str_converter_impl(module, a, b, c, c_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_buffer_converter__doc__, +"py_buffer_converter($module, a, b, /)\n" +"--\n" +"\n"); + +#define PY_BUFFER_CONVERTER_METHODDEF \ + {"py_buffer_converter", _PyCFunction_CAST(py_buffer_converter), METH_FASTCALL, py_buffer_converter__doc__}, + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b); + +static PyObject * +py_buffer_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer a = {NULL, NULL}; + Py_buffer b = {NULL, NULL}; + + if (!_PyArg_ParseStack(args, nargs, "z*w*:py_buffer_converter", + &a, &b)) { + goto exit; + } + return_value = py_buffer_converter_impl(module, &a, &b); + +exit: + /* Cleanup for a */ + if (a.obj) { + PyBuffer_Release(&a); + } + /* Cleanup for b */ + if (b.obj) { + PyBuffer_Release(&b); + } + + return return_value; +} + +PyDoc_STRVAR(keywords__doc__, +"keywords($module, /, a, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_METHODDEF \ + {"keywords", _PyCFunction_CAST(keywords), METH_FASTCALL|METH_KEYWORDS, keywords__doc__}, + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly__doc__, +"keywords_kwonly($module, /, a, *, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_METHODDEF \ + {"keywords_kwonly", _PyCFunction_CAST(keywords_kwonly), METH_FASTCALL|METH_KEYWORDS, keywords_kwonly__doc__}, + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt__doc__, +"keywords_opt($module, /, a, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_METHODDEF \ + {"keywords_opt", _PyCFunction_CAST(keywords_opt), METH_FASTCALL|METH_KEYWORDS, keywords_opt__doc__}, + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); + +static PyObject * +keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = keywords_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt_kwonly__doc__, +"keywords_opt_kwonly($module, /, a, b=None, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_KWONLY_METHODDEF \ + {"keywords_opt_kwonly", _PyCFunction_CAST(keywords_opt_kwonly), METH_FASTCALL|METH_KEYWORDS, keywords_opt_kwonly__doc__}, + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt_kwonly", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = keywords_opt_kwonly_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly_opt__doc__, +"keywords_kwonly_opt($module, /, a, *, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_OPT_METHODDEF \ + {"keywords_kwonly_opt", _PyCFunction_CAST(keywords_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, keywords_kwonly_opt__doc__}, + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[2]; +skip_optional_kwonly: + return_value = keywords_kwonly_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords__doc__, +"posonly_keywords($module, a, /, b)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_METHODDEF \ + {"posonly_keywords", _PyCFunction_CAST(posonly_keywords), METH_FASTCALL|METH_KEYWORDS, posonly_keywords__doc__}, + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly__doc__, +"posonly_kwonly($module, a, /, *, b)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_METHODDEF \ + {"posonly_kwonly", _PyCFunction_CAST(posonly_kwonly), METH_FASTCALL|METH_KEYWORDS, posonly_kwonly__doc__}, + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly__doc__, +"posonly_keywords_kwonly($module, a, /, b, *, c)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_METHODDEF \ + {"posonly_keywords_kwonly", _PyCFunction_CAST(posonly_keywords_kwonly), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly__doc__}, + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly", 0}; + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = posonly_keywords_kwonly_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt__doc__, +"posonly_keywords_opt($module, a, /, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_METHODDEF \ + {"posonly_keywords_opt", _PyCFunction_CAST(posonly_keywords_opt), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt__doc__}, + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt__doc__, +"posonly_opt_keywords_opt($module, a, b=None, /, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_METHODDEF \ + {"posonly_opt_keywords_opt", _PyCFunction_CAST(posonly_opt_keywords_opt), METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_opt_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly_opt__doc__, +"posonly_kwonly_opt($module, a, /, *, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_OPT_METHODDEF \ + {"posonly_kwonly_opt", _PyCFunction_CAST(posonly_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_kwonly_opt__doc__}, + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_kwonly_opt__doc__, +"posonly_opt_kwonly_opt($module, a, b=None, /, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_kwonly_opt", _PyCFunction_CAST(posonly_opt_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly_opt__doc__, +"posonly_keywords_kwonly_opt($module, a, /, b, *, c, d=None, e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_kwonly_opt", _PyCFunction_CAST(posonly_keywords_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt_kwonly_opt__doc__, +"posonly_keywords_opt_kwonly_opt($module, a, /, b, c=None, *, d=None,\n" +" e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_opt_kwonly_opt", _PyCFunction_CAST(posonly_keywords_opt_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e); + +static PyObject * +posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_opt_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt_kwonly_opt__doc__, +"posonly_opt_keywords_opt_kwonly_opt($module, a, b=None, /, c=None, *,\n" +" d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_keywords_opt_kwonly_opt", _PyCFunction_CAST(posonly_opt_keywords_opt_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d); + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_keywords_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keyword_only_parameter__doc__, +"keyword_only_parameter($module, /, *, a)\n" +"--\n" +"\n"); + +#define KEYWORD_ONLY_PARAMETER_METHODDEF \ + {"keyword_only_parameter", _PyCFunction_CAST(keyword_only_parameter), METH_FASTCALL|METH_KEYWORDS, keyword_only_parameter__doc__}, + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a); + +static PyObject * +keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keyword_only_parameter", 0}; + PyObject *argsbuf[1]; + PyObject *a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = keyword_only_parameter_impl(module, a); + +exit: + return return_value; +} +/*[clinic end generated code: output=83f439d06635a2e9 input=a9049054013a1b77]*/ diff --git a/Tools/scripts/generate_stdlib_module_names.py b/Tools/scripts/generate_stdlib_module_names.py index fe1e429ebce1..02d2710ca4ad 100644 --- a/Tools/scripts/generate_stdlib_module_names.py +++ b/Tools/scripts/generate_stdlib_module_names.py @@ -27,6 +27,7 @@ '_ctypes_test', '_testbuffer', '_testcapi', + '_testclinic', '_testconsole', '_testimportmultiple', '_testinternalcapi', diff --git a/configure b/configure index 9567ac3a62c8..9e76287725b9 100755 --- a/configure +++ b/configure @@ -640,6 +640,8 @@ MODULE__TESTBUFFER_FALSE MODULE__TESTBUFFER_TRUE MODULE__TESTINTERNALCAPI_FALSE MODULE__TESTINTERNALCAPI_TRUE +MODULE__TESTCLINIC_FALSE +MODULE__TESTCLINIC_TRUE MODULE__TESTCAPI_FALSE MODULE__TESTCAPI_TRUE MODULE__HASHLIB_FALSE @@ -25495,6 +25497,40 @@ fi $as_echo "$py_cv_module__testcapi" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testclinic" >&5 +$as_echo_n "checking for stdlib extension module _testclinic... " >&6; } + if test "$py_cv_module__testclinic" != "n/a"; then : + + if test "$TEST_MODULES" = yes; then : + if true; then : + py_cv_module__testclinic=yes +else + py_cv_module__testclinic=missing +fi +else + py_cv_module__testclinic=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTCLINIC_STATE=$py_cv_module__testclinic$as_nl" + if test "x$py_cv_module__testclinic" = xyes; then : + + + + +fi + if test "$py_cv_module__testclinic" = yes; then + MODULE__TESTCLINIC_TRUE= + MODULE__TESTCLINIC_FALSE='#' +else + MODULE__TESTCLINIC_TRUE='#' + MODULE__TESTCLINIC_FALSE= +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testclinic" >&5 +$as_echo "$py_cv_module__testclinic" >&6; } + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testinternalcapi" >&5 $as_echo_n "checking for stdlib extension module _testinternalcapi... " >&6; } if test "$py_cv_module__testinternalcapi" != "n/a"; then : @@ -26160,6 +26196,10 @@ if test -z "${MODULE__TESTCAPI_TRUE}" && test -z "${MODULE__TESTCAPI_FALSE}"; th as_fn_error $? "conditional \"MODULE__TESTCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTCLINIC_TRUE}" && test -z "${MODULE__TESTCLINIC_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTCLINIC\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__TESTINTERNALCAPI_TRUE}" && test -z "${MODULE__TESTINTERNALCAPI_FALSE}"; then as_fn_error $? "conditional \"MODULE__TESTINTERNALCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 90008bcae17f..c62a565eb68f 100644 --- a/configure.ac +++ b/configure.ac @@ -7090,6 +7090,7 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) diff --git a/setup.py b/setup.py index 15d0d4576a47..4f122b62e0e7 100644 --- a/setup.py +++ b/setup.py @@ -1012,6 +1012,9 @@ def detect_test_extensions(self): # Python C API test module self.addext(Extension('_testcapi', ['_testcapimodule.c'])) + # Python Argument Clinc functional test module + self.addext(Extension('_testclinic', ['_testclinic.c'])) + # Python Internal C API test module self.addext(Extension('_testinternalcapi', ['_testinternalcapi.c'])) From webhook-mailer at python.org Sat Dec 17 01:36:11 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 06:36:11 -0000 Subject: [Python-checkins] [3.10] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100232) Message-ID: <mailman.2953.1671258972.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3144aca3dae51d3d88cc58fd8e9a04d6d6601ced commit: 3144aca3dae51d3d88cc58fd8e9a04d6d6601ced branch: 3.10 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:06:06+05:30 summary: [3.10] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100232) (cherry picked from commit c450c8c9ed6e420025f39d0e4850a79f8160cdcd) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst A Modules/_testclinic.c A Modules/clinic/_testclinic.c.h M Lib/test/test_clinic.py M Tools/scripts/generate_stdlib_module_names.py M setup.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4aa9691a4829..86c4f94ac028 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3,7 +3,7 @@ # Licensed to the PSF under a contributor agreement. from test import support, test_tools -from test.support import os_helper +from test.support import import_helper, os_helper from unittest import TestCase import collections import inspect @@ -820,5 +820,397 @@ def test_external(self): self.assertEqual(new_mtime_ns, old_mtime_ns) +ac_tester = import_helper.import_module('_testclinic') + + +class ClinicFunctionalTest(unittest.TestCase): + locals().update((name, getattr(ac_tester, name)) + for name in dir(ac_tester) if name.startswith('test_')) + + def test_objects_converter(self): + with self.assertRaises(TypeError): + ac_tester.objects_converter() + self.assertEqual(ac_tester.objects_converter(1, 2), (1, 2)) + self.assertEqual(ac_tester.objects_converter([], 'whatever class'), ([], 'whatever class')) + self.assertEqual(ac_tester.objects_converter(1), (1, None)) + + def test_bytes_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.bytes_object_converter(1) + self.assertEqual(ac_tester.bytes_object_converter(b'BytesObject'), (b'BytesObject',)) + + def test_byte_array_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.byte_array_object_converter(1) + byte_arr = bytearray(b'ByteArrayObject') + self.assertEqual(ac_tester.byte_array_object_converter(byte_arr), (byte_arr,)) + + def test_unicode_converter(self): + with self.assertRaises(TypeError): + ac_tester.unicode_converter(1) + self.assertEqual(ac_tester.unicode_converter('unicode'), ('unicode',)) + + def test_bool_converter(self): + with self.assertRaises(TypeError): + ac_tester.bool_converter(False, False, 'not a int') + self.assertEqual(ac_tester.bool_converter(), (True, True, True)) + self.assertEqual(ac_tester.bool_converter('', [], 5), (False, False, True)) + self.assertEqual(ac_tester.bool_converter(('not empty',), {1: 2}, 0), (True, True, False)) + + def test_char_converter(self): + with self.assertRaises(TypeError): + ac_tester.char_converter(1) + with self.assertRaises(TypeError): + ac_tester.char_converter(b'ab') + chars = [b'A', b'\a', b'\b', b'\t', b'\n', b'\v', b'\f', b'\r', b'"', b"'", b'?', b'\\', b'\000', b'\377'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(), expected) + chars = [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'a', b'b', b'c', b'd'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(*chars), expected) + + def test_unsigned_char_converter(self): + from _testcapi import UCHAR_MAX + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(UCHAR_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(0, UCHAR_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_char_converter([]) + self.assertEqual(ac_tester.unsigned_char_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_short_converter(self): + from _testcapi import SHRT_MIN, SHRT_MAX + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.short_converter([]) + self.assertEqual(ac_tester.short_converter(-1234), (-1234,)) + self.assertEqual(ac_tester.short_converter(4321), (4321,)) + + def test_unsigned_short_converter(self): + from _testcapi import USHRT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_short_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(USHRT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(0, USHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_short_converter([]) + self.assertEqual(ac_tester.unsigned_short_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_int_converter(self): + from _testcapi import INT_MIN, INT_MAX + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.int_converter(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.int_converter([]) + self.assertEqual(ac_tester.int_converter(), (12, 34, 45)) + self.assertEqual(ac_tester.int_converter(1, 2, '3'), (1, 2, ord('3'))) + + def test_unsigned_int_converter(self): + from _testcapi import UINT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_int_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(UINT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(0, UINT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_int_converter([]) + self.assertEqual(ac_tester.unsigned_int_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_converter(self): + from _testcapi import LONG_MIN, LONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_converter([]) + self.assertEqual(ac_tester.long_converter(), (12,)) + self.assertEqual(ac_tester.long_converter(-1234), (-1234,)) + + def test_unsigned_long_converter(self): + from _testcapi import ULONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(ULONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(0, ULONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_long_converter(self): + from _testcapi import LLONG_MIN, LLONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_long_converter([]) + self.assertEqual(ac_tester.long_long_converter(), (12,)) + self.assertEqual(ac_tester.long_long_converter(-1234), (-1234,)) + + def test_unsigned_long_long_converter(self): + from _testcapi import ULLONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(ULLONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(0, ULLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_py_ssize_t_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.py_ssize_t_converter([]) + self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None), (1, 2, 56)) + + def test_slice_index_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(TypeError): + ac_tester.slice_index_converter([]) + self.assertEqual(ac_tester.slice_index_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.slice_index_converter(1, 2, None), (1, 2, 56)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MAX, PY_SSIZE_T_MAX + 1, PY_SSIZE_T_MAX + 1234), + (PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MIN, PY_SSIZE_T_MIN - 1, PY_SSIZE_T_MIN - 1234), + (PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, PY_SSIZE_T_MIN)) + + def test_size_t_converter(self): + with self.assertRaises(ValueError): + ac_tester.size_t_converter(-1) + with self.assertRaises(TypeError): + ac_tester.size_t_converter([]) + self.assertEqual(ac_tester.size_t_converter(), (12,)) + + def test_float_converter(self): + with self.assertRaises(TypeError): + ac_tester.float_converter([]) + self.assertEqual(ac_tester.float_converter(), (12.5,)) + self.assertEqual(ac_tester.float_converter(-0.5), (-0.5,)) + + def test_double_converter(self): + with self.assertRaises(TypeError): + ac_tester.double_converter([]) + self.assertEqual(ac_tester.double_converter(), (12.5,)) + self.assertEqual(ac_tester.double_converter(-0.5), (-0.5,)) + + def test_py_complex_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_complex_converter([]) + self.assertEqual(ac_tester.py_complex_converter(complex(1, 2)), (complex(1, 2),)) + self.assertEqual(ac_tester.py_complex_converter(complex('-1-2j')), (complex('-1-2j'),)) + self.assertEqual(ac_tester.py_complex_converter(-0.5), (-0.5,)) + self.assertEqual(ac_tester.py_complex_converter(10), (10,)) + + def test_str_converter(self): + with self.assertRaises(TypeError): + ac_tester.str_converter(1) + with self.assertRaises(TypeError): + ac_tester.str_converter('a', 'b', 'c') + with self.assertRaises(ValueError): + ac_tester.str_converter('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter('a', b'b', 'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + + def test_py_buffer_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_buffer_converter('a', 'b') + self.assertEqual(ac_tester.py_buffer_converter('abc', bytearray([1, 2, 3])), (b'abc', b'\x01\x02\x03')) + + def test_keywords(self): + self.assertEqual(ac_tester.keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.keywords(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords(a=1, b=2), (1, 2)) + + def test_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly(1, 2) + self.assertEqual(ac_tester.keywords_kwonly(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords_kwonly(a=1, b=2), (1, 2)) + + def test_keywords_opt(self): + self.assertEqual(ac_tester.keywords_opt(1), (1, None, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, 3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_keywords_opt_kwonly(self): + self.assertEqual(ac_tester.keywords_opt_kwonly(1), (1, None, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_opt_kwonly(1, 2, 3) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, c=3), (1, None, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_keywords_kwonly_opt(self): + self.assertEqual(ac_tester.keywords_kwonly_opt(1), (1, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly_opt(1, 2) + self.assertEqual(ac_tester.keywords_kwonly_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.posonly_keywords(1, b=2), (1, 2)) + + def test_posonly_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(a=1, b=2) + self.assertEqual(ac_tester.posonly_kwonly(1, b=2), (1, 2)) + + def test_posonly_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(a=1, b=2, c=3, d=4) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_opt_keywords_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(a=1, b=2, c=3, d=4) + + def test_posonly_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1, 2) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(a=1, b=2, c=3, d=4) + + def test_posonly_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_keywords_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, b=2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_keywords_opt_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2), (1, 2, None, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, 4) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4, e=5), (1, 2, 3, 4, 5)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_opt_keywords_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, 4) + + def test_keyword_only_parameter(self): + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter() + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter(1) + self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst new file mode 100644 index 000000000000..dc86e1d70f12 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst @@ -0,0 +1 @@ +Add functional test for Argument Clinic. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c new file mode 100644 index 000000000000..63427ae6373e --- /dev/null +++ b/Modules/_testclinic.c @@ -0,0 +1,952 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +/* Always enable assertions */ +#undef NDEBUG + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" + +#include "clinic/_testclinic.c.h" + + +/* Pack arguments to a tuple, implicitly increase all the arguments' refcount. + * NULL arguments will be replaced to Py_None. */ +static PyObject * +pack_arguments_newref(int argc, ...) +{ + assert(!PyErr_Occurred()); + PyObject *tuple = PyTuple_New(argc); + if (!tuple) { + return NULL; + } + + va_list vargs; + va_start(vargs, argc); + for (int i = 0; i < argc; i++) { + PyObject *arg = va_arg(vargs, PyObject *); + if (arg) { + if (_PyObject_IsFreed(arg)) { + PyErr_Format(PyExc_AssertionError, + "argument %d at %p is freed or corrupted!", + i, arg); + va_end(vargs); + Py_DECREF(tuple); + return NULL; + } + } + else { + arg = Py_None; + } + PyTuple_SET_ITEM(tuple, i, Py_NewRef(arg)); + } + va_end(vargs); + return tuple; +} + +/* Pack arguments to a tuple. + * `wrapper` is function which converts primitive type to PyObject. + * `arg_type` is type that arguments should be converted to before wrapped. */ +#define RETURN_PACKED_ARGS(argc, wrapper, arg_type, ...) do { \ + assert(!PyErr_Occurred()); \ + arg_type in[argc] = {__VA_ARGS__}; \ + PyObject *out[argc] = {NULL,}; \ + for (int _i = 0; _i < argc; _i++) { \ + out[_i] = wrapper(in[_i]); \ + assert(out[_i] || PyErr_Occurred()); \ + if (!out[_i]) { \ + for (int _j = 0; _j < _i; _j++) { \ + Py_DECREF(out[_j]); \ + } \ + return NULL; \ + } \ + } \ + PyObject *tuple = PyTuple_New(argc); \ + if (!tuple) { \ + for (int _i = 0; _i < argc; _i++) { \ + Py_DECREF(out[_i]); \ + } \ + return NULL; \ + } \ + for (int _i = 0; _i < argc; _i++) { \ + PyTuple_SET_ITEM(tuple, _i, out[_i]); \ + } \ + return tuple; \ + } while (0) + + +/*[clinic input] +module _testclinic +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d4981b80d6efdb12]*/ + + +/*[clinic input] +test_empty_function + +[clinic start generated code]*/ + +static PyObject * +test_empty_function_impl(PyObject *module) +/*[clinic end generated code: output=0f8aeb3ddced55cb input=0dd7048651ad4ae4]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +objects_converter + + a: object + b: object = NULL + / + +[clinic start generated code]*/ + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=3f9c9415ec86c695 input=1533b1bd94187de4]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +bytes_object_converter + + a: PyBytesObject + / + +[clinic start generated code]*/ + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a) +/*[clinic end generated code: output=7732da869d74b784 input=94211751e7996236]*/ +{ + if (!PyBytes_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyBytesObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +byte_array_object_converter + + a: PyByteArrayObject + / + +[clinic start generated code]*/ + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a) +/*[clinic end generated code: output=51f15c76f302b1f7 input=b04d253db51c6f56]*/ +{ + if (!PyByteArray_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyByteArrayObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +unicode_converter + + a: unicode + / + +[clinic start generated code]*/ + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=1b4a4adbb6ac6e34 input=de7b5adbf07435ba]*/ +{ + if (!PyUnicode_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a unicode object"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +bool_converter + + a: bool = True + b: bool(accept={object}) = True + c: bool(accept={int}) = True + / + +[clinic start generated code]*/ + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=17005b0c29afd590 input=7f6537705b2f32f4]*/ +{ + PyObject *obj_a = a ? Py_True : Py_False; + PyObject *obj_b = b ? Py_True : Py_False; + PyObject *obj_c = c ? Py_True : Py_False; + return pack_arguments_newref(3, obj_a, obj_b, obj_c); +} + + +/*[clinic input] +char_converter + + a: char = b'A' + b: char = b'\a' + c: char = b'\b' + d: char = b'\t' + e: char = b'\n' + f: char = b'\v' + g: char = b'\f' + h: char = b'\r' + i: char = b'"' + j: char = b"'" + k: char = b'?' + l: char = b'\\' + m: char = b'\000' + n: char = b'\377' + / + +[clinic start generated code]*/ + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n) +/*[clinic end generated code: output=f929dbd2e55a9871 input=b601bc5bc7fe85e3]*/ +{ + RETURN_PACKED_ARGS(14, PyLong_FromUnsignedLong, unsigned char, + a, b, c, d, e, f, g, h, i, j, k, l, m, n); +} + + +/*[clinic input] +unsigned_char_converter + + a: unsigned_char = 12 + b: unsigned_char(bitwise=False) = 34 + c: unsigned_char(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c) +/*[clinic end generated code: output=490af3b39ce0b199 input=e859502fbe0b3185]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned char, a, b, c); +} + + +/*[clinic input] +short_converter + + a: short = 12 + / + +[clinic start generated code]*/ + +static PyObject * +short_converter_impl(PyObject *module, short a) +/*[clinic end generated code: output=1ebb7ddb64248988 input=b4e2309a66f650ae]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_short_converter + + a: unsigned_short = 12 + b: unsigned_short(bitwise=False) = 34 + c: unsigned_short(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c) +/*[clinic end generated code: output=5f92cc72fc8707a7 input=9d15cd11e741d0c6]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +int_converter + + a: int = 12 + b: int(accept={int}) = 34 + c: int(accept={str}) = 45 + / + +[clinic start generated code]*/ + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=8e56b59be7d0c306 input=a1dbc6344853db7a]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromLong, long, a, b, c); +} + + +/*[clinic input] +unsigned_int_converter + + a: unsigned_int = 12 + b: unsigned_int(bitwise=False) = 34 + c: unsigned_int(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c) +/*[clinic end generated code: output=399a57a05c494cc7 input=8427ed9a3f96272d]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_converter + + a: long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_converter_impl(PyObject *module, long a) +/*[clinic end generated code: output=9663d936a652707a input=84ad0ef28f24bd85]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_long_converter + + a: unsigned_long = 12 + b: unsigned_long(bitwise=False) = 34 + c: unsigned_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c) +/*[clinic end generated code: output=120b82ea9ebd93a8 input=440dd6f1817f5d91]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_long_converter + + a: long_long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_long_converter_impl(PyObject *module, long long a) +/*[clinic end generated code: output=5fb5f2220770c3e1 input=730fcb3eecf4d993]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLongLong, long long, a); +} + + +/*[clinic input] +unsigned_long_long_converter + + a: unsigned_long_long = 12 + b: unsigned_long_long(bitwise=False) = 34 + c: unsigned_long_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c) +/*[clinic end generated code: output=65b7273e63501762 input=300737b0bdb230e9]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLongLong, unsigned long long, + a, b, c); +} + + +/*[clinic input] +py_ssize_t_converter + + a: Py_ssize_t = 12 + b: Py_ssize_t(accept={int}) = 34 + c: Py_ssize_t(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=ce252143e0ed0372 input=76d0f342e9317a1f]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +slice_index_converter + + a: slice_index = 12 + b: slice_index(accept={int}) = 34 + c: slice_index(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=923c6cac77666a6b input=64f99f3f83265e47]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +size_t_converter + + a: size_t = 12 + / + +[clinic start generated code]*/ + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a) +/*[clinic end generated code: output=412b5b7334ab444d input=83ae7d9171fbf208]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromSize_t, size_t, a); +} + + +/*[clinic input] +float_converter + + a: float = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +float_converter_impl(PyObject *module, float a) +/*[clinic end generated code: output=1c98f64f2cf1d55c input=a625b59ad68047d8]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +double_converter + + a: double = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +double_converter_impl(PyObject *module, double a) +/*[clinic end generated code: output=a4e8532d284d035d input=098df188f24e7c62]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +py_complex_converter + + a: Py_complex + / + +[clinic start generated code]*/ + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a) +/*[clinic end generated code: output=9e6ca2eb53b14846 input=e9148a8ca1dbf195]*/ +{ + RETURN_PACKED_ARGS(1, PyComplex_FromCComplex, Py_complex, a); +} + + +/*[clinic input] +str_converter + + a: str = "a" + b: str(accept={robuffer}) = "b" + c: str(accept={robuffer, str}, zeroes=True) = "c" + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_clean_t c_length) +/*[clinic end generated code: output=203880b5c6d9fbb4 input=bff2656c92ee25de]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + +static PyObject * +bytes_from_buffer(Py_buffer *buf) +{ + PyObject *bytes_obj = PyBytes_FromStringAndSize(NULL, buf->len); + if (!bytes_obj) { + return NULL; + } + void *bytes_obj_buf = ((PyBytesObject *)bytes_obj)->ob_sval; + if (PyBuffer_ToContiguous(bytes_obj_buf, buf, buf->len, 'C') < 0) { + Py_DECREF(bytes_obj); + return NULL; + } + return bytes_obj; +} + +/*[clinic input] +py_buffer_converter + + a: Py_buffer(accept={str, buffer, NoneType}) + b: Py_buffer(accept={rwbuffer}) + / + +[clinic start generated code]*/ + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b) +/*[clinic end generated code: output=52fb13311e3d6d03 input=775de727de5c7421]*/ +{ + RETURN_PACKED_ARGS(2, bytes_from_buffer, Py_buffer *, a, b); +} + + +/*[clinic input] +keywords + + a: object + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=850aaed53e26729e input=f44b89e718c1a93b]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_kwonly + + a: object + * + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=a45c48241da584dc input=1f08e39c3312b015]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_opt + + a: object + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) +/*[clinic end generated code: output=25e4b67d91c76a66 input=b0ba0e4f04904556]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +keywords_opt_kwonly + + a: object + b: object = None + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=6aa5b655a6e9aeb0 input=f79da689d6c51076]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keywords_kwonly_opt + + a: object + * + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=707f78eb0f55c2b1 input=e0fa1a0e46dca791]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords + + a: object + / + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=6ac88f4a5f0bfc8d input=fde0a2f79fe82b06]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_kwonly + + a: object + / + * + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=483e6790d3482185 input=78b3712768da9a19]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_keywords_kwonly + + a: object + / + b: object + * + c: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=2fae573e8cc3fad8 input=a1ad5d2295eb803c]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords_opt + + a: object + / + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=f5eb66241bcf68fb input=51c10de2a120e279]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_keywords_opt + + a: object + b: object = None + / + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=d54a30e549296ffd input=f408a1de7dfaf31f]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_kwonly_opt + + a: object + / + * + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=a20503fe36b4fd62 input=3494253975272f52]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_kwonly_opt + + a: object + b: object = None + / + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=64f3204a3a0413b6 input=d17516581e478412]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_keywords_kwonly_opt + + a: object + / + b: object + * + c: object + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=dbd7e7ddd6257fa0 input=33529f29e97e5adb]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_keywords_opt_kwonly_opt + + a: object + / + b: object + c: object = None + * + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e) +/*[clinic end generated code: output=775d12ae44653045 input=4d4cc62f11441301]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_opt_keywords_opt_kwonly_opt + + a: object + b: object = None + / + c: object = None + * + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=40c6dc422591eade input=3964960a68622431]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keyword_only_parameter + + * + a: object + +[clinic start generated code]*/ + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=c454b6ce98232787 input=8d2868b8d0b27bdb]*/ +{ + return pack_arguments_newref(1, a); +} + + +static PyMethodDef tester_methods[] = { + TEST_EMPTY_FUNCTION_METHODDEF + OBJECTS_CONVERTER_METHODDEF + BYTES_OBJECT_CONVERTER_METHODDEF + BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF + UNICODE_CONVERTER_METHODDEF + BOOL_CONVERTER_METHODDEF + CHAR_CONVERTER_METHODDEF + UNSIGNED_CHAR_CONVERTER_METHODDEF + SHORT_CONVERTER_METHODDEF + UNSIGNED_SHORT_CONVERTER_METHODDEF + INT_CONVERTER_METHODDEF + UNSIGNED_INT_CONVERTER_METHODDEF + LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_CONVERTER_METHODDEF + LONG_LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_LONG_CONVERTER_METHODDEF + PY_SSIZE_T_CONVERTER_METHODDEF + SLICE_INDEX_CONVERTER_METHODDEF + SIZE_T_CONVERTER_METHODDEF + FLOAT_CONVERTER_METHODDEF + DOUBLE_CONVERTER_METHODDEF + PY_COMPLEX_CONVERTER_METHODDEF + STR_CONVERTER_METHODDEF + PY_BUFFER_CONVERTER_METHODDEF + KEYWORDS_METHODDEF + KEYWORDS_KWONLY_METHODDEF + KEYWORDS_OPT_METHODDEF + KEYWORDS_OPT_KWONLY_METHODDEF + KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_METHODDEF + POSONLY_KWONLY_METHODDEF + POSONLY_KEYWORDS_KWONLY_METHODDEF + POSONLY_KEYWORDS_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_METHODDEF + POSONLY_KWONLY_OPT_METHODDEF + POSONLY_OPT_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + KEYWORD_ONLY_PARAMETER_METHODDEF + {NULL, NULL} +}; + +static struct PyModuleDef _testclinic_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testclinic", + .m_size = 0, + .m_methods = tester_methods, +}; + +PyMODINIT_FUNC +PyInit__testclinic(void) +{ + return PyModule_Create(&_testclinic_module); +} + +#undef RETURN_PACKED_ARGS diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h new file mode 100644 index 000000000000..a8273a5e6e2f --- /dev/null +++ b/Modules/clinic/_testclinic.c.h @@ -0,0 +1,1917 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(test_empty_function__doc__, +"test_empty_function($module, /)\n" +"--\n" +"\n"); + +#define TEST_EMPTY_FUNCTION_METHODDEF \ + {"test_empty_function", (PyCFunction)test_empty_function, METH_NOARGS, test_empty_function__doc__}, + +static PyObject * +test_empty_function_impl(PyObject *module); + +static PyObject * +test_empty_function(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_empty_function_impl(module); +} + +PyDoc_STRVAR(objects_converter__doc__, +"objects_converter($module, a, b=<unrepresentable>, /)\n" +"--\n" +"\n"); + +#define OBJECTS_CONVERTER_METHODDEF \ + {"objects_converter", (PyCFunction)(void(*)(void))objects_converter, METH_FASTCALL, objects_converter__doc__}, + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +objects_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b = NULL; + + if (!_PyArg_CheckPositional("objects_converter", nargs, 1, 2)) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + return_value = objects_converter_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_object_converter__doc__, +"bytes_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTES_OBJECT_CONVERTER_METHODDEF \ + {"bytes_object_converter", (PyCFunction)bytes_object_converter, METH_O, bytes_object_converter__doc__}, + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a); + +static PyObject * +bytes_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyBytesObject *a; + + if (!PyBytes_Check(arg)) { + _PyArg_BadArgument("bytes_object_converter", "argument", "bytes", arg); + goto exit; + } + a = (PyBytesObject *)arg; + return_value = bytes_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(byte_array_object_converter__doc__, +"byte_array_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF \ + {"byte_array_object_converter", (PyCFunction)byte_array_object_converter, METH_O, byte_array_object_converter__doc__}, + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a); + +static PyObject * +byte_array_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyByteArrayObject *a; + + if (!PyByteArray_Check(arg)) { + _PyArg_BadArgument("byte_array_object_converter", "argument", "bytearray", arg); + goto exit; + } + a = (PyByteArrayObject *)arg; + return_value = byte_array_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_converter__doc__, +"unicode_converter($module, a, /)\n" +"--\n" +"\n"); + +#define UNICODE_CONVERTER_METHODDEF \ + {"unicode_converter", (PyCFunction)unicode_converter, METH_O, unicode_converter__doc__}, + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a); + +static PyObject * +unicode_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("unicode_converter", "argument", "str", arg); + goto exit; + } + if (PyUnicode_READY(arg) == -1) { + goto exit; + } + a = arg; + return_value = unicode_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(bool_converter__doc__, +"bool_converter($module, a=True, b=True, c=True, /)\n" +"--\n" +"\n"); + +#define BOOL_CONVERTER_METHODDEF \ + {"bool_converter", (PyCFunction)(void(*)(void))bool_converter, METH_FASTCALL, bool_converter__doc__}, + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +bool_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 1; + int b = 1; + int c = 1; + + if (!_PyArg_CheckPositional("bool_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyObject_IsTrue(args[0]); + if (a < 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = PyObject_IsTrue(args[1]); + if (b < 0) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = _PyLong_AsInt(args[2]); + if (c == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = bool_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(char_converter__doc__, +"char_converter($module, a=b\'A\', b=b\'\\x07\', c=b\'\\x08\', d=b\'\\t\', e=b\'\\n\',\n" +" f=b\'\\x0b\', g=b\'\\x0c\', h=b\'\\r\', i=b\'\"\', j=b\"\'\", k=b\'?\',\n" +" l=b\'\\\\\', m=b\'\\x00\', n=b\'\\xff\', /)\n" +"--\n" +"\n"); + +#define CHAR_CONVERTER_METHODDEF \ + {"char_converter", (PyCFunction)(void(*)(void))char_converter, METH_FASTCALL, char_converter__doc__}, + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n); + +static PyObject * +char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char a = 'A'; + char b = '\x07'; + char c = '\x08'; + char d = '\t'; + char e = '\n'; + char f = '\x0b'; + char g = '\x0c'; + char h = '\r'; + char i = '"'; + char j = '\''; + char k = '?'; + char l = '\\'; + char m = '\x00'; + char n = '\xff'; + + if (!_PyArg_CheckPositional("char_converter", nargs, 0, 14)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + a = PyBytes_AS_STRING(args[0])[0]; + } + else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + a = PyByteArray_AS_STRING(args[0])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 1", "a byte string of length 1", args[0]); + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + b = PyBytes_AS_STRING(args[1])[0]; + } + else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + b = PyByteArray_AS_STRING(args[1])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 2", "a byte string of length 1", args[1]); + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + c = PyBytes_AS_STRING(args[2])[0]; + } + else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + c = PyByteArray_AS_STRING(args[2])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 3", "a byte string of length 1", args[2]); + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + d = PyBytes_AS_STRING(args[3])[0]; + } + else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + d = PyByteArray_AS_STRING(args[3])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 4", "a byte string of length 1", args[3]); + goto exit; + } + if (nargs < 5) { + goto skip_optional; + } + if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + e = PyBytes_AS_STRING(args[4])[0]; + } + else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + e = PyByteArray_AS_STRING(args[4])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 5", "a byte string of length 1", args[4]); + goto exit; + } + if (nargs < 6) { + goto skip_optional; + } + if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + f = PyBytes_AS_STRING(args[5])[0]; + } + else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + f = PyByteArray_AS_STRING(args[5])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 6", "a byte string of length 1", args[5]); + goto exit; + } + if (nargs < 7) { + goto skip_optional; + } + if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + g = PyBytes_AS_STRING(args[6])[0]; + } + else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + g = PyByteArray_AS_STRING(args[6])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 7", "a byte string of length 1", args[6]); + goto exit; + } + if (nargs < 8) { + goto skip_optional; + } + if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + h = PyBytes_AS_STRING(args[7])[0]; + } + else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + h = PyByteArray_AS_STRING(args[7])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 8", "a byte string of length 1", args[7]); + goto exit; + } + if (nargs < 9) { + goto skip_optional; + } + if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + i = PyBytes_AS_STRING(args[8])[0]; + } + else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + i = PyByteArray_AS_STRING(args[8])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 9", "a byte string of length 1", args[8]); + goto exit; + } + if (nargs < 10) { + goto skip_optional; + } + if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + j = PyBytes_AS_STRING(args[9])[0]; + } + else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + j = PyByteArray_AS_STRING(args[9])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 10", "a byte string of length 1", args[9]); + goto exit; + } + if (nargs < 11) { + goto skip_optional; + } + if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + k = PyBytes_AS_STRING(args[10])[0]; + } + else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + k = PyByteArray_AS_STRING(args[10])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 11", "a byte string of length 1", args[10]); + goto exit; + } + if (nargs < 12) { + goto skip_optional; + } + if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + l = PyBytes_AS_STRING(args[11])[0]; + } + else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + l = PyByteArray_AS_STRING(args[11])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 12", "a byte string of length 1", args[11]); + goto exit; + } + if (nargs < 13) { + goto skip_optional; + } + if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + m = PyBytes_AS_STRING(args[12])[0]; + } + else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + m = PyByteArray_AS_STRING(args[12])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 13", "a byte string of length 1", args[12]); + goto exit; + } + if (nargs < 14) { + goto skip_optional; + } + if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + n = PyBytes_AS_STRING(args[13])[0]; + } + else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + n = PyByteArray_AS_STRING(args[13])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 14", "a byte string of length 1", args[13]); + goto exit; + } +skip_optional: + return_value = char_converter_impl(module, a, b, c, d, e, f, g, h, i, j, k, l, m, n); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_char_converter__doc__, +"unsigned_char_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_CHAR_CONVERTER_METHODDEF \ + {"unsigned_char_converter", (PyCFunction)(void(*)(void))unsigned_char_converter, METH_FASTCALL, unsigned_char_converter__doc__}, + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c); + +static PyObject * +unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned char a = 12; + unsigned char b = 34; + unsigned char c = 56; + + if (!_PyArg_CheckPositional("unsigned_char_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + a = (unsigned char) ival; + } + } + if (nargs < 2) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + b = (unsigned char) ival; + } + } + if (nargs < 3) { + goto skip_optional; + } + { + unsigned long ival = PyLong_AsUnsignedLongMask(args[2]); + if (ival == (unsigned long)-1 && PyErr_Occurred()) { + goto exit; + } + else { + c = (unsigned char) ival; + } + } +skip_optional: + return_value = unsigned_char_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(short_converter__doc__, +"short_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SHORT_CONVERTER_METHODDEF \ + {"short_converter", (PyCFunction)(void(*)(void))short_converter, METH_FASTCALL, short_converter__doc__}, + +static PyObject * +short_converter_impl(PyObject *module, short a); + +static PyObject * +short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + short a = 12; + + if (!_PyArg_CheckPositional("short_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + a = (short) ival; + } + } +skip_optional: + return_value = short_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_short_converter__doc__, +"unsigned_short_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_SHORT_CONVERTER_METHODDEF \ + {"unsigned_short_converter", (PyCFunction)(void(*)(void))unsigned_short_converter, METH_FASTCALL, unsigned_short_converter__doc__}, + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c); + +static PyObject * +unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned short a = 12; + unsigned short b = 34; + unsigned short c = 56; + + if (!_PyArg_CheckPositional("unsigned_short_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned short)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned short)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_short_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(int_converter__doc__, +"int_converter($module, a=12, b=34, c=45, /)\n" +"--\n" +"\n"); + +#define INT_CONVERTER_METHODDEF \ + {"int_converter", (PyCFunction)(void(*)(void))int_converter, METH_FASTCALL, int_converter__doc__}, + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 12; + int b = 34; + int c = 45; + + if (!_PyArg_CheckPositional("int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = _PyLong_AsInt(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = _PyLong_AsInt(args[1]); + if (b == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyUnicode_Check(args[2])) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + if (PyUnicode_READY(args[2])) { + goto exit; + } + if (PyUnicode_GET_LENGTH(args[2]) != 1) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + c = PyUnicode_READ_CHAR(args[2], 0); +skip_optional: + return_value = int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_int_converter__doc__, +"unsigned_int_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_INT_CONVERTER_METHODDEF \ + {"unsigned_int_converter", (PyCFunction)(void(*)(void))unsigned_int_converter, METH_FASTCALL, unsigned_int_converter__doc__}, + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c); + +static PyObject * +unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int a = 12; + unsigned int b = 34; + unsigned int c = 56; + + if (!_PyArg_CheckPositional("unsigned_int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned int)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned int)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_converter__doc__, +"long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_CONVERTER_METHODDEF \ + {"long_converter", (PyCFunction)(void(*)(void))long_converter, METH_FASTCALL, long_converter__doc__}, + +static PyObject * +long_converter_impl(PyObject *module, long a); + +static PyObject * +long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long a = 12; + + if (!_PyArg_CheckPositional("long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_converter__doc__, +"unsigned_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_converter", (PyCFunction)(void(*)(void))unsigned_long_converter, METH_FASTCALL, unsigned_long_converter__doc__}, + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c); + +static PyObject * +unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long a = 12; + unsigned long b = 34; + unsigned long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongMask(args[2]); +skip_optional: + return_value = unsigned_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_long_converter__doc__, +"long_long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_LONG_CONVERTER_METHODDEF \ + {"long_long_converter", (PyCFunction)(void(*)(void))long_long_converter, METH_FASTCALL, long_long_converter__doc__}, + +static PyObject * +long_long_converter_impl(PyObject *module, long long a); + +static PyObject * +long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long long a = 12; + + if (!_PyArg_CheckPositional("long_long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLongLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_long_converter__doc__, +"unsigned_long_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_long_converter", (PyCFunction)(void(*)(void))unsigned_long_long_converter, METH_FASTCALL, unsigned_long_long_converter__doc__}, + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c); + +static PyObject * +unsigned_long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long long a = 12; + unsigned long long b = 34; + unsigned long long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongLongMask(args[2]); +skip_optional: + return_value = unsigned_long_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_ssize_t_converter__doc__, +"py_ssize_t_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define PY_SSIZE_T_CONVERTER_METHODDEF \ + {"py_ssize_t_converter", (PyCFunction)(void(*)(void))py_ssize_t_converter, METH_FASTCALL, py_ssize_t_converter__doc__}, + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("py_ssize_t_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + a = ival; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + b = ival; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = py_ssize_t_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(slice_index_converter__doc__, +"slice_index_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define SLICE_INDEX_CONVERTER_METHODDEF \ + {"slice_index_converter", (PyCFunction)(void(*)(void))slice_index_converter, METH_FASTCALL, slice_index_converter__doc__}, + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +slice_index_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("slice_index_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = slice_index_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(size_t_converter__doc__, +"size_t_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SIZE_T_CONVERTER_METHODDEF \ + {"size_t_converter", (PyCFunction)(void(*)(void))size_t_converter, METH_FASTCALL, size_t_converter__doc__}, + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a); + +static PyObject * +size_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + size_t a = 12; + + if (!_PyArg_CheckPositional("size_t_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_Size_t_Converter(args[0], &a)) { + goto exit; + } +skip_optional: + return_value = size_t_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(float_converter__doc__, +"float_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define FLOAT_CONVERTER_METHODDEF \ + {"float_converter", (PyCFunction)(void(*)(void))float_converter, METH_FASTCALL, float_converter__doc__}, + +static PyObject * +float_converter_impl(PyObject *module, float a); + +static PyObject * +float_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float a = 12.5; + + if (!_PyArg_CheckPositional("float_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + a = (float) PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = float_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(double_converter__doc__, +"double_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define DOUBLE_CONVERTER_METHODDEF \ + {"double_converter", (PyCFunction)(void(*)(void))double_converter, METH_FASTCALL, double_converter__doc__}, + +static PyObject * +double_converter_impl(PyObject *module, double a); + +static PyObject * +double_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double a = 12.5; + + if (!_PyArg_CheckPositional("double_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = PyFloat_AS_DOUBLE(args[0]); + } + else + { + a = PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = double_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_complex_converter__doc__, +"py_complex_converter($module, a, /)\n" +"--\n" +"\n"); + +#define PY_COMPLEX_CONVERTER_METHODDEF \ + {"py_complex_converter", (PyCFunction)py_complex_converter, METH_O, py_complex_converter__doc__}, + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a); + +static PyObject * +py_complex_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_complex a; + + a = PyComplex_AsCComplex(arg); + if (PyErr_Occurred()) { + goto exit; + } + return_value = py_complex_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(str_converter__doc__, +"str_converter($module, a=\'a\', b=\'b\', c=\'c\', /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_METHODDEF \ + {"str_converter", (PyCFunction)(void(*)(void))str_converter, METH_FASTCALL, str_converter__doc__}, + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_clean_t c_length); + +static PyObject * +str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + const char *a = "a"; + const char *b = "b"; + const char *c = "c"; + Py_ssize_clean_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "|sys#:str_converter", + &a, &b, &c, &c_length)) { + goto exit; + } + return_value = str_converter_impl(module, a, b, c, c_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_buffer_converter__doc__, +"py_buffer_converter($module, a, b, /)\n" +"--\n" +"\n"); + +#define PY_BUFFER_CONVERTER_METHODDEF \ + {"py_buffer_converter", (PyCFunction)(void(*)(void))py_buffer_converter, METH_FASTCALL, py_buffer_converter__doc__}, + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b); + +static PyObject * +py_buffer_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer a = {NULL, NULL}; + Py_buffer b = {NULL, NULL}; + + if (!_PyArg_ParseStack(args, nargs, "z*w*:py_buffer_converter", + &a, &b)) { + goto exit; + } + return_value = py_buffer_converter_impl(module, &a, &b); + +exit: + /* Cleanup for a */ + if (a.obj) { + PyBuffer_Release(&a); + } + /* Cleanup for b */ + if (b.obj) { + PyBuffer_Release(&b); + } + + return return_value; +} + +PyDoc_STRVAR(keywords__doc__, +"keywords($module, /, a, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_METHODDEF \ + {"keywords", (PyCFunction)(void(*)(void))keywords, METH_FASTCALL|METH_KEYWORDS, keywords__doc__}, + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly__doc__, +"keywords_kwonly($module, /, a, *, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_METHODDEF \ + {"keywords_kwonly", (PyCFunction)(void(*)(void))keywords_kwonly, METH_FASTCALL|METH_KEYWORDS, keywords_kwonly__doc__}, + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt__doc__, +"keywords_opt($module, /, a, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_METHODDEF \ + {"keywords_opt", (PyCFunction)(void(*)(void))keywords_opt, METH_FASTCALL|METH_KEYWORDS, keywords_opt__doc__}, + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); + +static PyObject * +keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = keywords_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt_kwonly__doc__, +"keywords_opt_kwonly($module, /, a, b=None, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_KWONLY_METHODDEF \ + {"keywords_opt_kwonly", (PyCFunction)(void(*)(void))keywords_opt_kwonly, METH_FASTCALL|METH_KEYWORDS, keywords_opt_kwonly__doc__}, + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt_kwonly", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = keywords_opt_kwonly_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly_opt__doc__, +"keywords_kwonly_opt($module, /, a, *, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_OPT_METHODDEF \ + {"keywords_kwonly_opt", (PyCFunction)(void(*)(void))keywords_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, keywords_kwonly_opt__doc__}, + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[2]; +skip_optional_kwonly: + return_value = keywords_kwonly_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords__doc__, +"posonly_keywords($module, a, /, b)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_METHODDEF \ + {"posonly_keywords", (PyCFunction)(void(*)(void))posonly_keywords, METH_FASTCALL|METH_KEYWORDS, posonly_keywords__doc__}, + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly__doc__, +"posonly_kwonly($module, a, /, *, b)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_METHODDEF \ + {"posonly_kwonly", (PyCFunction)(void(*)(void))posonly_kwonly, METH_FASTCALL|METH_KEYWORDS, posonly_kwonly__doc__}, + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly__doc__, +"posonly_keywords_kwonly($module, a, /, b, *, c)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_METHODDEF \ + {"posonly_keywords_kwonly", (PyCFunction)(void(*)(void))posonly_keywords_kwonly, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly__doc__}, + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly", 0}; + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = posonly_keywords_kwonly_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt__doc__, +"posonly_keywords_opt($module, a, /, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_METHODDEF \ + {"posonly_keywords_opt", (PyCFunction)(void(*)(void))posonly_keywords_opt, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt__doc__}, + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt__doc__, +"posonly_opt_keywords_opt($module, a, b=None, /, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_METHODDEF \ + {"posonly_opt_keywords_opt", (PyCFunction)(void(*)(void))posonly_opt_keywords_opt, METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_opt_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly_opt__doc__, +"posonly_kwonly_opt($module, a, /, *, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_OPT_METHODDEF \ + {"posonly_kwonly_opt", (PyCFunction)(void(*)(void))posonly_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_kwonly_opt__doc__}, + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_kwonly_opt__doc__, +"posonly_opt_kwonly_opt($module, a, b=None, /, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_kwonly_opt", (PyCFunction)(void(*)(void))posonly_opt_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly_opt__doc__, +"posonly_keywords_kwonly_opt($module, a, /, b, *, c, d=None, e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_kwonly_opt", (PyCFunction)(void(*)(void))posonly_keywords_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt_kwonly_opt__doc__, +"posonly_keywords_opt_kwonly_opt($module, a, /, b, c=None, *, d=None,\n" +" e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_opt_kwonly_opt", (PyCFunction)(void(*)(void))posonly_keywords_opt_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e); + +static PyObject * +posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_opt_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt_kwonly_opt__doc__, +"posonly_opt_keywords_opt_kwonly_opt($module, a, b=None, /, c=None, *,\n" +" d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_keywords_opt_kwonly_opt", (PyCFunction)(void(*)(void))posonly_opt_keywords_opt_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d); + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_keywords_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keyword_only_parameter__doc__, +"keyword_only_parameter($module, /, *, a)\n" +"--\n" +"\n"); + +#define KEYWORD_ONLY_PARAMETER_METHODDEF \ + {"keyword_only_parameter", (PyCFunction)(void(*)(void))keyword_only_parameter, METH_FASTCALL|METH_KEYWORDS, keyword_only_parameter__doc__}, + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a); + +static PyObject * +keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keyword_only_parameter", 0}; + PyObject *argsbuf[1]; + PyObject *a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = keyword_only_parameter_impl(module, a); + +exit: + return return_value; +} +/*[clinic end generated code: output=2d5ac6b41daadf6f input=a9049054013a1b77]*/ diff --git a/Tools/scripts/generate_stdlib_module_names.py b/Tools/scripts/generate_stdlib_module_names.py index dc6a07427471..efea9a7b20a1 100644 --- a/Tools/scripts/generate_stdlib_module_names.py +++ b/Tools/scripts/generate_stdlib_module_names.py @@ -24,6 +24,7 @@ '_ctypes_test', '_testbuffer', '_testcapi', + '_testclinic', '_testconsole', '_testimportmultiple', '_testinternalcapi', diff --git a/setup.py b/setup.py index 85a2b26357db..a39610a1c7cd 100644 --- a/setup.py +++ b/setup.py @@ -1043,6 +1043,9 @@ def detect_test_extensions(self): self.add(Extension('_testcapi', ['_testcapimodule.c'], depends=['testcapi_long.h'])) + # Python Argument Clinc functional test module + self.add(Extension('_testclinic', ['_testclinic.c'])) + # Python Internal C API test module self.add(Extension('_testinternalcapi', ['_testinternalcapi.c'], extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) From webhook-mailer at python.org Sat Dec 17 01:40:56 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 06:40:56 -0000 Subject: [Python-checkins] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (#99890) Message-ID: <mailman.2954.1671259256.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a6f82f1fc68cb24e2d88d35fde4cfb663213a744 commit: a6f82f1fc68cb24e2d88d35fde4cfb663213a744 branch: main author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:10:51+05:30 summary: gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (#99890) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: From webhook-mailer at python.org Sat Dec 17 01:53:25 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 06:53:25 -0000 Subject: [Python-checkins] Document that zipfile's pwd parameter is a `bytes` object (GH-100209) Message-ID: <mailman.2955.1671260006.3313.python-checkins@python.org> https://github.com/python/cpython/commit/697a07a8549fa117c047796ec8a59eb7f1fd5741 commit: 697a07a8549fa117c047796ec8a59eb7f1fd5741 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T22:53:20-08:00 summary: Document that zipfile's pwd parameter is a `bytes` object (GH-100209) (cherry picked from commit 5ee7eb9debb12914f36c5ccee92460a681516fd6) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index f6288709fec1..10fffb70dbdb 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -251,7 +251,8 @@ ZipFile Objects Access a member of the archive as a binary file-like object. *name* can be either the name of a file within the archive or a :class:`ZipInfo` object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files. + or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files as a + :class:`bytes` object. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -303,7 +304,7 @@ ZipFile Objects must be its full name or a :class:`ZipInfo` object. Its file information is extracted as accurately as possible. *path* specifies a different directory to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files. + *pwd* is the password used for encrypted files as a :class:`bytes` object. Returns the normalized path created (a directory or new file). @@ -330,7 +331,7 @@ ZipFile Objects Extract all members from the archive to the current working directory. *path* specifies a different directory to extract to. *members* is optional and must be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files. + used for encrypted files as a :class:`bytes` object. .. warning:: @@ -355,16 +356,16 @@ ZipFile Objects .. method:: ZipFile.setpassword(pwd) - Set *pwd* as default password to extract encrypted files. + Set *pwd* (a :class:`bytes` object) as default password to extract encrypted files. .. method:: ZipFile.read(name, pwd=None) Return the bytes of the file *name* in the archive. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files and, if specified, - it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a ZipFile that uses a compression method other than + read or append. *pwd* is the password used for encrypted files as a :class:`bytes` + object and, if specified, overrides the default password set with :meth:`setpassword`. + Calling :meth:`read` on a ZipFile that uses a compression method other than :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also be raised if the corresponding compression module is not available. From webhook-mailer at python.org Sat Dec 17 01:54:38 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 06:54:38 -0000 Subject: [Python-checkins] Document that zipfile's pwd parameter is a `bytes` object (GH-100209) Message-ID: <mailman.2956.1671260080.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e3d85783e56d9621b432819805c452f2f65669e0 commit: e3d85783e56d9621b432819805c452f2f65669e0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T22:54:33-08:00 summary: Document that zipfile's pwd parameter is a `bytes` object (GH-100209) (cherry picked from commit 5ee7eb9debb12914f36c5ccee92460a681516fd6) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4dd9fa961a8d..d02e32d320bc 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -273,7 +273,8 @@ ZipFile Objects Access a member of the archive as a binary file-like object. *name* can be either the name of a file within the archive or a :class:`ZipInfo` object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files. + or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files as a + :class:`bytes` object. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -325,7 +326,7 @@ ZipFile Objects must be its full name or a :class:`ZipInfo` object. Its file information is extracted as accurately as possible. *path* specifies a different directory to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files. + *pwd* is the password used for encrypted files as a :class:`bytes` object. Returns the normalized path created (a directory or new file). @@ -352,7 +353,7 @@ ZipFile Objects Extract all members from the archive to the current working directory. *path* specifies a different directory to extract to. *members* is optional and must be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files. + used for encrypted files as a :class:`bytes` object. .. warning:: @@ -377,16 +378,16 @@ ZipFile Objects .. method:: ZipFile.setpassword(pwd) - Set *pwd* as default password to extract encrypted files. + Set *pwd* (a :class:`bytes` object) as default password to extract encrypted files. .. method:: ZipFile.read(name, pwd=None) Return the bytes of the file *name* in the archive. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files and, if specified, - it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a ZipFile that uses a compression method other than + read or append. *pwd* is the password used for encrypted files as a :class:`bytes` + object and, if specified, overrides the default password set with :meth:`setpassword`. + Calling :meth:`read` on a ZipFile that uses a compression method other than :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also be raised if the corresponding compression module is not available. From webhook-mailer at python.org Sat Dec 17 02:12:45 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 07:12:45 -0000 Subject: [Python-checkins] gh-97909: Fix markup for `PyMethodDef` members (#100089) Message-ID: <mailman.2957.1671261165.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8edcb30c3f8bdd8099a093146fedbd9b63a3f667 commit: 8edcb30c3f8bdd8099a093146fedbd9b63a3f667 branch: main author: ram vikram singh <ramvikrams243 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:42:39+05:30 summary: gh-97909: Fix markup for `PyMethodDef` members (#100089) files: M Doc/c-api/structures.rst diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 827d624fc99e..be6e7b5faf9f 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -228,29 +228,30 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`ml_name` | const char \* | name of the method | - +------------------+---------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+---------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+---------------+-------------------------------+ - | :attr:`ml_doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ - -The :attr:`ml_meth` is a C function pointer. The functions may be of different + .. c:member:: const char* ml_name + + name of the method + + .. c:member:: PyCFunction ml_meth + + pointer to the C implementation + + .. c:member:: int ml_flags + + flags bits indicating how the call should be constructed + + .. c:member:: const char* ml_doc + + points to the contents of the docstring + +The :c:member:`ml_meth` is a C function pointer. The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :attr:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding convention. From webhook-mailer at python.org Sat Dec 17 02:19:50 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 07:19:50 -0000 Subject: [Python-checkins] gh-97909: Fix markup for `PyMethodDef` members (GH-100089) Message-ID: <mailman.2958.1671261591.3313.python-checkins@python.org> https://github.com/python/cpython/commit/370498b12ee9ce9edb93da0498a5cec1806147fc commit: 370498b12ee9ce9edb93da0498a5cec1806147fc branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T23:19:45-08:00 summary: gh-97909: Fix markup for `PyMethodDef` members (GH-100089) (cherry picked from commit 8edcb30c3f8bdd8099a093146fedbd9b63a3f667) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/c-api/structures.rst diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 2b5d9a84822b..30760c0db88c 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -247,29 +247,30 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`ml_name` | const char \* | name of the method | - +------------------+---------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+---------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+---------------+-------------------------------+ - | :attr:`ml_doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ + .. c:member:: const char* ml_name + + name of the method + + .. c:member:: PyCFunction ml_meth + + pointer to the C implementation + + .. c:member:: int ml_flags + + flags bits indicating how the call should be constructed + + .. c:member:: const char* ml_doc + + points to the contents of the docstring -The :attr:`ml_meth` is a C function pointer. The functions may be of different +The :c:member:`ml_meth` is a C function pointer. The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :attr:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding convention. From webhook-mailer at python.org Sat Dec 17 02:20:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 07:20:00 -0000 Subject: [Python-checkins] gh-97909: Fix markup for `PyMethodDef` members (GH-100089) Message-ID: <mailman.2959.1671261601.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e38a525c20f0fe68cfade983a472a1f011d5e3df commit: e38a525c20f0fe68cfade983a472a1f011d5e3df branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T23:19:54-08:00 summary: gh-97909: Fix markup for `PyMethodDef` members (GH-100089) (cherry picked from commit 8edcb30c3f8bdd8099a093146fedbd9b63a3f667) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/c-api/structures.rst diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 069774745654..4ad85d0428a0 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -235,29 +235,30 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`ml_name` | const char \* | name of the method | - +------------------+---------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+---------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+---------------+-------------------------------+ - | :attr:`ml_doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ + .. c:member:: const char* ml_name + + name of the method + + .. c:member:: PyCFunction ml_meth + + pointer to the C implementation + + .. c:member:: int ml_flags + + flags bits indicating how the call should be constructed + + .. c:member:: const char* ml_doc + + points to the contents of the docstring -The :attr:`ml_meth` is a C function pointer. The functions may be of different +The :c:member:`ml_meth` is a C function pointer. The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :attr:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding convention. From webhook-mailer at python.org Sat Dec 17 02:42:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 07:42:24 -0000 Subject: [Python-checkins] Docs: Use `PY_VERSION_HEX` for version comparison (#100179) Message-ID: <mailman.2960.1671262945.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0264f634f720fbf12afaf1715a53cd1495fbd85b commit: 0264f634f720fbf12afaf1715a53cd1495fbd85b branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T13:12:18+05:30 summary: Docs: Use `PY_VERSION_HEX` for version comparison (#100179) files: M Doc/c-api/apiabiversion.rst M Doc/whatsnew/3.11.rst diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 85b6e2f37387..62d542966622 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -58,6 +58,8 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is hexversion ``0x030a00f0``. + Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. + This version is also available via the symbol :data:`Py_Version`. .. c:var:: const unsigned long Py_Version diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7931988ed0b0..6f283f1d8002 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2319,7 +2319,7 @@ Porting to Python 3.11 can define the following macros and use them throughout the code (credit: these were copied from the ``mypy`` codebase):: - #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 + #if PY_VERSION_HEX >= 0x03080000 # define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) # define CPy_TRASHCAN_END(op) Py_TRASHCAN_END #else From webhook-mailer at python.org Sat Dec 17 02:49:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 07:49:59 -0000 Subject: [Python-checkins] Docs: Use `PY_VERSION_HEX` for version comparison (GH-100179) Message-ID: <mailman.2961.1671263401.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bf0a334c4e46041c5621f0271cb70feef934c501 commit: bf0a334c4e46041c5621f0271cb70feef934c501 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T23:49:49-08:00 summary: Docs: Use `PY_VERSION_HEX` for version comparison (GH-100179) (cherry picked from commit 0264f634f720fbf12afaf1715a53cd1495fbd85b) Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> files: M Doc/c-api/apiabiversion.rst M Doc/whatsnew/3.11.rst diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 85b6e2f37387..62d542966622 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -58,6 +58,8 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is hexversion ``0x030a00f0``. + Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. + This version is also available via the symbol :data:`Py_Version`. .. c:var:: const unsigned long Py_Version diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index f09ccb133fbc..4244875f6c69 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2320,7 +2320,7 @@ Porting to Python 3.11 can define the following macros and use them throughout the code (credit: these were copied from the ``mypy`` codebase):: - #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 + #if PY_VERSION_HEX >= 0x03080000 # define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) # define CPy_TRASHCAN_END(op) Py_TRASHCAN_END #else From webhook-mailer at python.org Sat Dec 17 03:53:42 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 17 Dec 2022 08:53:42 -0000 Subject: [Python-checkins] gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551) Message-ID: <mailman.2962.1671267223.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b38a9aa747785f6d540e95e6846d6510de6b306 commit: 2b38a9aa747785f6d540e95e6846d6510de6b306 branch: main author: Erlend E. Aasland <erlend.aasland at protonmail.com> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-17T09:53:36+01:00 summary: gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551) files: A Lib/test/test_capi/test_mem.py M Lib/test/test_capi/test_misc.py M Modules/_testcapi/mem.c M Modules/_testcapimodule.c diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py new file mode 100644 index 000000000000..a9ff410cb93a --- /dev/null +++ b/Lib/test/test_capi/test_mem.py @@ -0,0 +1,169 @@ +import re +import textwrap +import unittest + + +from test import support +from test.support import import_helper, requires_subprocess +from test.support.script_helper import assert_python_failure, assert_python_ok + + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + + at requires_subprocess() +class PyMemDebugTests(unittest.TestCase): + PYTHONMALLOC = 'debug' + # '0x04c06e0' or '04C06E0' + PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + # FreeBSD: instruct jemalloc to not fill freed() memory + # with junk byte 0x5a, see JEMALLOC(3) + MALLOC_CONF="junk:false", + ) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE $0x[0-9a-f]{{2}}$:\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfd\n" + r" at tail\+2: 0xfd\n" + r" .*\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") + regex = regex.format(ptr=self.PTR_REGEX) + regex = re.compile(regex, flags=re.DOTALL) + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") + regex = regex.format(ptr=self.PTR_REGEX) + self.assertRegex(out, regex) + + def check_malloc_without_gil(self, code): + out = self.check(code) + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without holding the GIL') + self.assertIn(expected, out) + + def test_pymem_malloc_without_gil(self): + # Debug hooks must raise an error if PyMem_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def test_pyobject_malloc_without_gil(self): + # Debug hooks must raise an error if PyObject_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def check_pyobject_is_freed(self, func_name): + code = textwrap.dedent(f''' + import gc, os, sys, _testcapi + # Disable the GC to avoid crash on GC collection + gc.disable() + try: + _testcapi.{func_name}() + # Exit immediately to avoid a crash while deallocating + # the invalid object + os._exit(0) + except _testcapi.error: + os._exit(1) + ''') + assert_python_ok( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + MALLOC_CONF="junk:false", + ) + + def test_pyobject_null_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_null_is_freed') + + def test_pyobject_uninitialized_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') + + def test_pyobject_forbidden_bytes_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') + + def test_pyobject_freed_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_freed_is_freed') + + def test_set_nomemory(self): + code = """if 1: + import _testcapi + + class C(): pass + + # The first loop tests both functions and that remove_mem_hooks() + # can be called twice in a row. The second loop checks a call to + # set_nomemory() after a call to remove_mem_hooks(). The third + # loop checks the start and stop arguments of set_nomemory(). + for outer_cnt in range(1, 4): + start = 10 * outer_cnt + for j in range(100): + if j == 0: + if outer_cnt != 3: + _testcapi.set_nomemory(start) + else: + _testcapi.set_nomemory(start, start + 1) + try: + C() + except MemoryError as e: + if outer_cnt != 3: + _testcapi.remove_mem_hooks() + print('MemoryError', outer_cnt, j) + _testcapi.remove_mem_hooks() + break + """ + rc, out, err = assert_python_ok('-c', code) + lines = out.splitlines() + for i, line in enumerate(lines, 1): + self.assertIn(b'MemoryError', out) + *_, count = line.split(b' ') + count = int(count) + self.assertLessEqual(count, i*5) + self.assertGreaterEqual(count, i*5-2) + + +class PyMemMallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'malloc_debug' + + + at unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') +class PyMemPymallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'pymalloc_debug' + + + at unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') +class PyMemDefaultTests(PyMemDebugTests): + # test default allocator of Python compiled in debug mode + PYTHONMALLOC = '' + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 06a51aa3cc21..dace37c362e5 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -323,42 +323,6 @@ def test_getitem_with_error(self): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() - def test_set_nomemory(self): - code = """if 1: - import _testcapi - - class C(): pass - - # The first loop tests both functions and that remove_mem_hooks() - # can be called twice in a row. The second loop checks a call to - # set_nomemory() after a call to remove_mem_hooks(). The third - # loop checks the start and stop arguments of set_nomemory(). - for outer_cnt in range(1, 4): - start = 10 * outer_cnt - for j in range(100): - if j == 0: - if outer_cnt != 3: - _testcapi.set_nomemory(start) - else: - _testcapi.set_nomemory(start, start + 1) - try: - C() - except MemoryError as e: - if outer_cnt != 3: - _testcapi.remove_mem_hooks() - print('MemoryError', outer_cnt, j) - _testcapi.remove_mem_hooks() - break - """ - rc, out, err = assert_python_ok('-c', code) - lines = out.splitlines() - for i, line in enumerate(lines, 1): - self.assertIn(b'MemoryError', out) - *_, count = line.split(b' ') - count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) - def test_mapping_keys_values_items(self): class Mapping1(dict): def keys(self): @@ -1470,124 +1434,6 @@ class Test_testinternalcapi(unittest.TestCase): if name.startswith('test_')) - at support.requires_subprocess() -class PyMemDebugTests(unittest.TestCase): - PYTHONMALLOC = 'debug' - # '0x04c06e0' or '04C06E0' - PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' - - def check(self, code): - with support.SuppressCrashReport(): - out = assert_python_failure( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - # FreeBSD: instruct jemalloc to not fill freed() memory - # with junk byte 0x5a, see JEMALLOC(3) - MALLOC_CONF="junk:false", - ) - stderr = out.err - return stderr.decode('ascii', 'replace') - - def test_buffer_overflow(self): - out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE $0x[0-9a-f]{{2}}$:\n" - r" at tail\+0: 0x78 \*\*\* OUCH\n" - r" at tail\+1: 0xfd\n" - r" at tail\+2: 0xfd\n" - r" .*\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") - regex = regex.format(ptr=self.PTR_REGEX) - regex = re.compile(regex, flags=re.DOTALL) - self.assertRegex(out, regex) - - def test_api_misuse(self): - out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") - regex = regex.format(ptr=self.PTR_REGEX) - self.assertRegex(out, regex) - - def check_malloc_without_gil(self, code): - out = self.check(code) - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without holding the GIL') - self.assertIn(expected, out) - - def test_pymem_malloc_without_gil(self): - # Debug hooks must raise an error if PyMem_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def test_pyobject_malloc_without_gil(self): - # Debug hooks must raise an error if PyObject_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def check_pyobject_is_freed(self, func_name): - code = textwrap.dedent(f''' - import gc, os, sys, _testcapi - # Disable the GC to avoid crash on GC collection - gc.disable() - try: - _testcapi.{func_name}() - # Exit immediately to avoid a crash while deallocating - # the invalid object - os._exit(0) - except _testcapi.error: - os._exit(1) - ''') - assert_python_ok( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - MALLOC_CONF="junk:false", - ) - - def test_pyobject_null_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_null_is_freed') - - def test_pyobject_uninitialized_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') - - def test_pyobject_forbidden_bytes_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') - - def test_pyobject_freed_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_freed_is_freed') - - -class PyMemMallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'malloc_debug' - - - at unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') -class PyMemPymallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'pymalloc_debug' - - - at unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') -class PyMemDefaultTests(PyMemDebugTests): - # test default allocator of Python compiled in debug mode - PYTHONMALLOC = '' - - @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") class Test_ModuleStateAccess(unittest.TestCase): """Test access to module start (PEP 573)""" diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index ef723bf06585..ae3f7a4372dc 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -596,6 +596,82 @@ check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) #endif } +// Tracemalloc tests +static PyObject * +tracemalloc_track(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + Py_ssize_t size; + int release_gil = 0; + + if (!PyArg_ParseTuple(args, "IOn|i", + &domain, &ptr_obj, &size, &release_gil)) + { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + int res; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); + Py_END_ALLOW_THREADS + } + else { + res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); + } + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_untrack(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_get_traceback(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); +} + static PyMethodDef test_methods[] = { {"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, @@ -617,6 +693,11 @@ static PyMethodDef test_methods[] = { {"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS}, {"test_pyobject_new", test_pyobject_new, METH_NOARGS}, {"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS}, + + // Tracemalloc tests + {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, + {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, + {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 83eef73a875d..35c895d9ceb2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2065,78 +2065,6 @@ getitem_with_error(PyObject *self, PyObject *args) return PyObject_GetItem(map, key); } -static PyObject * -tracemalloc_track(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - Py_ssize_t size; - int release_gil = 0; - int res; - - if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - if (release_gil) { - Py_BEGIN_ALLOW_THREADS - res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); - Py_END_ALLOW_THREADS - } - else { - res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); - } - - if (res < 0) { - PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error"); - return NULL; - } - - Py_RETURN_NONE; -} - -static PyObject * -tracemalloc_untrack(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - int res; - - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); - if (res < 0) { - PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); - return NULL; - } - - Py_RETURN_NONE; -} - -static PyObject * -tracemalloc_get_traceback(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); -} - static PyObject * dict_get_version(PyObject *self, PyObject *args) { @@ -3301,9 +3229,6 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, - {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, - {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, - {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, From webhook-mailer at python.org Sat Dec 17 07:33:06 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 17 Dec 2022 12:33:06 -0000 Subject: [Python-checkins] gh-100272: Fix JSON serialization of OrderedDict (GH-100273) Message-ID: <mailman.2963.1671280387.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0fe61d0838218b18bcff4bf434beba3e8dbcc42b commit: 0fe61d0838218b18bcff4bf434beba3e8dbcc42b branch: main author: Serhiy Storchaka <storchaka at gmail.com> committer: serhiy-storchaka <storchaka at gmail.com> date: 2022-12-17T14:32:48+02:00 summary: gh-100272: Fix JSON serialization of OrderedDict (GH-100273) It now preserves the order of keys. files: A Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst M Lib/test/test_json/test_default.py M Modules/_json.c diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 9b8325e9c381..3ce16684a082 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -1,3 +1,4 @@ +import collections from test.test_json import PyTest, CTest @@ -7,6 +8,16 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + def test_ordereddict(self): + od = collections.OrderedDict(a=1, b=2, c=3, d=4) + od.move_to_end('b') + self.assertEqual( + self.dumps(od), + '{"a": 1, "c": 3, "d": 4, "b": 2}') + self.assertEqual( + self.dumps(od, sort_keys=True), + '{"a": 1, "b": 2, "c": 3, "d": 4}') + class TestPyDefault(TestDefault, PyTest): pass class TestCDefault(TestDefault, CTest): pass diff --git a/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst b/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst new file mode 100644 index 000000000000..2fb089261792 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst @@ -0,0 +1 @@ +Fix JSON serialization of OrderedDict. It now preserves the order of keys. diff --git a/Modules/_json.c b/Modules/_json.c index 429b4ee0fa8d..6879ad3d0722 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1570,10 +1570,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, */ } - if (s->sort_keys) { - - items = PyDict_Items(dct); - if (items == NULL || PyList_Sort(items) < 0) + if (s->sort_keys || !PyDict_CheckExact(dct)) { + items = PyMapping_Items(dct); + if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) goto bail; for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { From webhook-mailer at python.org Sat Dec 17 20:10:37 2022 From: webhook-mailer at python.org (rhettinger) Date: Sun, 18 Dec 2022 01:10:37 -0000 Subject: [Python-checkins] [3.11] GH-98363: Update batched() recipe in docs to match 3.12 (#100323) Message-ID: <mailman.2964.1671325838.3313.python-checkins@python.org> https://github.com/python/cpython/commit/09186676cf3d483d19d64f5f1ed23b5a7e6edbbb commit: 09186676cf3d483d19d64f5f1ed23b5a7e6edbbb branch: 3.11 author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-17T19:10:03-06:00 summary: [3.11] GH-98363: Update batched() recipe in docs to match 3.12 (#100323) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index ee70b16f42fb..8eb843ab0a67 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -881,12 +881,12 @@ which incur interpreter overhead. raise ValueError('Expected fill, strict, or ignore') def batched(iterable, n): - "Batch data into lists of length n. The last batch may be shorter." + "Batch data into tuples of length n. The last batch may be shorter." # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: raise ValueError('n must be at least one') it = iter(iterable) - while (batch := list(islice(it, n))): + while (batch := tuple(islice(it, n))): yield batch def triplewise(iterable): @@ -1248,25 +1248,25 @@ which incur interpreter overhead. [('a', 'b', 'c'), ('d', 'e', 'f')] >>> list(batched('ABCDEFG', 3)) - [['A', 'B', 'C'], ['D', 'E', 'F'], ['G']] + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)] >>> list(batched('ABCDEF', 3)) - [['A', 'B', 'C'], ['D', 'E', 'F']] + [('A', 'B', 'C'), ('D', 'E', 'F')] >>> list(batched('ABCDE', 3)) - [['A', 'B', 'C'], ['D', 'E']] + [('A', 'B', 'C'), ('D', 'E')] >>> list(batched('ABCD', 3)) - [['A', 'B', 'C'], ['D']] + [('A', 'B', 'C'), ('D',)] >>> list(batched('ABC', 3)) - [['A', 'B', 'C']] + [('A', 'B', 'C')] >>> list(batched('AB', 3)) - [['A', 'B']] + [('A', 'B')] >>> list(batched('A', 3)) - [['A']] + [('A',)] >>> list(batched('', 3)) [] >>> list(batched('ABCDEFG', 2)) - [['A', 'B'], ['C', 'D'], ['E', 'F'], ['G']] + [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)] >>> list(batched('ABCDEFG', 1)) - [['A'], ['B'], ['C'], ['D'], ['E'], ['F'], ['G']] + [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)] >>> s = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' >>> all(list(flatten(batched(s[:n], 5))) == list(s[:n]) for n in range(len(s))) True From webhook-mailer at python.org Sun Dec 18 02:11:34 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 18 Dec 2022 07:11:34 -0000 Subject: [Python-checkins] Correct CVE-2020-10735 documentation (#100306) Message-ID: <mailman.2965.1671347495.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1cf3d78c92eb07dc09d15cc2e773b0b1b9436825 commit: 1cf3d78c92eb07dc09d15cc2e773b0b1b9436825 branch: main author: Jeremy Paige <ucodery at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-18T12:41:21+05:30 summary: Correct CVE-2020-10735 documentation (#100306) files: M Doc/library/stdtypes.rst M Python/clinic/sysmodule.c.h M Python/sysmodule.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c785336944f5..5a6b7d83edf7 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5480,7 +5480,7 @@ to mitigate denial of service attacks. This limit *only* applies to decimal or other non-power-of-two number bases. Hexadecimal, octal, and binary conversions are unlimited. The limit can be configured. -The :class:`int` type in CPython is an abitrary length number stored in binary +The :class:`int` type in CPython is an arbitrary length number stored in binary form (commonly known as a "bignum"). There exists no algorithm that can convert a string to a binary integer or a binary integer to a string in linear time, *unless* the base is a power of 2. Even the best known algorithms for base 10 @@ -5544,7 +5544,7 @@ and :class:`str` or :class:`bytes`: * ``int(string)`` with default base 10. * ``int(string, base)`` for all bases that are not a power of 2. * ``str(integer)``. -* ``repr(integer)`` +* ``repr(integer)``. * any other string conversion to base 10, for example ``f"{integer}"``, ``"{}".format(integer)``, or ``b"%d" % integer``. @@ -5572,7 +5572,7 @@ command line flag to configure the limit: :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. If both the env var and the ``-X`` option are set, the ``-X`` option takes precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initilization. + :data:`sys.int_info.default_max_str_digits` was used during initialization. From code, you can inspect the current limit and set a new one using these :mod:`sys` APIs: diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 5678d0ac2a60..03eeda8126eb 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -749,7 +749,7 @@ PyDoc_STRVAR(sys_get_int_max_str_digits__doc__, "get_int_max_str_digits($module, /)\n" "--\n" "\n" -"Set the maximum string digits limit for non-binary int<->str conversions."); +"Return the maximum string digits limit for non-binary int<->str conversions."); #define SYS_GET_INT_MAX_STR_DIGITS_METHODDEF \ {"get_int_max_str_digits", (PyCFunction)sys_get_int_max_str_digits, METH_NOARGS, sys_get_int_max_str_digits__doc__}, @@ -1318,4 +1318,4 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=79228e569529129c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b32b444538dfd354 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 91f5c487c98f..3f0baf98890b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1699,12 +1699,12 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] sys.get_int_max_str_digits -Set the maximum string digits limit for non-binary int<->str conversions. +Return the maximum string digits limit for non-binary int<->str conversions. [clinic start generated code]*/ static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=8dab13e2023e60d5]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromLong(interp->long_state.max_str_digits); From webhook-mailer at python.org Sun Dec 18 14:13:36 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sun, 18 Dec 2022 19:13:36 -0000 Subject: [Python-checkins] gh-94912: Added marker for non-standard coroutine function detection (#99247) Message-ID: <mailman.2966.1671390817.3313.python-checkins@python.org> https://github.com/python/cpython/commit/532aa4e4e019812d0388920768ede7c04232ebe1 commit: 532aa4e4e019812d0388920768ede7c04232ebe1 branch: main author: Carlton Gibson <carlton at noumenal.es> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-18T11:13:24-08:00 summary: gh-94912: Added marker for non-standard coroutine function detection (#99247) This introduces a new decorator `@inspect.markcoroutinefunction`, which, applied to a sync function, makes it appear async to `inspect.iscoroutinefunction()`. files: A Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst M Doc/library/inspect.rst M Doc/whatsnew/3.12.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 6705577551dc..58b84a35a890 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -343,8 +343,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: iscoroutinefunction(object) - Return ``True`` if the object is a :term:`coroutine function` - (a function defined with an :keyword:`async def` syntax). + Return ``True`` if the object is a :term:`coroutine function` (a function + defined with an :keyword:`async def` syntax), a :func:`functools.partial` + wrapping a :term:`coroutine function`, or a sync function marked with + :func:`markcoroutinefunction`. .. versionadded:: 3.5 @@ -352,6 +354,25 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a :term:`coroutine function`. + .. versionchanged:: 3.12 + Sync functions marked with :func:`markcoroutinefunction` now return + ``True``. + + +.. function:: markcoroutinefunction(func) + + Decorator to mark a callable as a :term:`coroutine function` if it would not + otherwise be detected by :func:`iscoroutinefunction`. + + This may be of use for sync functions that return a :term:`coroutine`, if + the function is passed to an API that requires :func:`iscoroutinefunction`. + + When possible, using an :keyword:`async def` function is preferred. Also + acceptable is calling the function and testing the return with + :func:`iscoroutine`. + + .. versionadded:: 3.12 + .. function:: iscoroutine(object) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 73dc462f0b33..0cc4471364b6 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -225,6 +225,12 @@ asyncio a custom event loop factory. (Contributed by Kumar Aditya in :gh:`99388`.) +inspect +------- + +* Add :func:`inspect.markcoroutinefunction` to mark sync functions that return + a :term:`coroutine` for use with :func:`iscoroutinefunction`. + (Contributed Carlton Gibson in :gh:`99247`.) pathlib ------- diff --git a/Lib/inspect.py b/Lib/inspect.py index e92c355220fd..052f0bfe64b6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -125,6 +125,7 @@ "ismodule", "isroutine", "istraceback", + "markcoroutinefunction", "signature", "stack", "trace", @@ -391,12 +392,33 @@ def isgeneratorfunction(obj): See help(isfunction) for a list of attributes.""" return _has_code_flag(obj, CO_GENERATOR) +# A marker for markcoroutinefunction and iscoroutinefunction. +_is_coroutine_marker = object() + +def _has_coroutine_mark(f): + while ismethod(f): + f = f.__func__ + f = functools._unwrap_partial(f) + if not (isfunction(f) or _signature_is_functionlike(f)): + return False + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker + +def markcoroutinefunction(func): + """ + Decorator to ensure callable is recognised as a coroutine function. + """ + if hasattr(func, '__func__'): + func = func.__func__ + func._is_coroutine_marker = _is_coroutine_marker + return func + def iscoroutinefunction(obj): """Return true if the object is a coroutine function. - Coroutine functions are defined with "async def" syntax. + Coroutine functions are normally defined with "async def" syntax, but may + be marked via markcoroutinefunction. """ - return _has_code_flag(obj, CO_COROUTINE) + return _has_code_flag(obj, CO_COROUTINE) or _has_coroutine_mark(obj) def isasyncgenfunction(obj): """Return true if the object is an asynchronous generator function. diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 0f8217ba3b75..78e6e9e34e50 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -202,6 +202,51 @@ def test_iscoroutine(self): gen_coroutine_function_example)))) self.assertTrue(inspect.isgenerator(gen_coro)) + async def _fn3(): + pass + + @inspect.markcoroutinefunction + def fn3(): + return _fn3() + + self.assertTrue(inspect.iscoroutinefunction(fn3)) + self.assertTrue( + inspect.iscoroutinefunction( + inspect.markcoroutinefunction(lambda: _fn3()) + ) + ) + + class Cl: + async def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl)) + # instances with async def __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl())) + + class Cl2: + @inspect.markcoroutinefunction + def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl2)) + # instances with marked __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl2())) + + class Cl3: + @inspect.markcoroutinefunction + @classmethod + def do_something_classy(cls): + pass + + @inspect.markcoroutinefunction + @staticmethod + def do_something_static(): + pass + + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_classy)) + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_static)) + self.assertFalse( inspect.iscoroutinefunction(unittest.mock.Mock())) self.assertTrue( diff --git a/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst b/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst new file mode 100644 index 000000000000..ee00f9d8d03f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst @@ -0,0 +1,2 @@ +Add :func:`inspect.markcoroutinefunction` decorator which manually marks +a function as a coroutine for the benefit of :func:`iscoroutinefunction`. From webhook-mailer at python.org Mon Dec 19 03:40:22 2022 From: webhook-mailer at python.org (hugovk) Date: Mon, 19 Dec 2022 08:40:22 -0000 Subject: [Python-checkins] Docs: Don't upload CI artifacts (#100330) Message-ID: <mailman.2967.1671439223.3313.python-checkins@python.org> https://github.com/python/cpython/commit/702a5bc4637c82dc011e98b84f0cede98eb08dda commit: 702a5bc4637c82dc011e98b84f0cede98eb08dda branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-19T10:40:11+02:00 summary: Docs: Don't upload CI artifacts (#100330) files: M .github/workflows/doc.yml diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 44a1f206df1e..465da12fa1be 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -50,18 +50,8 @@ jobs: run: make -C Doc/ venv - name: 'Check documentation' run: make -C Doc/ check - - name: 'Upload NEWS' - uses: actions/upload-artifact at v3 - with: - name: NEWS - path: Doc/build/NEWS - name: 'Build HTML documentation' run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - - name: 'Upload docs' - uses: actions/upload-artifact at v3 - with: - name: doc-html - path: Doc/build/html # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: From webhook-mailer at python.org Mon Dec 19 04:49:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 19 Dec 2022 09:49:23 -0000 Subject: [Python-checkins] [3.11] gh-64490: Fix refcount error when arguments are packed to tuple in argument clinic (GH-99233) (#100338) Message-ID: <mailman.2968.1671443363.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c42a4ad58749273841ae6f504a4e661fb7c5ca3b commit: c42a4ad58749273841ae6f504a4e661fb7c5ca3b branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-19T15:18:49+05:30 summary: [3.11] gh-64490: Fix refcount error when arguments are packed to tuple in argument clinic (GH-99233) (#100338) (cherry picked from commit 69f6cc77d0f1664f983a83b6ae707d99a99f5c4f) files: A Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 0228d6be0326..8643f3fe78ae 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3339,7 +3339,7 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg a = args[0]; __clinic_args = PyTuple_New(nargs - 1); for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, args[1 + i]); + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } return_value = test_vararg_and_posonly_impl(module, a, __clinic_args); @@ -3350,7 +3350,7 @@ exit: static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=548bca3a127c22c1 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=081a953b8cbe7617 input=08dc2bf7afbf1613]*/ /*[clinic input] test_vararg diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 86c4f94ac028..af86d685866c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1211,6 +1211,20 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + def test_vararg_and_posonly(self): + with self.assertRaises(TypeError): + ac_tester.vararg_and_posonly() + with self.assertRaises(TypeError): + ac_tester.vararg_and_posonly(1, b=2) + self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_gh_99233_refcount(self): + arg = '*A unique string is not referenced by anywhere else.*' + arg_refcount_origin = sys.getrefcount(arg) + ac_tester.gh_99233_refcount(arg) + arg_refcount_after = sys.getrefcount(arg) + self.assertEqual(arg_refcount_origin, arg_refcount_after) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst b/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst new file mode 100644 index 000000000000..f98c181cc9c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst @@ -0,0 +1 @@ +Fix refcount error when arguments are packed to tuple in Argument Clinic. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index c9858e964457..a23ece2ae035 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -892,6 +892,41 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) } +/*[clinic input] +vararg_and_posonly + + a: object + *args: object + / + +[clinic start generated code]*/ + +static PyObject * +vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=42792f799465a14d input=defe017b19ba52e8]*/ +{ + return pack_arguments_newref(2, a, args); +} + + +/*[clinic input] +gh_99233_refcount + + *args: object + / + +Proof-of-concept of GH-99233 refcount error bug. + +[clinic start generated code]*/ + +static PyObject * +gh_99233_refcount_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=585855abfbca9a7f input=85f5fb47ac91a626]*/ +{ + Py_RETURN_NONE; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -933,6 +968,8 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF + VARARG_AND_POSONLY_METHODDEF + GH_99233_REFCOUNT_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index d363dd3f358e..c108d9d626a1 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1914,4 +1914,69 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=83f439d06635a2e9 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(vararg_and_posonly__doc__, +"vararg_and_posonly($module, a, /, *args)\n" +"--\n" +"\n"); + +#define VARARG_AND_POSONLY_METHODDEF \ + {"vararg_and_posonly", _PyCFunction_CAST(vararg_and_posonly), METH_FASTCALL, vararg_and_posonly__doc__}, + +static PyObject * +vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + __clinic_args = PyTuple_New(nargs - 1); + for (Py_ssize_t i = 0; i < nargs - 1; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); + } + return_value = vararg_and_posonly_impl(module, a, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(gh_99233_refcount__doc__, +"gh_99233_refcount($module, /, *args)\n" +"--\n" +"\n" +"Proof-of-concept of GH-99233 refcount error bug."); + +#define GH_99233_REFCOUNT_METHODDEF \ + {"gh_99233_refcount", _PyCFunction_CAST(gh_99233_refcount), METH_FASTCALL, gh_99233_refcount__doc__}, + +static PyObject * +gh_99233_refcount_impl(PyObject *module, PyObject *args); + +static PyObject * +gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("gh_99233_refcount", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = gh_99233_refcount_impl(module, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} +/*[clinic end generated code: output=16848e04bd5ddf7d input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 97d8d0a9410b..acd463bdd894 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -897,7 +897,7 @@ def parser_body(prototype, *fields, declarations=''): parser_code.append(normalize_snippet(""" %s = PyTuple_New(%s); for (Py_ssize_t i = 0; i < %s; ++i) {{ - PyTuple_SET_ITEM(%s, i, args[%d + i]); + PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); }} """ % ( p.converter.parser_name, From webhook-mailer at python.org Mon Dec 19 13:59:07 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Mon, 19 Dec 2022 18:59:07 -0000 Subject: [Python-checkins] gh-89727: Fix os.walk RecursionError on deep trees (#99803) Message-ID: <mailman.2969.1671476348.3313.python-checkins@python.org> https://github.com/python/cpython/commit/797edb28c3dd02a5727f0374e937e906a389ab77 commit: 797edb28c3dd02a5727f0374e937e906a389ab77 branch: main author: Jon Burdo <jon at jonburdo.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-19T10:59:01-08:00 summary: gh-89727: Fix os.walk RecursionError on deep trees (#99803) Use a stack to implement os.walk iteratively instead of recursively to avoid hitting recursion limits on deeply nested trees. files: A Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst M Lib/os.py M Lib/test/support/__init__.py M Lib/test/test_os.py diff --git a/Lib/os.py b/Lib/os.py index fd1e774fdcbc..73a5442ee8b8 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -340,89 +340,95 @@ def walk(top, topdown=True, onerror=None, followlinks=False): """ sys.audit("os.walk", top, topdown, onerror, followlinks) - return _walk(fspath(top), topdown, onerror, followlinks) - -def _walk(top, topdown, onerror, followlinks): - dirs = [] - nondirs = [] - walk_dirs = [] - - # We may not have read permission for top, in which case we can't - # get a list of the files the directory contains. os.walk - # always suppressed the exception then, rather than blow up for a - # minor reason when (say) a thousand readable directories are still - # left to visit. That logic is copied here. - try: - # Note that scandir is global in this module due - # to earlier import-*. - scandir_it = scandir(top) - except OSError as error: - if onerror is not None: - onerror(error) - return - with scandir_it: - while True: - try: + stack = [(False, fspath(top))] + islink, join = path.islink, path.join + while stack: + must_yield, top = stack.pop() + if must_yield: + yield top + continue + + dirs = [] + nondirs = [] + walk_dirs = [] + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. + # We suppress the exception here, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. + try: + scandir_it = scandir(top) + except OSError as error: + if onerror is not None: + onerror(error) + continue + + cont = False + with scandir_it: + while True: try: - entry = next(scandir_it) - except StopIteration: + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + cont = True break - except OSError as error: - if onerror is not None: - onerror(error) - return - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True + if is_dir: + dirs.append(entry.name) else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - - # Yield before recursion if going top down - if topdown: - yield top, dirs, nondirs - - # Recurse into sub-directories - islink, join = path.islink, path.join - for dirname in dirs: - new_path = join(top, dirname) - # Issue #23605: os.path.islink() is used instead of caching - # entry.is_symlink() result during the loop on os.scandir() because - # the caller can replace the directory entry during the "yield" - # above. - if followlinks or not islink(new_path): - yield from _walk(new_path, topdown, onerror, followlinks) - else: - # Recurse into sub-directories - for new_path in walk_dirs: - yield from _walk(new_path, topdown, onerror, followlinks) - # Yield after recursion if going bottom up - yield top, dirs, nondirs + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) + if cont: + continue + + if topdown: + # Yield before sub-directory traversal if going top down + yield top, dirs, nondirs + # Traverse into sub-directories + for dirname in reversed(dirs): + new_path = join(top, dirname) + # bpo-23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + stack.append((False, new_path)) + else: + # Yield after sub-directory traversal if going bottom up + stack.append((True, (top, dirs, nondirs))) + # Traverse into sub-directories + for new_path in reversed(walk_dirs): + stack.append((False, new_path)) __all__.append("walk") diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a631bfc80cfa..b7186057990a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2178,19 +2178,23 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) @contextlib.contextmanager +def set_recursion_limit(limit): + """Temporarily change the recursion limit.""" + original_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(limit) + yield + finally: + sys.setrecursionlimit(original_limit) + def infinite_recursion(max_depth=75): """Set a lower limit for tests that interact with infinite recursions (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some debug windows builds, due to not enough functions being inlined the stack size might not handle the default recursion limit (1000). See bpo-11105 for details.""" + return set_recursion_limit(max_depth) - original_depth = sys.getrecursionlimit() - try: - sys.setrecursionlimit(max_depth) - yield - finally: - sys.setrecursionlimit(original_depth) def ignore_deprecations_from(module: str, *, like: str) -> object: token = object() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e0577916428a..e6e25b507de0 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -33,6 +33,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper +from test.support import set_recursion_limit from test.support import warnings_helper from platform import win32_is_iot @@ -1471,6 +1472,46 @@ def test_walk_many_open_files(self): self.assertEqual(next(it), expected) p = os.path.join(p, 'd') + def test_walk_above_recursion_limit(self): + depth = 50 + os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) + with set_recursion_limit(depth - 5): + all = list(self.walk(self.walk_path)) + + sub2_path = self.sub2_tree[0] + for root, dirs, files in all: + if root == sub2_path: + dirs.sort() + files.sort() + + d_entries = [] + d_path = self.walk_path + for _ in range(depth): + d_path = os.path.join(d_path, "d") + d_entries.append((d_path, ["d"], [])) + d_entries[-1][1].clear() + + # Sub-sequences where the order is known + sections = { + "SUB1": [ + (self.sub1_path, ["SUB11"], ["tmp2"]), + (self.sub11_path, [], []), + ], + "SUB2": [self.sub2_tree], + "d": d_entries, + } + + # The ordering of sub-dirs is arbitrary but determines the order in + # which sub-sequences appear + dirs = all[0][1] + expected = [(self.walk_path, dirs, ["tmp1"])] + for d in dirs: + expected.extend(sections[d]) + + self.assertEqual(len(all), depth + 4) + self.assertEqual(sorted(dirs), ["SUB1", "SUB2", "d"]) + self.assertEqual(all, expected) + @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): @@ -1545,6 +1586,8 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None + # fwalk() still uses recursion + test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): diff --git a/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst new file mode 100644 index 000000000000..8a5fdb64b87f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.walk` where a :exc:`RecursionError` would occur on +deep directory structures by adjusting the implementation of +:func:`os.walk` to be iterative instead of recursive. From webhook-mailer at python.org Mon Dec 19 22:07:57 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Tue, 20 Dec 2022 03:07:57 -0000 Subject: [Python-checkins] gh-69929: re docs: Add more specific definition of \w (#92015) Message-ID: <mailman.2970.1671505677.3313.python-checkins@python.org> https://github.com/python/cpython/commit/36a0b1d0dddbdf324d98071ea31e7e9151eea6d5 commit: 36a0b1d0dddbdf324d98071ea31e7e9151eea6d5 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-19T19:07:31-08:00 summary: gh-69929: re docs: Add more specific definition of \w (#92015) Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index f7d46586cf75..cbee70b01d9f 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -591,10 +591,9 @@ character ``'$'``. ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched. + Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + as well as the underscore (``_``). + If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; From webhook-mailer at python.org Mon Dec 19 22:15:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 03:15:02 -0000 Subject: [Python-checkins] gh-69929: re docs: Add more specific definition of \w (GH-92015) Message-ID: <mailman.2971.1671506104.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cfa78ecc12255dd648c0d96bdeb4b74a1a686a95 commit: cfa78ecc12255dd648c0d96bdeb4b74a1a686a95 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-19T19:14:52-08:00 summary: gh-69929: re docs: Add more specific definition of \w (GH-92015) (cherry picked from commit 36a0b1d0dddbdf324d98071ea31e7e9151eea6d5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 8e279049a1c2..670569956522 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -589,10 +589,9 @@ character ``'$'``. ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched. + Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + as well as the underscore (``_``). + If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; From webhook-mailer at python.org Mon Dec 19 22:15:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 03:15:11 -0000 Subject: [Python-checkins] gh-69929: re docs: Add more specific definition of \w (GH-92015) Message-ID: <mailman.2972.1671506111.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b81d1c3be360c5f9a9dff0761e7f1f70a200f0aa commit: b81d1c3be360c5f9a9dff0761e7f1f70a200f0aa branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-19T19:14:48-08:00 summary: gh-69929: re docs: Add more specific definition of \w (GH-92015) (cherry picked from commit 36a0b1d0dddbdf324d98071ea31e7e9151eea6d5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index a40c99bf702f..716d3401a2bb 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -526,10 +526,9 @@ character ``'$'``. ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched. + Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + as well as the underscore (``_``). + If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; From webhook-mailer at python.org Tue Dec 20 02:10:39 2022 From: webhook-mailer at python.org (hugovk) Date: Tue, 20 Dec 2022 07:10:39 -0000 Subject: [Python-checkins] gh-89051: Add ssl.OP_LEGACY_SERVER_CONNECT (#93927) Message-ID: <mailman.2973.1671520239.3313.python-checkins@python.org> https://github.com/python/cpython/commit/79ccc03b62d819d83e592c6c8038545d9263a0d4 commit: 79ccc03b62d819d83e592c6c8038545d9263a0d4 branch: main author: Thomas Grainger <tagrain at gmail.com> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-20T09:10:30+02:00 summary: gh-89051: Add ssl.OP_LEGACY_SERVER_CONNECT (#93927) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Christian Heimes <christian at python.org> Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> Fixes https://github.com/python/cpython/issues/89051 files: A Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst M Doc/library/ssl.rst M Lib/test/test_ssl.py M Modules/_ssl.c diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 08824feeb395..78d44a23a83b 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -823,6 +823,13 @@ Constants .. versionadded:: 3.12 +.. data:: OP_LEGACY_SERVER_CONNECT + + Allow legacy insecure renegotiation between OpenSSL and unpatched servers + only. + + .. versionadded:: 3.12 + .. data:: HAS_ALPN Whether the OpenSSL library has built-in support for the *Application-Layer diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e926fc5e88e5..d4eb2d2e81fe 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1461,6 +1461,8 @@ def _assert_context_options(self, ctx): if OP_CIPHER_SERVER_PREFERENCE != 0: self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, OP_CIPHER_SERVER_PREFERENCE) + self.assertEqual(ctx.options & ssl.OP_LEGACY_SERVER_CONNECT, + 0 if IS_OPENSSL_3_0_0 else ssl.OP_LEGACY_SERVER_CONNECT) def test_create_default_context(self): ctx = ssl.create_default_context() @@ -3815,6 +3817,20 @@ def test_compression_disabled(self): sni_name=hostname) self.assertIs(stats['compression'], None) + def test_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_no_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options &= ~ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") def test_dh_params(self): # Check we can get a connection with ephemeral Diffie-Hellman diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst new file mode 100644 index 000000000000..5c8164863b81 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst @@ -0,0 +1 @@ +Add :data:`ssl.OP_LEGACY_SERVER_CONNECT` diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 591eb91dd0f3..8f03a846aed0 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -5845,6 +5845,8 @@ sslmodule_init_constants(PyObject *m) SSL_OP_CIPHER_SERVER_PREFERENCE); PyModule_AddIntConstant(m, "OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE); PyModule_AddIntConstant(m, "OP_NO_TICKET", SSL_OP_NO_TICKET); + PyModule_AddIntConstant(m, "OP_LEGACY_SERVER_CONNECT", + SSL_OP_LEGACY_SERVER_CONNECT); #ifdef SSL_OP_SINGLE_ECDH_USE PyModule_AddIntConstant(m, "OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE); #endif From webhook-mailer at python.org Tue Dec 20 05:10:37 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:10:37 -0000 Subject: [Python-checkins] gh-88211: Change lower-case and upper-case to match recommendations in imaplib docs (#99625) Message-ID: <mailman.2974.1671531038.3313.python-checkins@python.org> https://github.com/python/cpython/commit/39dfbb2d5dc47635f332bc13ca667293de6989ab commit: 39dfbb2d5dc47635f332bc13ca667293de6989ab branch: main author: Brad Wolfe <brad.wolfe at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T15:40:31+05:30 summary: gh-88211: Change lower-case and upper-case to match recommendations in imaplib docs (#99625) files: M Doc/library/imaplib.rst diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 0c10e7afee40..8c28fce99ff9 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -187,7 +187,7 @@ IMAP4 Objects ------------- All IMAP4rev1 commands are represented by methods of the same name, either -upper-case or lower-case. +uppercase or lowercase. All arguments to commands are converted to strings, except for ``AUTHENTICATE``, and the last argument to ``APPEND`` which is passed as an IMAP4 literal. If From webhook-mailer at python.org Tue Dec 20 05:20:16 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:20:16 -0000 Subject: [Python-checkins] [3.11] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100352) Message-ID: <mailman.2975.1671531616.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ba8e30c56ba4833f572318e1cf4108d9f206d1a0 commit: ba8e30c56ba4833f572318e1cf4108d9f206d1a0 branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T15:49:53+05:30 summary: [3.11] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100352) (cherry picked from commit 8dbe08eb7c807f484fe9870f5b7f5ae2881fd966) Fix double-free bug mentioned at GH-99240, by moving memory clean up out of "exit" label. files: A Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 8643f3fe78ae..991e82d4f966 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -1740,29 +1740,18 @@ test_str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } return_value = test_str_converter_encoding_impl(module, a, b, c, d, d_length, e, e_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + /* Post parse cleanup for d */ + PyMem_FREE(d); + /* Post parse cleanup for e */ + PyMem_FREE(e); exit: - /* Cleanup for a */ - if (a) { - PyMem_FREE(a); - } - /* Cleanup for b */ - if (b) { - PyMem_FREE(b); - } - /* Cleanup for c */ - if (c) { - PyMem_FREE(c); - } - /* Cleanup for d */ - if (d) { - PyMem_FREE(d); - } - /* Cleanup for e */ - if (e) { - PyMem_FREE(e); - } - return return_value; } @@ -1770,7 +1759,7 @@ static PyObject * test_str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, char *d, Py_ssize_t d_length, char *e, Py_ssize_t e_length) -/*[clinic end generated code: output=8acb886a3843f3bc input=eb4c38e1f898f402]*/ +/*[clinic end generated code: output=999c1deecfa15b0a input=eb4c38e1f898f402]*/ /*[clinic input] diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index af86d685866c..3f6cc6069669 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1045,6 +1045,17 @@ def test_str_converter(self): self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + def test_str_converter_encoding(self): + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding(1) + self.assertEqual(ac_tester.str_converter_encoding('a', 'b', 'c'), ('a', 'b', 'c')) + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c')])), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c'), 0, ord('c')])), + ('a', 'b', 'c\x00c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', b'c\x00c'), ('a', 'b', 'c\x00c')) + def test_py_buffer_converter(self): with self.assertRaises(TypeError): ac_tester.py_buffer_converter('a', 'b') @@ -1225,6 +1236,10 @@ def test_gh_99233_refcount(self): arg_refcount_after = sys.getrefcount(arg) self.assertEqual(arg_refcount_origin, arg_refcount_after) + def test_gh_99240_double_free(self): + expected_error = r'gh_99240_double_free argument 2 must be encoded string without null bytes, not str' + with self.assertRaisesRegex(TypeError, expected_error): + ac_tester.gh_99240_double_free('a', '\0b') if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst new file mode 100644 index 000000000000..0a4db052755f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst @@ -0,0 +1,2 @@ +Fix double-free bug in Argument Clinic ``str_converter`` by +extracting memory clean up to a new ``post_parsing`` section. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index a23ece2ae035..56eddfd6fdbf 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -551,6 +551,64 @@ str_converter_impl(PyObject *module, const char *a, const char *b, } +/*[clinic input] +str_converter_encoding + + a: str(encoding="idna") + b: str(encoding="idna", accept={bytes, bytearray, str}) + c: str(encoding="idna", accept={bytes, bytearray, str}, zeroes=True) + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_t c_length) +/*[clinic end generated code: output=af68766049248a1c input=0c5cf5159d0e870d]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + static PyObject * bytes_from_buffer(Py_buffer *buf) { @@ -927,6 +985,25 @@ gh_99233_refcount_impl(PyObject *module, PyObject *args) } +/*[clinic input] +gh_99240_double_free + + a: str(encoding="idna") + b: str(encoding="idna") + / + +Proof-of-concept of GH-99240 double-free bug. + +[clinic start generated code]*/ + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b) +/*[clinic end generated code: output=586dc714992fe2ed input=23db44aa91870fc7]*/ +{ + Py_RETURN_NONE; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -951,6 +1028,7 @@ static PyMethodDef tester_methods[] = { DOUBLE_CONVERTER_METHODDEF PY_COMPLEX_CONVERTER_METHODDEF STR_CONVERTER_METHODDEF + STR_CONVERTER_ENCODING_METHODDEF PY_BUFFER_CONVERTER_METHODDEF KEYWORDS_METHODDEF KEYWORDS_KWONLY_METHODDEF @@ -970,6 +1048,7 @@ static PyMethodDef tester_methods[] = { KEYWORD_ONLY_PARAMETER_METHODDEF VARARG_AND_POSONLY_METHODDEF GH_99233_REFCOUNT_METHODDEF + GH_99240_DOUBLE_FREE_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index c108d9d626a1..222ea783486e 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1159,6 +1159,43 @@ str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(str_converter_encoding__doc__, +"str_converter_encoding($module, a, b, c, /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_ENCODING_METHODDEF \ + {"str_converter_encoding", _PyCFunction_CAST(str_converter_encoding), METH_FASTCALL, str_converter_encoding__doc__}, + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_t c_length); + +static PyObject * +str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + char *c = NULL; + Py_ssize_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "esetet#:str_converter_encoding", + "idna", &a, "idna", &b, "idna", &c, &c_length)) { + goto exit; + } + return_value = str_converter_encoding_impl(module, a, b, c, c_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + +exit: + return return_value; +} + PyDoc_STRVAR(py_buffer_converter__doc__, "py_buffer_converter($module, a, b, /)\n" "--\n" @@ -1979,4 +2016,37 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_XDECREF(__clinic_args); return return_value; } -/*[clinic end generated code: output=16848e04bd5ddf7d input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gh_99240_double_free__doc__, +"gh_99240_double_free($module, a, b, /)\n" +"--\n" +"\n" +"Proof-of-concept of GH-99240 double-free bug."); + +#define GH_99240_DOUBLE_FREE_METHODDEF \ + {"gh_99240_double_free", _PyCFunction_CAST(gh_99240_double_free), METH_FASTCALL, gh_99240_double_free__doc__}, + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b); + +static PyObject * +gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + + if (!_PyArg_ParseStack(args, nargs, "eses:gh_99240_double_free", + "idna", &a, "idna", &b)) { + goto exit; + } + return_value = gh_99240_double_free_impl(module, a, b); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + +exit: + return return_value; +} +/*[clinic end generated code: output=4147b953eebc3c82 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index acd463bdd894..c49805ce1a7e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -349,6 +349,12 @@ def __init__(self): # "goto exit" if there are any. self.return_conversion = [] + # The C statements required to do some operations + # after the end of parsing but before cleaning up. + # These operations may be, for example, memory deallocations which + # can only be done without any error happening during argument parsing. + self.post_parsing = [] + # The C statements required to clean up after the impl call. self.cleanup = [] @@ -761,6 +767,7 @@ def parser_body(prototype, *fields, declarations=''): {modifications} {return_value} = {c_basename}_impl({impl_arguments}); {return_conversion} + {post_parsing} {exit_label} {cleanup} @@ -1405,6 +1412,7 @@ def render_function(self, clinic, f): template_dict['impl_parameters'] = ", ".join(data.impl_parameters) template_dict['impl_arguments'] = ", ".join(data.impl_arguments) template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) template_dict['cleanup'] = format_escape("".join(data.cleanup)) template_dict['return_value'] = data.return_value @@ -1429,6 +1437,7 @@ def render_function(self, clinic, f): return_conversion=template_dict['return_conversion'], initializers=template_dict['initializers'], modifications=template_dict['modifications'], + post_parsing=template_dict['post_parsing'], cleanup=template_dict['cleanup'], ) @@ -2660,6 +2669,10 @@ def _render_non_self(self, parameter, data): # parse_arguments self.parse_argument(data.parse_arguments) + # post_parsing + if post_parsing := self.post_parsing(): + data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') + # cleanup cleanup = self.cleanup() if cleanup: @@ -2755,6 +2768,14 @@ def modify(self): """ return "" + def post_parsing(self): + """ + The C statements required to do some operations after the end of parsing but before cleaning up. + Return a string containing this code indented at column 0. + If no operation is necessary, return an empty string. + """ + return "" + def cleanup(self): """ The C statements required to clean up after this variable. @@ -3351,10 +3372,10 @@ def converter_init(self, *, accept={str}, encoding=None, zeroes=False): if NoneType in accept and self.c_default == "Py_None": self.c_default = "NULL" - def cleanup(self): + def post_parsing(self): if self.encoding: name = self.name - return "".join(["if (", name, ") {\n PyMem_FREE(", name, ");\n}\n"]) + return f"PyMem_FREE({name});\n" def parse_arg(self, argname, displayname): if self.format_unit == 's': From webhook-mailer at python.org Tue Dec 20 05:20:48 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:20:48 -0000 Subject: [Python-checkins] [3.10] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100353) Message-ID: <mailman.2976.1671531649.3313.python-checkins@python.org> https://github.com/python/cpython/commit/53063b7ffa8f8edf13dd7c81f89f034a3b3b71b6 commit: 53063b7ffa8f8edf13dd7c81f89f034a3b3b71b6 branch: 3.10 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T15:50:42+05:30 summary: [3.10] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100353) (cherry picked from commit 8dbe08eb7c807f484fe9870f5b7f5ae2881fd966) Fix double-free bug mentioned at GH-99240, by moving memory clean up out of "exit" label. files: A Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 3a3fbd174d86..af111dbee2b8 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -1740,29 +1740,18 @@ test_str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } return_value = test_str_converter_encoding_impl(module, a, b, c, d, d_length, e, e_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + /* Post parse cleanup for d */ + PyMem_FREE(d); + /* Post parse cleanup for e */ + PyMem_FREE(e); exit: - /* Cleanup for a */ - if (a) { - PyMem_FREE(a); - } - /* Cleanup for b */ - if (b) { - PyMem_FREE(b); - } - /* Cleanup for c */ - if (c) { - PyMem_FREE(c); - } - /* Cleanup for d */ - if (d) { - PyMem_FREE(d); - } - /* Cleanup for e */ - if (e) { - PyMem_FREE(e); - } - return return_value; } @@ -1770,7 +1759,7 @@ static PyObject * test_str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, char *d, Py_ssize_clean_t d_length, char *e, Py_ssize_clean_t e_length) -/*[clinic end generated code: output=f579dd9e795a364e input=eb4c38e1f898f402]*/ +/*[clinic end generated code: output=2d7e8b6203db31aa input=eb4c38e1f898f402]*/ /*[clinic input] diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 86c4f94ac028..feebb6add09a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1045,6 +1045,17 @@ def test_str_converter(self): self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + def test_str_converter_encoding(self): + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding(1) + self.assertEqual(ac_tester.str_converter_encoding('a', 'b', 'c'), ('a', 'b', 'c')) + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c')])), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c'), 0, ord('c')])), + ('a', 'b', 'c\x00c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', b'c\x00c'), ('a', 'b', 'c\x00c')) + def test_py_buffer_converter(self): with self.assertRaises(TypeError): ac_tester.py_buffer_converter('a', 'b') @@ -1211,6 +1222,11 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + def test_gh_99240_double_free(self): + expected_error = r'gh_99240_double_free argument 2 must be encoded string without null bytes, not str' + with self.assertRaisesRegex(TypeError, expected_error): + ac_tester.gh_99240_double_free('a', '\0b') + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst new file mode 100644 index 000000000000..0a4db052755f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst @@ -0,0 +1,2 @@ +Fix double-free bug in Argument Clinic ``str_converter`` by +extracting memory clean up to a new ``post_parsing`` section. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 63427ae6373e..079a40130b1d 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -551,6 +551,64 @@ str_converter_impl(PyObject *module, const char *a, const char *b, } +/*[clinic input] +str_converter_encoding + + a: str(encoding="idna") + b: str(encoding="idna", accept={bytes, bytearray, str}) + c: str(encoding="idna", accept={bytes, bytearray, str}, zeroes=True) + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_clean_t c_length) +/*[clinic end generated code: output=ebef1a3f9b730a24 input=0c5cf5159d0e870d]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + static PyObject * bytes_from_buffer(Py_buffer *buf) { @@ -892,6 +950,25 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) } +/*[clinic input] +gh_99240_double_free + + a: str(encoding="idna") + b: str(encoding="idna") + / + +Proof-of-concept of GH-99240 double-free bug. + +[clinic start generated code]*/ + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b) +/*[clinic end generated code: output=586dc714992fe2ed input=23db44aa91870fc7]*/ +{ + Py_RETURN_NONE; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -916,6 +993,7 @@ static PyMethodDef tester_methods[] = { DOUBLE_CONVERTER_METHODDEF PY_COMPLEX_CONVERTER_METHODDEF STR_CONVERTER_METHODDEF + STR_CONVERTER_ENCODING_METHODDEF PY_BUFFER_CONVERTER_METHODDEF KEYWORDS_METHODDEF KEYWORDS_KWONLY_METHODDEF @@ -933,6 +1011,7 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF + GH_99240_DOUBLE_FREE_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index a8273a5e6e2f..ed79fac50869 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1159,6 +1159,43 @@ str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(str_converter_encoding__doc__, +"str_converter_encoding($module, a, b, c, /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_ENCODING_METHODDEF \ + {"str_converter_encoding", (PyCFunction)(void(*)(void))str_converter_encoding, METH_FASTCALL, str_converter_encoding__doc__}, + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_clean_t c_length); + +static PyObject * +str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + char *c = NULL; + Py_ssize_clean_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "esetet#:str_converter_encoding", + "idna", &a, "idna", &b, "idna", &c, &c_length)) { + goto exit; + } + return_value = str_converter_encoding_impl(module, a, b, c, c_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + +exit: + return return_value; +} + PyDoc_STRVAR(py_buffer_converter__doc__, "py_buffer_converter($module, a, b, /)\n" "--\n" @@ -1914,4 +1951,37 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=2d5ac6b41daadf6f input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gh_99240_double_free__doc__, +"gh_99240_double_free($module, a, b, /)\n" +"--\n" +"\n" +"Proof-of-concept of GH-99240 double-free bug."); + +#define GH_99240_DOUBLE_FREE_METHODDEF \ + {"gh_99240_double_free", (PyCFunction)(void(*)(void))gh_99240_double_free, METH_FASTCALL, gh_99240_double_free__doc__}, + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b); + +static PyObject * +gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + + if (!_PyArg_ParseStack(args, nargs, "eses:gh_99240_double_free", + "idna", &a, "idna", &b)) { + goto exit; + } + return_value = gh_99240_double_free_impl(module, a, b); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + +exit: + return return_value; +} +/*[clinic end generated code: output=ecc55cbeac4adae6 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1a59fa57ac41..b0d1717596f6 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -346,6 +346,12 @@ def __init__(self): # "goto exit" if there are any. self.return_conversion = [] + # The C statements required to do some operations + # after the end of parsing but before cleaning up. + # These operations may be, for example, memory deallocations which + # can only be done without any error happening during argument parsing. + self.post_parsing = [] + # The C statements required to clean up after the impl call. self.cleanup = [] @@ -751,6 +757,7 @@ def parser_body(prototype, *fields, declarations=''): {modifications} {return_value} = {c_basename}_impl({impl_arguments}); {return_conversion} + {post_parsing} {exit_label} {cleanup} @@ -1343,6 +1350,7 @@ def render_function(self, clinic, f): template_dict['impl_parameters'] = ", ".join(data.impl_parameters) template_dict['impl_arguments'] = ", ".join(data.impl_arguments) template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) template_dict['cleanup'] = format_escape("".join(data.cleanup)) template_dict['return_value'] = data.return_value @@ -1367,6 +1375,7 @@ def render_function(self, clinic, f): return_conversion=template_dict['return_conversion'], initializers=template_dict['initializers'], modifications=template_dict['modifications'], + post_parsing=template_dict['post_parsing'], cleanup=template_dict['cleanup'], ) @@ -2595,6 +2604,10 @@ def _render_non_self(self, parameter, data): # parse_arguments self.parse_argument(data.parse_arguments) + # post_parsing + if post_parsing := self.post_parsing(): + data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') + # cleanup cleanup = self.cleanup() if cleanup: @@ -2686,6 +2699,14 @@ def modify(self): """ return "" + def post_parsing(self): + """ + The C statements required to do some operations after the end of parsing but before cleaning up. + Return a string containing this code indented at column 0. + If no operation is necessary, return an empty string. + """ + return "" + def cleanup(self): """ The C statements required to clean up after this variable. @@ -3278,10 +3299,10 @@ def converter_init(self, *, accept={str}, encoding=None, zeroes=False): if NoneType in accept and self.c_default == "Py_None": self.c_default = "NULL" - def cleanup(self): + def post_parsing(self): if self.encoding: name = self.name - return "".join(["if (", name, ") {\n PyMem_FREE(", name, ");\n}\n"]) + return f"PyMem_FREE({name});\n" def parse_arg(self, argname, displayname): if self.format_unit == 's': From webhook-mailer at python.org Tue Dec 20 05:41:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:41:23 -0000 Subject: [Python-checkins] gh-100348: Fix ref cycle in `asyncio._SelectorSocketTransport` with `_read_ready_cb` (#100349) Message-ID: <mailman.2977.1671532884.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a6331b605e8044a205a113e1db87d2b0a53d0222 commit: a6331b605e8044a205a113e1db87d2b0a53d0222 branch: main author: Richard Kojedzinszky <rkojedzinszky at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T16:10:56+05:30 summary: gh-100348: Fix ref cycle in `asyncio._SelectorSocketTransport` with `_read_ready_cb` (#100349) files: A Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst M Lib/asyncio/selector_events.py diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 3d30006198f6..74f289f0e6f8 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1133,6 +1133,10 @@ def _make_empty_waiter(self): def _reset_empty_waiter(self): self._empty_waiter = None + def close(self): + self._read_ready_cb = None + super().close() + class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): diff --git a/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst b/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst new file mode 100644 index 000000000000..b5d4f7ca998c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst @@ -0,0 +1,2 @@ +Fix ref cycle in :class:`!asyncio._SelectorSocketTransport` by removing ``_read_ready_cb`` in ``close``. + From webhook-mailer at python.org Tue Dec 20 05:55:10 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Tue, 20 Dec 2022 10:55:10 -0000 Subject: [Python-checkins] gh-99925: Fix inconsistency in `json.dumps()` error messages (GH-99926) Message-ID: <mailman.2978.1671533711.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d98ca8172c39326bb200308a5191ceeb4a262d53 commit: d98ca8172c39326bb200308a5191ceeb4a262d53 branch: main author: Franti?ek Nesveda <fnesveda at users.noreply.github.com> committer: serhiy-storchaka <storchaka at gmail.com> date: 2022-12-20T12:54:56+02:00 summary: gh-99925: Fix inconsistency in `json.dumps()` error messages (GH-99926) files: A Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst M Lib/test/test_json/test_float.py M Modules/_json.c diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index d0c7214334d6..61540a3a02c2 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -26,7 +26,8 @@ def test_allow_nan(self): res = self.loads(out) self.assertEqual(len(res), 1) self.assertNotEqual(res[0], res[0]) - self.assertRaises(ValueError, self.dumps, [val], allow_nan=False) + msg = f'Out of range float values are not JSON compliant: {val}' + self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False) class TestPyFloat(TestFloat, PyTest): pass diff --git a/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst new file mode 100644 index 000000000000..660635a03963 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst @@ -0,0 +1,4 @@ +Unify error messages in JSON serialization between +``json.dumps(float('nan'), allow_nan=False)`` and ``json.dumps(float('nan'), +allow_nan=False, indent=<SOMETHING>)``. Now both include the representation +of the value that could not be serialized. diff --git a/Modules/_json.c b/Modules/_json.c index 6879ad3d0722..fa8e2a936d2c 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1319,9 +1319,10 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) double i = PyFloat_AS_DOUBLE(obj); if (!Py_IS_FINITE(i)) { if (!s->allow_nan) { - PyErr_SetString( + PyErr_Format( PyExc_ValueError, - "Out of range float values are not JSON compliant" + "Out of range float values are not JSON compliant: %R", + obj ); return NULL; } From webhook-mailer at python.org Tue Dec 20 06:36:26 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 20 Dec 2022 11:36:26 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (#99246) Message-ID: <mailman.2979.1671536188.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cb60b6131bc2bb11c48a15f808914d8b242b9fc5 commit: cb60b6131bc2bb11c48a15f808914d8b242b9fc5 branch: main author: Pablo Galindo Salgado <Pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-20T11:35:48Z summary: Clarify that every thread has its own default context in contextvars (#99246) files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 08a7c7d74eab..0ac2f3d85749 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 06:45:01 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 11:45:01 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (GH-99246) Message-ID: <mailman.2980.1671536702.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1332fdabbab75bc9e4bced064dc4daab2d7acb47 commit: 1332fdabbab75bc9e4bced064dc4daab2d7acb47 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T03:44:55-08:00 summary: Clarify that every thread has its own default context in contextvars (GH-99246) (cherry picked from commit cb60b6131bc2bb11c48a15f808914d8b242b9fc5) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 08a7c7d74eab..0ac2f3d85749 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 06:45:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 11:45:55 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (GH-99246) Message-ID: <mailman.2981.1671536756.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a757c1b7d3dfec875be15f8bd83777142d75b376 commit: a757c1b7d3dfec875be15f8bd83777142d75b376 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T03:45:48-08:00 summary: Clarify that every thread has its own default context in contextvars (GH-99246) (cherry picked from commit cb60b6131bc2bb11c48a15f808914d8b242b9fc5) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 08a7c7d74eab..0ac2f3d85749 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 06:57:14 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 11:57:14 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (GH-99246) Message-ID: <mailman.2982.1671537435.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e8f61ede93ecbda3b2055a2e1d8c0b43b2049737 commit: e8f61ede93ecbda3b2055a2e1d8c0b43b2049737 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T03:57:08-08:00 summary: Clarify that every thread has its own default context in contextvars (GH-99246) (cherry picked from commit cb60b6131bc2bb11c48a15f808914d8b242b9fc5) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index be1dd0c9eb57..3580b3537348 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 11:51:32 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 20 Dec 2022 16:51:32 -0000 Subject: [Python-checkins] gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) Message-ID: <mailman.2983.1671555094.3313.python-checkins@python.org> https://github.com/python/cpython/commit/44892d45b038f919b0378590a776580a9d73b291 commit: 44892d45b038f919b0378590a776580a9d73b291 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: ambv <lukasz at langa.pl> date: 2022-12-20T17:51:26+01:00 summary: gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst M Lib/http/cookiejar.py M Lib/test/test_http_cookiejar.py diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index b0161a86fdbb..e3df007033b3 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1890,7 +1890,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. @@ -2081,7 +2084,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index f8291c2aa32c..97e9c82cde9e 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -397,6 +397,32 @@ def test_mozilla_filepermissions(self): finally: os_helper.unlink(filename) + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_cookie_files_are_truncated(self): + filename = os_helper.TESTFN + for cookiejar_class in (LWPCookieJar, MozillaCookieJar): + c = cookiejar_class(filename) + + req = urllib.request.Request("http://www.acme.com/") + headers = ["Set-Cookie: pll_lang=en; Max-Age=31536000; path=/"] + res = FakeResponse(headers, "http://www.acme.com/") + c.extract_cookies(res, req) + self.assertEqual(len(c), 1) + + try: + # Save the first version with contents: + c.save() + # Now, clear cookies and re-save: + c.clear() + c.save() + # Check that file was truncated: + c.load() + finally: + os_helper.unlink(filename) + + self.assertEqual(len(c), 0) + def test_bad_magic(self): # OSErrors (eg. file doesn't exist) are allowed to propagate filename = os_helper.TESTFN diff --git a/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst new file mode 100644 index 000000000000..9cbeb64b5625 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst @@ -0,0 +1,2 @@ +Fix ``.save()`` method for ``LWPCookieJar`` and ``MozillaCookieJar``: saved +file was not truncated on repeated save. From webhook-mailer at python.org Tue Dec 20 12:22:55 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 20 Dec 2022 17:22:55 -0000 Subject: [Python-checkins] [3.11] gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) (GH-100377) Message-ID: <mailman.2984.1671556976.3313.python-checkins@python.org> https://github.com/python/cpython/commit/73d2b15bd278e84e89de83dbd2ee64eac2e63cda commit: 73d2b15bd278e84e89de83dbd2ee64eac2e63cda branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv <lukasz at langa.pl> date: 2022-12-20T18:22:39+01:00 summary: [3.11] gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) (GH-100377) (cherry picked from commit 44892d45b038f919b0378590a776580a9d73b291) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst M Lib/http/cookiejar.py M Lib/test/test_http_cookiejar.py diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 65c45e2b17df..e622fc36cbfc 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1890,7 +1890,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. @@ -2086,7 +2089,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index f0d205a60a4c..b90612867fef 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -398,6 +398,32 @@ def test_mozilla_filepermissions(self): finally: os_helper.unlink(filename) + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_cookie_files_are_truncated(self): + filename = os_helper.TESTFN + for cookiejar_class in (LWPCookieJar, MozillaCookieJar): + c = cookiejar_class(filename) + + req = urllib.request.Request("http://www.acme.com/") + headers = ["Set-Cookie: pll_lang=en; Max-Age=31536000; path=/"] + res = FakeResponse(headers, "http://www.acme.com/") + c.extract_cookies(res, req) + self.assertEqual(len(c), 1) + + try: + # Save the first version with contents: + c.save() + # Now, clear cookies and re-save: + c.clear() + c.save() + # Check that file was truncated: + c.load() + finally: + os_helper.unlink(filename) + + self.assertEqual(len(c), 0) + def test_bad_magic(self): # OSErrors (eg. file doesn't exist) are allowed to propagate filename = os_helper.TESTFN diff --git a/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst new file mode 100644 index 000000000000..9cbeb64b5625 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst @@ -0,0 +1,2 @@ +Fix ``.save()`` method for ``LWPCookieJar`` and ``MozillaCookieJar``: saved +file was not truncated on repeated save. From webhook-mailer at python.org Tue Dec 20 15:46:26 2022 From: webhook-mailer at python.org (sweeneyde) Date: Tue, 20 Dec 2022 20:46:26 -0000 Subject: [Python-checkins] gh-100188: Reduce misses in BINARY_SUBSCR_(LIST/TUPLE)_INT (#100189) Message-ID: <mailman.2985.1671569188.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c18d83118881333b9a0afd0add83afb2ba7300f7 commit: c18d83118881333b9a0afd0add83afb2ba7300f7 branch: main author: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com> committer: sweeneyde <36520290+sweeneyde at users.noreply.github.com> date: 2022-12-20T15:46:16-05:00 summary: gh-100188: Reduce misses in BINARY_SUBSCR_(LIST/TUPLE)_INT (#100189) Don't specialize if the index is negative. files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst M Python/specialize.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst new file mode 100644 index 000000000000..ec62fbd582fb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst @@ -0,0 +1,3 @@ +The ``BINARY_SUBSCR_LIST_INT`` and ``BINARY_SUBSCR_TUPLE_INT`` +instructions are no longer used for negative integers because +those instructions always miss when encountering negative integers. diff --git a/Python/specialize.c b/Python/specialize.c index a1666ccc9159..c6c502716478 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1302,8 +1302,12 @@ _Py_Specialize_BinarySubscr( PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opcode(instr, BINARY_SUBSCR_LIST_INT); - goto success; + if (Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) { + _py_set_opcode(instr, BINARY_SUBSCR_LIST_INT); + goto success; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; } SPECIALIZATION_FAIL(BINARY_SUBSCR, PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER); @@ -1311,8 +1315,12 @@ _Py_Specialize_BinarySubscr( } if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opcode(instr, BINARY_SUBSCR_TUPLE_INT); - goto success; + if (Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) { + _py_set_opcode(instr, BINARY_SUBSCR_TUPLE_INT); + goto success; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; } SPECIALIZATION_FAIL(BINARY_SUBSCR, PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER); From webhook-mailer at python.org Tue Dec 20 21:02:20 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 02:02:20 -0000 Subject: [Python-checkins] gh-99991: improve docs on str.encode and bytes.decode (#100198) Message-ID: <mailman.2986.1671588141.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a2bb3b7f9d8d15c81b724726454d68357fb31d1c commit: a2bb3b7f9d8d15c81b724726454d68357fb31d1c branch: main author: Bisola Olasehinde <horlasehinde at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T18:02:14-08:00 summary: gh-99991: improve docs on str.encode and bytes.decode (#100198) Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 5a6b7d83edf7..624284f7092f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1624,25 +1624,28 @@ expression support in the :mod:`re` module). .. method:: str.encode(encoding="utf-8", errors="strict") - Return an encoded version of the string as a bytes object. Default encoding - is ``'utf-8'``. *errors* may be given to set a different error handling scheme. - The default for *errors* is ``'strict'``, meaning that encoding errors raise - a :exc:`UnicodeError`. Other possible - values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, - ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - By default, the *errors* argument is not checked for best performances, but - only used at the first encoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check - *errors*. + Return the string encoded to :class:`bytes`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how encoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, + ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any + other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. + + For performance reasons, the value of *errors* is not checked for validity + unless an encoding error actually occurs, + :ref:`devmode` is enabled + or a :ref:`debug build <debug-build>` is used. .. versionchanged:: 3.1 - Support for keyword arguments added. + Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. @@ -2759,29 +2762,32 @@ arbitrary binary data. .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") - Return a string decoded from the given bytes. Default encoding is - ``'utf-8'``. *errors* may be given to set a different - error handling scheme. The default for *errors* is ``'strict'``, meaning - that encoding errors raise a :exc:`UnicodeError`. Other possible values are - ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. + Return the bytes decoded to a :class:`str`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how decoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, ``'replace'``, + and any other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. - By default, the *errors* argument is not checked for best performances, but - only used at the first decoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check *errors*. + For performance reasons, the value of *errors* is not checked for validity + unless a decoding error actually occurs, + :ref:`devmode` is enabled or a :ref:`debug build <debug-build>` is used. .. note:: Passing the *encoding* argument to :class:`str` allows decoding any :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. + :class:`!bytes` or :class:`!bytearray` object. .. versionchanged:: 3.1 Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. From webhook-mailer at python.org Tue Dec 20 21:11:26 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 02:11:26 -0000 Subject: [Python-checkins] gh-99991: improve docs on str.encode and bytes.decode (GH-100198) Message-ID: <mailman.2987.1671588687.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b3d39c75990722aadd15586478d5dbe6882a2b76 commit: b3d39c75990722aadd15586478d5dbe6882a2b76 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T18:11:20-08:00 summary: gh-99991: improve docs on str.encode and bytes.decode (GH-100198) (cherry picked from commit a2bb3b7f9d8d15c81b724726454d68357fb31d1c) Co-authored-by: Bisola Olasehinde <horlasehinde at gmail.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a6a7eea9f071..d71d6936c568 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1585,25 +1585,28 @@ expression support in the :mod:`re` module). .. method:: str.encode(encoding="utf-8", errors="strict") - Return an encoded version of the string as a bytes object. Default encoding - is ``'utf-8'``. *errors* may be given to set a different error handling scheme. - The default for *errors* is ``'strict'``, meaning that encoding errors raise - a :exc:`UnicodeError`. Other possible - values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, - ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - By default, the *errors* argument is not checked for best performances, but - only used at the first encoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check - *errors*. + Return the string encoded to :class:`bytes`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how encoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, + ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any + other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. + + For performance reasons, the value of *errors* is not checked for validity + unless an encoding error actually occurs, + :ref:`devmode` is enabled + or a :ref:`debug build <debug-build>` is used. .. versionchanged:: 3.1 - Support for keyword arguments added. + Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. @@ -2715,29 +2718,32 @@ arbitrary binary data. .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") - Return a string decoded from the given bytes. Default encoding is - ``'utf-8'``. *errors* may be given to set a different - error handling scheme. The default for *errors* is ``'strict'``, meaning - that encoding errors raise a :exc:`UnicodeError`. Other possible values are - ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. + Return the bytes decoded to a :class:`str`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how decoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, ``'replace'``, + and any other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. - By default, the *errors* argument is not checked for best performances, but - only used at the first decoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check *errors*. + For performance reasons, the value of *errors* is not checked for validity + unless a decoding error actually occurs, + :ref:`devmode` is enabled or a :ref:`debug build <debug-build>` is used. .. note:: Passing the *encoding* argument to :class:`str` allows decoding any :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. + :class:`!bytes` or :class:`!bytearray` object. .. versionchanged:: 3.1 Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. From webhook-mailer at python.org Tue Dec 20 21:12:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 02:12:59 -0000 Subject: [Python-checkins] gh-99991: improve docs on str.encode and bytes.decode (GH-100198) Message-ID: <mailman.2988.1671588779.3313.python-checkins@python.org> https://github.com/python/cpython/commit/23fa1667b364a73ed29074f257d2bcf41bd35019 commit: 23fa1667b364a73ed29074f257d2bcf41bd35019 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T18:12:53-08:00 summary: gh-99991: improve docs on str.encode and bytes.decode (GH-100198) (cherry picked from commit a2bb3b7f9d8d15c81b724726454d68357fb31d1c) Co-authored-by: Bisola Olasehinde <horlasehinde at gmail.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 426eec620566..a1514f611d4f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1623,25 +1623,28 @@ expression support in the :mod:`re` module). .. method:: str.encode(encoding="utf-8", errors="strict") - Return an encoded version of the string as a bytes object. Default encoding - is ``'utf-8'``. *errors* may be given to set a different error handling scheme. - The default for *errors* is ``'strict'``, meaning that encoding errors raise - a :exc:`UnicodeError`. Other possible - values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, - ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - By default, the *errors* argument is not checked for best performances, but - only used at the first encoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check - *errors*. + Return the string encoded to :class:`bytes`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how encoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, + ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any + other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. + + For performance reasons, the value of *errors* is not checked for validity + unless an encoding error actually occurs, + :ref:`devmode` is enabled + or a :ref:`debug build <debug-build>` is used. .. versionchanged:: 3.1 - Support for keyword arguments added. + Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. @@ -2755,29 +2758,32 @@ arbitrary binary data. .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") - Return a string decoded from the given bytes. Default encoding is - ``'utf-8'``. *errors* may be given to set a different - error handling scheme. The default for *errors* is ``'strict'``, meaning - that encoding errors raise a :exc:`UnicodeError`. Other possible values are - ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. + Return the bytes decoded to a :class:`str`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how decoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, ``'replace'``, + and any other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. - By default, the *errors* argument is not checked for best performances, but - only used at the first decoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check *errors*. + For performance reasons, the value of *errors* is not checked for validity + unless a decoding error actually occurs, + :ref:`devmode` is enabled or a :ref:`debug build <debug-build>` is used. .. note:: Passing the *encoding* argument to :class:`str` allows decoding any :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. + :class:`!bytes` or :class:`!bytearray` object. .. versionchanged:: 3.1 Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. From webhook-mailer at python.org Tue Dec 20 22:27:07 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 03:27:07 -0000 Subject: [Python-checkins] gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (#91499) Message-ID: <mailman.2989.1671593228.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c615286e8576f2555d4380f38a966c300805b1a5 commit: c615286e8576f2555d4380f38a966c300805b1a5 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T19:27:02-08:00 summary: gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (#91499) Co-authored-by: Pieter Eendebak <P.T.eendebak at tudelft.nl> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/weakref.rst diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 73e7b21ae405..1406b663c6a8 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -172,6 +172,30 @@ See :ref:`__slots__ documentation <slots>` for details. application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. + Note that when a key with equal value to an existing key (but not equal identity) + is inserted into the dictionary, it replaces the value but does not replace the + existing key. Due to this, when the reference to the original key is deleted, it + also deletes the entry in the dictionary:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> d[k2] = 2 # d = {k1: 2} + >>> del k1 # d = {} + + A workaround would be to remove the key prior to reassignment:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> del d[k1] + >>> d[k2] = 2 # d = {k2: 2} + >>> del k1 # d = {k2: 2} + .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. From webhook-mailer at python.org Tue Dec 20 22:29:08 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 03:29:08 -0000 Subject: [Python-checkins] gh-85267: Improvements to inspect.signature __text_signature__ handling (#98796) Message-ID: <mailman.2990.1671593349.3313.python-checkins@python.org> https://github.com/python/cpython/commit/79311cbfe718f17c89bab67d7f89da3931bfa2ac commit: 79311cbfe718f17c89bab67d7f89da3931bfa2ac branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T19:29:02-08:00 summary: gh-85267: Improvements to inspect.signature __text_signature__ handling (#98796) This makes a couple related changes to inspect.signature's behaviour when parsing a signature from `__text_signature__`. First, `inspect.signature` is documented as only raising ValueError or TypeError. However, in some cases, we could raise RuntimeError. This PR changes that, thereby fixing #83685. (Note that the new ValueErrors in RewriteSymbolics are caught and then reraised with a message) Second, `inspect.signature` could randomly drop parameters that it didn't understand (corresponding to `return None` in the `p` function). This is the core issue in #85267. I think this is very surprising behaviour and it seems better to fail outright. Third, adding this new failure broke a couple tests. To fix them (and to e.g. allow `inspect.signature(select.epoll.register)` as in #85267), I add constant folding of a couple binary operations to RewriteSymbolics. (There's some discussion of making signature expression evaluation arbitrary powerful in #68155. I think that's out of scope. The additional constant folding here is pretty straightforward, useful, and not much of a slippery slope) Fourth, while #85267 is incorrect about the cause of the issue, it turns out if you had consecutive newlines in __text_signature__, you'd get `tokenize.TokenError`. Finally, the `if name is invalid:` code path was dead, since `parse_name` never returned `invalid`. files: A Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 052f0bfe64b6..3db7745e8a5e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2122,7 +2122,7 @@ def _signature_strip_non_python_syntax(signature): self_parameter = None last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) @@ -2221,11 +2221,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2235,7 +2235,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2245,6 +2245,21 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) if default_node and default_node is not _empty: @@ -2252,7 +2267,7 @@ def p(name_node, default_node, default=empty): default_node = RewriteSymbolics().visit(default_node) default = ast.literal_eval(default_node) except ValueError: - return None + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 78e6e9e34e50..1b589c8df2a2 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2525,7 +2525,7 @@ def p(name): return signature.parameters[name].default self.assertEqual(p('f'), False) self.assertEqual(p('local'), 3) self.assertEqual(p('sys'), sys.maxsize) - self.assertNotIn('exp', signature.parameters) + self.assertEqual(p('exp'), sys.maxsize - 1) test_callable(object) @@ -4323,10 +4323,29 @@ def func(*args, **kwargs): sig = inspect.signature(func) self.assertIsNotNone(sig) self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' sig = inspect.signature(func) self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + def test_base_class_have_text_signature(self): # see issue 43118 from test.ann_module7 import BufferedReader diff --git a/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst new file mode 100644 index 000000000000..e69fd1ca1c2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst @@ -0,0 +1,6 @@ +Several improvements to :func:`inspect.signature`'s handling of ``__text_signature``. +- Fixes a case where :func:`inspect.signature` dropped parameters +- Fixes a case where :func:`inspect.signature` raised :exc:`tokenize.TokenError` +- Allows :func:`inspect.signature` to understand defaults involving binary operations of constants +- :func:`inspect.signature` is documented as only raising :exc:`TypeError` or :exc:`ValueError`, but sometimes raised :exc:`RuntimeError`. These cases now raise :exc:`ValueError` +- Removed a dead code path From webhook-mailer at python.org Tue Dec 20 22:34:23 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 03:34:23 -0000 Subject: [Python-checkins] gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) Message-ID: <mailman.2991.1671593664.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fe828ec709ac9458697071b46c6d997cadacdbb5 commit: fe828ec709ac9458697071b46c6d997cadacdbb5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T19:34:17-08:00 summary: gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) (cherry picked from commit c615286e8576f2555d4380f38a966c300805b1a5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Pieter Eendebak <P.T.eendebak at tudelft.nl> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/weakref.rst diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 73e7b21ae405..1406b663c6a8 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -172,6 +172,30 @@ See :ref:`__slots__ documentation <slots>` for details. application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. + Note that when a key with equal value to an existing key (but not equal identity) + is inserted into the dictionary, it replaces the value but does not replace the + existing key. Due to this, when the reference to the original key is deleted, it + also deletes the entry in the dictionary:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> d[k2] = 2 # d = {k1: 2} + >>> del k1 # d = {} + + A workaround would be to remove the key prior to reassignment:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> del d[k1] + >>> d[k2] = 2 # d = {k2: 2} + >>> del k1 # d = {k2: 2} + .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. From webhook-mailer at python.org Tue Dec 20 22:34:29 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 03:34:29 -0000 Subject: [Python-checkins] gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) Message-ID: <mailman.2992.1671593669.3313.python-checkins@python.org> https://github.com/python/cpython/commit/561e15b85783b87cbc3023aa6cbebf5c76f60a64 commit: 561e15b85783b87cbc3023aa6cbebf5c76f60a64 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T19:34:24-08:00 summary: gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) (cherry picked from commit c615286e8576f2555d4380f38a966c300805b1a5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Pieter Eendebak <P.T.eendebak at tudelft.nl> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/weakref.rst diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 9a8289a7b533..ffe94be8087d 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -170,6 +170,30 @@ See :ref:`__slots__ documentation <slots>` for details. application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. + Note that when a key with equal value to an existing key (but not equal identity) + is inserted into the dictionary, it replaces the value but does not replace the + existing key. Due to this, when the reference to the original key is deleted, it + also deletes the entry in the dictionary:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> d[k2] = 2 # d = {k1: 2} + >>> del k1 # d = {} + + A workaround would be to remove the key prior to reassignment:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> del d[k1] + >>> d[k2] = 2 # d = {k2: 2} + >>> del k1 # d = {k2: 2} + .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. From webhook-mailer at python.org Wed Dec 21 00:25:19 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 05:25:19 -0000 Subject: [Python-checkins] [3.11] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100392) Message-ID: <mailman.2993.1671600321.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bee905184e9297e58d421ecfee556277dc01864d commit: bee905184e9297e58d421ecfee556277dc01864d branch: 3.11 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T21:25:13-08:00 summary: [3.11] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100392) This makes a couple related changes to inspect.signature's behaviour when parsing a signature from `__text_signature__`. First, `inspect.signature` is documented as only raising ValueError or TypeError. However, in some cases, we could raise RuntimeError. This PR changes that, thereby fixing GH-83685. (Note that the new ValueErrors in RewriteSymbolics are caught and then reraised with a message) Second, `inspect.signature` could randomly drop parameters that it didn't understand (corresponding to `return None` in the `p` function). This is the core issue in GH-85267. I think this is very surprising behaviour and it seems better to fail outright. Third, adding this new failure broke a couple tests. To fix them (and to e.g. allow `inspect.signature(select.epoll.register)` as in GH-85267), I add constant folding of a couple binary operations to RewriteSymbolics. (There's some discussion of making signature expression evaluation arbitrary powerful in GH-68155. I think that's out of scope. The additional constant folding here is pretty straightforward, useful, and not much of a slippery slope) Fourth, while GH-85267 is incorrect about the cause of the issue, it turns out if you had consecutive newlines in __text_signature__, you'd get `tokenize.TokenError`. Finally, the `if name is invalid:` code path was dead, since `parse_name` never returned `invalid`.. (cherry picked from commit 79311cbfe718f17c89bab67d7f89da3931bfa2ac) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index cab92b6b8fb8..bc49e680122d 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2116,7 +2116,7 @@ def _signature_strip_non_python_syntax(signature): self_parameter = None last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) @@ -2192,7 +2192,6 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): parameters = [] empty = Parameter.empty - invalid = object() module = None module_dict = {} @@ -2216,11 +2215,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2230,7 +2229,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2240,19 +2239,29 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) - if name is invalid: - return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) + default = ast.literal_eval(default_node) except ValueError: - o = invalid - if o is invalid: - return None - default = o if o is not invalid else default + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 6c5e674aa69f..9ea49854cbc6 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2480,7 +2480,7 @@ def p(name): return signature.parameters[name].default self.assertEqual(p('f'), False) self.assertEqual(p('local'), 3) self.assertEqual(p('sys'), sys.maxsize) - self.assertNotIn('exp', signature.parameters) + self.assertEqual(p('exp'), sys.maxsize - 1) test_callable(object) @@ -4245,10 +4245,29 @@ def func(*args, **kwargs): sig = inspect.signature(func) self.assertIsNotNone(sig) self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' sig = inspect.signature(func) self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + def test_base_class_have_text_signature(self): # see issue 43118 from test.ann_module7 import BufferedReader diff --git a/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst new file mode 100644 index 000000000000..e69fd1ca1c2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst @@ -0,0 +1,6 @@ +Several improvements to :func:`inspect.signature`'s handling of ``__text_signature``. +- Fixes a case where :func:`inspect.signature` dropped parameters +- Fixes a case where :func:`inspect.signature` raised :exc:`tokenize.TokenError` +- Allows :func:`inspect.signature` to understand defaults involving binary operations of constants +- :func:`inspect.signature` is documented as only raising :exc:`TypeError` or :exc:`ValueError`, but sometimes raised :exc:`RuntimeError`. These cases now raise :exc:`ValueError` +- Removed a dead code path From webhook-mailer at python.org Wed Dec 21 00:25:24 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 05:25:24 -0000 Subject: [Python-checkins] [3.10] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100393) Message-ID: <mailman.2994.1671600325.3313.python-checkins@python.org> https://github.com/python/cpython/commit/919045cb73709a3933ec807c09b929c7ad7f02f4 commit: 919045cb73709a3933ec807c09b929c7ad7f02f4 branch: 3.10 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T21:25:18-08:00 summary: [3.10] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100393) This makes a couple related changes to inspect.signature's behaviour when parsing a signature from `__text_signature__`. First, `inspect.signature` is documented as only raising ValueError or TypeError. However, in some cases, we could raise RuntimeError. This PR changes that, thereby fixing GH-83685. (Note that the new ValueErrors in RewriteSymbolics are caught and then reraised with a message) Second, `inspect.signature` could randomly drop parameters that it didn't understand (corresponding to `return None` in the `p` function). This is the core issue in GH-85267. I think this is very surprising behaviour and it seems better to fail outright. Third, adding this new failure broke a couple tests. To fix them (and to e.g. allow `inspect.signature(select.epoll.register)` as in GH-85267), I add constant folding of a couple binary operations to RewriteSymbolics. (There's some discussion of making signature expression evaluation arbitrary powerful in GH-68155. I think that's out of scope. The additional constant folding here is pretty straightforward, useful, and not much of a slippery slope) Fourth, while GH-85267 is incorrect about the cause of the issue, it turns out if you had consecutive newlines in __text_signature__, you'd get `tokenize.TokenError`. Finally, the `if name is invalid:` code path was dead, since `parse_name` never returned `invalid`.. (cherry picked from commit 79311cbfe718f17c89bab67d7f89da3931bfa2ac) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 417371f5cf14..2999a6019e0f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2061,7 +2061,7 @@ def _signature_strip_non_python_syntax(signature): self_parameter = None last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) @@ -2141,7 +2141,6 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): parameters = [] empty = Parameter.empty - invalid = object() module = None module_dict = {} @@ -2165,11 +2164,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2179,7 +2178,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2189,19 +2188,29 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) - if name is invalid: - return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) + default = ast.literal_eval(default_node) except ValueError: - o = invalid - if o is invalid: - return None - default = o if o is not invalid else default + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 5aa4a60de0bf..5e430dad061b 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2477,7 +2477,7 @@ def p(name): return signature.parameters[name].default self.assertEqual(p('f'), False) self.assertEqual(p('local'), 3) self.assertEqual(p('sys'), sys.maxsize) - self.assertNotIn('exp', signature.parameters) + self.assertEqual(p('exp'), sys.maxsize - 1) test_callable(object) @@ -4246,10 +4246,29 @@ def func(*args, **kwargs): sig = inspect.signature(func) self.assertIsNotNone(sig) self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' sig = inspect.signature(func) self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + def test_base_class_have_text_signature(self): # see issue 43118 from test.ann_module7 import BufferedReader diff --git a/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst new file mode 100644 index 000000000000..e69fd1ca1c2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst @@ -0,0 +1,6 @@ +Several improvements to :func:`inspect.signature`'s handling of ``__text_signature``. +- Fixes a case where :func:`inspect.signature` dropped parameters +- Fixes a case where :func:`inspect.signature` raised :exc:`tokenize.TokenError` +- Allows :func:`inspect.signature` to understand defaults involving binary operations of constants +- :func:`inspect.signature` is documented as only raising :exc:`TypeError` or :exc:`ValueError`, but sometimes raised :exc:`RuntimeError`. These cases now raise :exc:`ValueError` +- Removed a dead code path From webhook-mailer at python.org Wed Dec 21 04:49:51 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 09:49:51 -0000 Subject: [Python-checkins] GH-100363: Speed up `asyncio.get_running_loop` (#100364) Message-ID: <mailman.2995.1671616191.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4994f2488f8a436ebda3510c779cbfe292bb21a0 commit: 4994f2488f8a436ebda3510c779cbfe292bb21a0 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:19:44+05:30 summary: GH-100363: Speed up `asyncio.get_running_loop` (#100364) files: A Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst M Lib/asyncio/events.py M Modules/_asynciomodule.c diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 34a8869dff8d..6cff8c59ea47 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -836,6 +836,7 @@ def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _set_running_loop(None) signal.set_wakeup_fd(-1) os.register_at_fork(after_in_child=on_fork) diff --git a/Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst b/Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst new file mode 100644 index 000000000000..69bb52956137 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst @@ -0,0 +1 @@ +Speed up :func:`asyncio.get_running_loop` by removing redundant ``getpid`` checks. Patch by Kumar Aditya. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 60369d89dc39..32be537c00a5 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -23,7 +23,6 @@ typedef struct { PyTypeObject *TaskStepMethWrapper_Type; PyTypeObject *FutureType; PyTypeObject *TaskType; - PyTypeObject *PyRunningLoopHolder_Type; PyObject *asyncio_mod; PyObject *context_kwname; @@ -59,8 +58,8 @@ typedef struct { /* Imports from traceback. */ PyObject *traceback_extract_stack; - PyObject *cached_running_holder; // Borrowed ref. - volatile uint64_t cached_running_holder_tsid; + PyObject *cached_running_loop; // Borrowed reference + volatile uint64_t cached_running_loop_tsid; /* Counter for autogenerated Task names */ uint64_t task_name_counter; @@ -138,14 +137,6 @@ typedef struct { PyObject *sw_arg; } TaskStepMethWrapper; -typedef struct { - PyObject_HEAD - PyObject *rl_loop; -#if defined(HAVE_GETPID) && !defined(MS_WINDOWS) - pid_t rl_pid; -#endif -} PyRunningLoopHolder; - #define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) #define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) @@ -165,8 +156,6 @@ class _asyncio.Future "FutureObj *" "&Future_Type" /* Get FutureIter from Future */ static PyObject * future_new_iter(PyObject *); -static PyRunningLoopHolder * new_running_loop_holder(asyncio_state *, PyObject *); - static int _is_coroutine(asyncio_state *state, PyObject *coro) @@ -264,11 +253,11 @@ get_running_loop(asyncio_state *state, PyObject **loop) PyThreadState *ts = _PyThreadState_GET(); uint64_t ts_id = PyThreadState_GetID(ts); - if (state->cached_running_holder_tsid == ts_id && - state->cached_running_holder != NULL) + if (state->cached_running_loop_tsid == ts_id && + state->cached_running_loop != NULL) { // Fast path, check the cache. - rl = state->cached_running_holder; // borrowed + rl = state->cached_running_loop; } else { PyObject *ts_dict = _PyThreadState_GetDict(ts); // borrowed @@ -287,27 +276,16 @@ get_running_loop(asyncio_state *state, PyObject **loop) } } - state->cached_running_holder = rl; // borrowed - state->cached_running_holder_tsid = ts_id; + state->cached_running_loop = rl; + state->cached_running_loop_tsid = ts_id; } - assert(Py_IS_TYPE(rl, state->PyRunningLoopHolder_Type)); - PyObject *running_loop = ((PyRunningLoopHolder *)rl)->rl_loop; - - if (running_loop == Py_None) { - goto not_found; - } -#if defined(HAVE_GETPID) && !defined(MS_WINDOWS) - /* On Windows there is no getpid, but there is also no os.fork(), - so there is no need for this check. - */ - if (getpid() != ((PyRunningLoopHolder *)rl)->rl_pid) { + if (rl == Py_None) { goto not_found; } -#endif - *loop = Py_NewRef(running_loop); + *loop = Py_NewRef(rl); return 0; not_found: @@ -335,22 +313,14 @@ set_running_loop(asyncio_state *state, PyObject *loop) PyExc_RuntimeError, "thread-local storage is not available"); return -1; } - - PyRunningLoopHolder *rl = new_running_loop_holder(state, loop); - if (rl == NULL) { - return -1; - } - if (PyDict_SetItem( - ts_dict, &_Py_ID(__asyncio_running_event_loop__), (PyObject *)rl) < 0) + ts_dict, &_Py_ID(__asyncio_running_event_loop__), loop) < 0) { - Py_DECREF(rl); // will cleanup loop & current_pid return -1; } - Py_DECREF(rl); - state->cached_running_holder = (PyObject *)rl; - state->cached_running_holder_tsid = PyThreadState_GetID(tstate); + state->cached_running_loop = loop; // borrowed, kept alive by ts_dict + state->cached_running_loop_tsid = PyThreadState_GetID(tstate); return 0; } @@ -3344,79 +3314,6 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) } -/*********************** PyRunningLoopHolder ********************/ - - -static PyRunningLoopHolder * -new_running_loop_holder(asyncio_state *state, PyObject *loop) -{ - PyRunningLoopHolder *rl = PyObject_GC_New( - PyRunningLoopHolder, state->PyRunningLoopHolder_Type); - if (rl == NULL) { - return NULL; - } - -#if defined(HAVE_GETPID) && !defined(MS_WINDOWS) - rl->rl_pid = getpid(); -#endif - rl->rl_loop = Py_NewRef(loop); - - PyObject_GC_Track(rl); - return rl; -} - - -static int -PyRunningLoopHolder_clear(PyRunningLoopHolder *rl) -{ - Py_CLEAR(rl->rl_loop); - return 0; -} - - -static int -PyRunningLoopHolder_traverse(PyRunningLoopHolder *rl, visitproc visit, - void *arg) -{ - Py_VISIT(Py_TYPE(rl)); - Py_VISIT(rl->rl_loop); - return 0; -} - - -static void -PyRunningLoopHolder_tp_dealloc(PyRunningLoopHolder *rl) -{ - asyncio_state *state = get_asyncio_state_by_def((PyObject *)rl); - if (state->cached_running_holder == (PyObject *)rl) { - state->cached_running_holder = NULL; - } - PyTypeObject *tp = Py_TYPE(rl); - PyObject_GC_UnTrack(rl); - PyRunningLoopHolder_clear(rl); - PyObject_GC_Del(rl); - Py_DECREF(tp); -} - - -static PyType_Slot PyRunningLoopHolder_slots[] = { - {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_dealloc, (destructor)PyRunningLoopHolder_tp_dealloc}, - {Py_tp_traverse, (traverseproc)PyRunningLoopHolder_traverse}, - {Py_tp_clear, PyRunningLoopHolder_clear}, - {0, NULL}, -}; - - -static PyType_Spec PyRunningLoopHolder_spec = { - .name = "_asyncio._RunningLoopHolder", - .basicsize = sizeof(PyRunningLoopHolder), - .slots = PyRunningLoopHolder_slots, - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), -}; - - /*********************** Module **************************/ @@ -3448,7 +3345,6 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->TaskStepMethWrapper_Type); Py_VISIT(state->FutureType); Py_VISIT(state->TaskType); - Py_VISIT(state->PyRunningLoopHolder_Type); Py_VISIT(state->asyncio_mod); Py_VISIT(state->traceback_extract_stack); @@ -3486,7 +3382,6 @@ module_clear(PyObject *mod) Py_CLEAR(state->TaskStepMethWrapper_Type); Py_CLEAR(state->FutureType); Py_CLEAR(state->TaskType); - Py_CLEAR(state->PyRunningLoopHolder_Type); Py_CLEAR(state->asyncio_mod); Py_CLEAR(state->traceback_extract_stack); @@ -3625,7 +3520,6 @@ module_exec(PyObject *mod) } while (0) CREATE_TYPE(mod, state->TaskStepMethWrapper_Type, &TaskStepMethWrapper_spec, NULL); - CREATE_TYPE(mod, state->PyRunningLoopHolder_Type, &PyRunningLoopHolder_spec, NULL); CREATE_TYPE(mod, state->FutureIterType, &FutureIter_spec, NULL); CREATE_TYPE(mod, state->FutureType, &Future_spec, NULL); CREATE_TYPE(mod, state->TaskType, &Task_spec, state->FutureType); From webhook-mailer at python.org Wed Dec 21 04:50:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 09:50:32 -0000 Subject: [Python-checkins] GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (#100154) Message-ID: <mailman.2996.1671616234.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a7715ccfba5b86ab09f86ec56ac3755c93b46b48 commit: a7715ccfba5b86ab09f86ec56ac3755c93b46b48 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:20:26+05:30 summary: GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (#100154) files: A Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst M Lib/asyncio/base_subprocess.py M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index e15bb4141fc0..4c9b0dd5653c 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -215,9 +215,6 @@ def _process_exited(self, returncode): # object. On Python 3.6, it is required to avoid a ResourceWarning. self._proc.returncode = returncode self._call(self._protocol.process_exited) - for p in self._pipes.values(): - if p is not None: - p.pipe.close() self._try_finish() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 7411f735da3b..3830dea7d9ba 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -686,6 +686,23 @@ async def execute(): self.assertIsNone(self.loop.run_until_complete(execute())) + def test_subprocess_communicate_stdout(self): + # See https://github.com/python/cpython/issues/100133 + async def get_command_stdout(cmd, *args): + proc = await asyncio.create_subprocess_exec( + cmd, *args, stdout=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout.decode().strip() + + async def main(): + outputs = [f'foo{i}' for i in range(10)] + res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', + f'print({out!r})') for out in outputs]) + self.assertEqual(res, outputs) + + self.loop.run_until_complete(main()) + if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst new file mode 100644 index 000000000000..881e6ed80fed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst @@ -0,0 +1 @@ +Fix regression in :mod:`asyncio` where a subprocess would sometimes lose data received from pipe. From webhook-mailer at python.org Wed Dec 21 05:02:46 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 10:02:46 -0000 Subject: [Python-checkins] [3.11] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100385) Message-ID: <mailman.2997.1671616967.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bed1d141a95e8d69f522e6f4e46e0647c68a2460 commit: bed1d141a95e8d69f522e6f4e46e0647c68a2460 branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:32:29+05:30 summary: [3.11] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100385) (cherry picked from commit efbb1eb9f54cad4f7bf5df03eed3a6aba02d99f4) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst M Lib/test/test_capi/test_getargs.py M Modules/_testcapimodule.c M Python/getargs.c diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 72b6d64a4986..552fa2135247 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1098,6 +1098,10 @@ def test_Z_hash(self): warnings.simplefilter('error', DeprecationWarning) self.assertRaises(DeprecationWarning, getargs_Z_hash, 'abc\xe9') + def test_gh_99240_clear_args(self): + from _testcapi import gh_99240_clear_args + self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') + class Object_TestCase(unittest.TestCase): def test_S(self): diff --git a/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst new file mode 100644 index 000000000000..9a1bb3cf5a73 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst @@ -0,0 +1,2 @@ +In argument parsing, after deallocating newly allocated memory, reset its +pointer to NULL. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1291eff481cb..70242d78ab8c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6192,6 +6192,7 @@ function_get_module(PyObject *self, PyObject *func) static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); static PyObject *getargs_s_hash_int2(PyObject *, PyObject *, PyObject*); +static PyObject *gh_99240_clear_args(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -6308,6 +6309,7 @@ static PyMethodDef TestMethods[] = { METH_VARARGS|METH_KEYWORDS}, {"getargs_s_hash_int2", _PyCFunction_CAST(getargs_s_hash_int2), METH_VARARGS|METH_KEYWORDS}, + {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, {"getargs_z", getargs_z, METH_VARARGS}, {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, @@ -8089,3 +8091,21 @@ getargs_s_hash_int2(PyObject *self, PyObject *args, PyObject *kwargs) PyBuffer_Release(&buf); Py_RETURN_NONE; } + +static PyObject * +gh_99240_clear_args(PyObject *self, PyObject *args) +{ + char *a = NULL; + char *b = NULL; + + if (!PyArg_ParseTuple(args, "eses", "idna", &a, "idna", &b)) { + if (a || b) { + PyErr_Clear(); + PyErr_SetString(PyExc_AssertionError, "Arguments are not cleared."); + } + return NULL; + } + PyMem_Free(a); + PyMem_Free(b); + Py_RETURN_NONE; +} diff --git a/Python/getargs.c b/Python/getargs.c index c3e3cf155b87..9d6483f4fe0f 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -202,9 +202,9 @@ _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) static int cleanup_ptr(PyObject *self, void *ptr) { - if (ptr) { - PyMem_Free(ptr); - } + void **pptr = (void **)ptr; + PyMem_Free(*pptr); + *pptr = NULL; return 0; } @@ -1169,7 +1169,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( "(cleanup problem)", @@ -1215,7 +1215,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr("(cleanup problem)", arg, msgbuf, bufsize); From webhook-mailer at python.org Wed Dec 21 05:03:26 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 10:03:26 -0000 Subject: [Python-checkins] [3.10] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100386) Message-ID: <mailman.2998.1671617007.3313.python-checkins@python.org> https://github.com/python/cpython/commit/591365cd49cf1065e27bb1358aa5f07bd854dd33 commit: 591365cd49cf1065e27bb1358aa5f07bd854dd33 branch: 3.10 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:33:21+05:30 summary: [3.10] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100386) (cherry picked from commit efbb1eb9f54cad4f7bf5df03eed3a6aba02d99f4) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst M Lib/test/test_capi/test_getargs.py M Modules/_testcapimodule.c M Python/getargs.c diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 72b6d64a4986..552fa2135247 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1098,6 +1098,10 @@ def test_Z_hash(self): warnings.simplefilter('error', DeprecationWarning) self.assertRaises(DeprecationWarning, getargs_Z_hash, 'abc\xe9') + def test_gh_99240_clear_args(self): + from _testcapi import gh_99240_clear_args + self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') + class Object_TestCase(unittest.TestCase): def test_S(self): diff --git a/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst new file mode 100644 index 000000000000..9a1bb3cf5a73 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst @@ -0,0 +1,2 @@ +In argument parsing, after deallocating newly allocated memory, reset its +pointer to NULL. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 20c90c441e6b..c0c86722eb3c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5671,6 +5671,7 @@ test_fatal_error(PyObject *self, PyObject *args) static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); static PyObject *getargs_s_hash_int2(PyObject *, PyObject *, PyObject*); +static PyObject *gh_99240_clear_args(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -5784,6 +5785,7 @@ static PyMethodDef TestMethods[] = { METH_VARARGS|METH_KEYWORDS}, {"getargs_s_hash_int2", (PyCFunction)(void(*)(void))getargs_s_hash_int2, METH_VARARGS|METH_KEYWORDS}, + {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, {"getargs_z", getargs_z, METH_VARARGS}, {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, @@ -7500,3 +7502,21 @@ getargs_s_hash_int2(PyObject *self, PyObject *args, PyObject *kwargs) PyBuffer_Release(&buf); Py_RETURN_NONE; } + +static PyObject * +gh_99240_clear_args(PyObject *self, PyObject *args) +{ + char *a = NULL; + char *b = NULL; + + if (!PyArg_ParseTuple(args, "eses", "idna", &a, "idna", &b)) { + if (a || b) { + PyErr_Clear(); + PyErr_SetString(PyExc_AssertionError, "Arguments are not cleared."); + } + return NULL; + } + PyMem_Free(a); + PyMem_Free(b); + Py_RETURN_NONE; +} diff --git a/Python/getargs.c b/Python/getargs.c index fd49ee0f05d4..3b65cfad54f9 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -201,9 +201,9 @@ _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) static int cleanup_ptr(PyObject *self, void *ptr) { - if (ptr) { - PyMem_Free(ptr); - } + void **pptr = (void **)ptr; + PyMem_Free(*pptr); + *pptr = NULL; return 0; } @@ -1168,7 +1168,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( "(cleanup problem)", @@ -1214,7 +1214,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr("(cleanup problem)", arg, msgbuf, bufsize); From webhook-mailer at python.org Wed Dec 21 05:24:25 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 10:24:25 -0000 Subject: [Python-checkins] GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (GH-100154) Message-ID: <mailman.2999.1671618265.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ae8520c70992710903819f24dbce4e7dd05d7ea8 commit: ae8520c70992710903819f24dbce4e7dd05d7ea8 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T02:24:19-08:00 summary: GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (GH-100154) (cherry picked from commit a7715ccfba5b86ab09f86ec56ac3755c93b46b48) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst M Lib/asyncio/base_subprocess.py M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index e15bb4141fc0..4c9b0dd5653c 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -215,9 +215,6 @@ def _process_exited(self, returncode): # object. On Python 3.6, it is required to avoid a ResourceWarning. self._proc.returncode = returncode self._call(self._protocol.process_exited) - for p in self._pipes.values(): - if p is not None: - p.pipe.close() self._try_finish() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index f71ad72f999e..bea2314a5286 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -684,6 +684,23 @@ async def execute(): self.assertIsNone(self.loop.run_until_complete(execute())) + def test_subprocess_communicate_stdout(self): + # See https://github.com/python/cpython/issues/100133 + async def get_command_stdout(cmd, *args): + proc = await asyncio.create_subprocess_exec( + cmd, *args, stdout=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout.decode().strip() + + async def main(): + outputs = [f'foo{i}' for i in range(10)] + res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', + f'print({out!r})') for out in outputs]) + self.assertEqual(res, outputs) + + self.loop.run_until_complete(main()) + if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst new file mode 100644 index 000000000000..881e6ed80fed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst @@ -0,0 +1 @@ +Fix regression in :mod:`asyncio` where a subprocess would sometimes lose data received from pipe. From webhook-mailer at python.org Wed Dec 21 08:25:24 2022 From: webhook-mailer at python.org (corona10) Date: Wed, 21 Dec 2022 13:25:24 -0000 Subject: [Python-checkins] gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) Message-ID: <mailman.3000.1671629125.3313.python-checkins@python.org> https://github.com/python/cpython/commit/12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2 commit: 12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2 branch: main author: Dominic Socular <BBH at awsl.rip> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-21T22:25:04+09:00 summary: gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst M Lib/socket.py M Lib/test/test_socket.py diff --git a/Lib/socket.py b/Lib/socket.py index 1c8cef6ce658..3a4f94de9cc0 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -785,11 +785,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index d808f3f62b96..f1b4018c265e 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1769,6 +1769,10 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst new file mode 100644 index 000000000000..e78352fb188e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst @@ -0,0 +1 @@ +Fix incorrect result and delay in :func:`socket.getfqdn`. Patch by Dominic Socular. From webhook-mailer at python.org Wed Dec 21 09:06:23 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:06:23 -0000 Subject: [Python-checkins] gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) Message-ID: <mailman.3001.1671631584.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b2076b00710c4366dcfe6cd236e480d68a3c38b7 commit: b2076b00710c4366dcfe6cd236e480d68a3c38b7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:06:10-08:00 summary: gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) (cherry picked from commit 12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2) Co-authored-by: Dominic Socular <BBH at awsl.rip> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst M Lib/socket.py M Lib/test/test_socket.py diff --git a/Lib/socket.py b/Lib/socket.py index 0717c696b12e..5a896bee7c49 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -784,11 +784,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index b07954989fdd..13cb2a7945d1 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1760,6 +1760,10 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst new file mode 100644 index 000000000000..e78352fb188e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst @@ -0,0 +1 @@ +Fix incorrect result and delay in :func:`socket.getfqdn`. Patch by Dominic Socular. From webhook-mailer at python.org Wed Dec 21 09:06:33 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:06:33 -0000 Subject: [Python-checkins] gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) Message-ID: <mailman.3002.1671631594.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7f24056dfbcaa080ab825924667e9359d86570d9 commit: 7f24056dfbcaa080ab825924667e9359d86570d9 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:06:27-08:00 summary: gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) (cherry picked from commit 12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2) Co-authored-by: Dominic Socular <BBH at awsl.rip> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst M Lib/socket.py M Lib/test/test_socket.py diff --git a/Lib/socket.py b/Lib/socket.py index 63ba0acc908c..310b074ca278 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -783,11 +783,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index e0e0f2437bb0..211fd8c02da0 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1743,6 +1743,10 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst new file mode 100644 index 000000000000..e78352fb188e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst @@ -0,0 +1 @@ +Fix incorrect result and delay in :func:`socket.getfqdn`. Patch by Dominic Socular. From webhook-mailer at python.org Wed Dec 21 09:31:28 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 21 Dec 2022 14:31:28 -0000 Subject: [Python-checkins] gh-100129: Add tests for pickling all builtin types and functions (GH-100142) Message-ID: <mailman.3003.1671633090.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da commit: b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da branch: main author: Serhiy Storchaka <storchaka at gmail.com> committer: serhiy-storchaka <storchaka at gmail.com> date: 2022-12-21T16:31:22+02:00 summary: gh-100129: Add tests for pickling all builtin types and functions (GH-100142) files: M Lib/test/pickletester.py diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 499f80a15f34..6e87370c2065 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,3 +1,4 @@ +import builtins import collections import copyreg import dbm @@ -11,6 +12,7 @@ import struct import sys import threading +import types import unittest import weakref from textwrap import dedent @@ -1980,6 +1982,33 @@ def test_singleton_types(self): u = self.loads(s) self.assertIs(type(singleton), u) + def test_builtin_types(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and not issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + + def test_builtin_exceptions(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) + + def test_builtin_functions(self): + for t in builtins.__dict__.values(): + if isinstance(t, types.BuiltinFunctionType): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + # Tests for protocol 2 def test_proto(self): From webhook-mailer at python.org Wed Dec 21 09:58:39 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:58:39 -0000 Subject: [Python-checkins] gh-100129: Add tests for pickling all builtin types and functions (GH-100142) Message-ID: <mailman.3004.1671634720.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0397f040e239c82848f51dedef9af7b9bd99ef55 commit: 0397f040e239c82848f51dedef9af7b9bd99ef55 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:58:20-08:00 summary: gh-100129: Add tests for pickling all builtin types and functions (GH-100142) (cherry picked from commit b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/test/pickletester.py diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 499f80a15f34..6e87370c2065 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,3 +1,4 @@ +import builtins import collections import copyreg import dbm @@ -11,6 +12,7 @@ import struct import sys import threading +import types import unittest import weakref from textwrap import dedent @@ -1980,6 +1982,33 @@ def test_singleton_types(self): u = self.loads(s) self.assertIs(type(singleton), u) + def test_builtin_types(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and not issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + + def test_builtin_exceptions(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) + + def test_builtin_functions(self): + for t in builtins.__dict__.values(): + if isinstance(t, types.BuiltinFunctionType): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + # Tests for protocol 2 def test_proto(self): From webhook-mailer at python.org Wed Dec 21 09:58:39 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:58:39 -0000 Subject: [Python-checkins] gh-100129: Add tests for pickling all builtin types and functions (GH-100142) Message-ID: <mailman.3005.1671634721.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2dda6680ccb50bd1d8a8dcf0d524feb7daf5a3af commit: 2dda6680ccb50bd1d8a8dcf0d524feb7daf5a3af branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:58:31-08:00 summary: gh-100129: Add tests for pickling all builtin types and functions (GH-100142) (cherry picked from commit b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/test/pickletester.py diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 18d7f52ecffd..72d6f52c90de 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,3 +1,4 @@ +import builtins import collections import copyreg import dbm @@ -11,6 +12,7 @@ import struct import sys import threading +import types import unittest import weakref from textwrap import dedent @@ -1979,6 +1981,33 @@ def test_singleton_types(self): u = self.loads(s) self.assertIs(type(singleton), u) + def test_builtin_types(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and not issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + + def test_builtin_exceptions(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) + + def test_builtin_functions(self): + for t in builtins.__dict__.values(): + if isinstance(t, types.BuiltinFunctionType): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + # Tests for protocol 2 def test_proto(self): From webhook-mailer at python.org Wed Dec 21 11:55:10 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 16:55:10 -0000 Subject: [Python-checkins] Remove unused variable from `dis._find_imports` (#100396) Message-ID: <mailman.3006.1671641710.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e0b4d966a8d1867c4b535b043e08288ca49b3548 commit: e0b4d966a8d1867c4b535b043e08288ca49b3548 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T22:25:02+05:30 summary: Remove unused variable from `dis._find_imports` (#100396) files: M Lib/dis.py diff --git a/Lib/dis.py b/Lib/dis.py index 523bd01d9295..76104c6098d4 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -666,7 +666,6 @@ def _find_imports(co): the corresponding args to __import__. """ IMPORT_NAME = opmap['IMPORT_NAME'] - LOAD_CONST = opmap['LOAD_CONST'] consts = co.co_consts names = co.co_names From webhook-mailer at python.org Wed Dec 21 12:31:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 17:31:23 -0000 Subject: [Python-checkins] gh-78878: Fix crash when creating an instance of `_ctypes.CField` (#14837) Message-ID: <mailman.3007.1671643885.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d713c54ac8a2eba0616a5a07714696d935f1062e commit: d713c54ac8a2eba0616a5a07714696d935f1062e branch: main author: Hai Shi <shihai1992 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T23:01:17+05:30 summary: gh-78878: Fix crash when creating an instance of `_ctypes.CField` (#14837) files: A Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst M Lib/test/test_ctypes/test_struct_fields.py M Modules/_ctypes/cfield.c M Modules/_ctypes/stgdict.c diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index fefeaea1496a..e444f5e1f779 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -54,6 +54,12 @@ class X(Structure): x.char = b'a\0b\0' self.assertEqual(bytes(x), b'a\x00###') + def test_6(self): + class X(Structure): + _fields_ = [("x", c_int)] + CField = type(X.x) + self.assertRaises(TypeError, CField) + def test_gh99275(self): class BrokenStructure(Structure): def __init_subclass__(cls, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst new file mode 100644 index 000000000000..8b455fd2ef7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst @@ -0,0 +1 @@ +Fix crash when creating an instance of :class:`!_ctypes.CField`. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 791aeba66539..796a1bec966d 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -30,13 +30,6 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ -static PyObject * -PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CFieldObject *obj; - obj = (CFieldObject *)type->tp_alloc(type, 0); - return (PyObject *)obj; -} /* * Expects the size, index and offset for the current field in *psize and @@ -68,7 +61,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, #define CONT_BITFIELD 2 #define EXPAND_BITFIELD 3 - self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + self = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (self == NULL) return NULL; dict = PyType_stgdict(desc); @@ -341,7 +334,7 @@ PyTypeObject PyCField_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - PyCField_new, /* tp_new */ + 0, /* tp_new */ 0, /* tp_free */ }; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 099331ca8bdb..9a4041fb2528 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -257,7 +257,7 @@ MakeFields(PyObject *type, CFieldObject *descr, } continue; } - new_descr = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + new_descr = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (new_descr == NULL) { Py_DECREF(fdescr); Py_DECREF(fieldlist); From webhook-mailer at python.org Wed Dec 21 12:48:31 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 17:48:31 -0000 Subject: [Python-checkins] GH-69564: Clarify use of octal format of mode argument in help(os.chmod) (#20621) Message-ID: <mailman.3008.1671644913.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bbf4a66a18cc0fa9121e1bbc90d75d10f5b3a389 commit: bbf4a66a18cc0fa9121e1bbc90d75d10f5b3a389 branch: main author: amaajemyfren <32741226+amaajemyfren at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T23:18:25+05:30 summary: GH-69564: Clarify use of octal format of mode argument in help(os.chmod) (#20621) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst M Modules/clinic/posixmodule.c.h M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst b/Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst new file mode 100644 index 000000000000..019a1c42d88e --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst @@ -0,0 +1 @@ +Clarify use of octal format of mode argument in help(os.chmod) as well as help(os.fchmod) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 86251008b1bd..d4722cc533cb 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -501,6 +501,9 @@ PyDoc_STRVAR(os_chmod__doc__, " If this functionality is unavailable, using it raises an exception.\n" " mode\n" " Operating-system mode bitfield.\n" +" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" +" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" +" Python.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -602,6 +605,14 @@ PyDoc_STRVAR(os_fchmod__doc__, "\n" "Change the access permissions of the file given by file descriptor fd.\n" "\n" +" fd\n" +" The file descriptor of the file to be modified.\n" +" mode\n" +" Operating-system mode bitfield.\n" +" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" +" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" +" Python.\n" +"\n" "Equivalent to os.chmod(fd, mode)."); #define OS_FCHMOD_METHODDEF \ @@ -11549,4 +11560,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=04fd23c89ab41f75 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=41eab6c3523792a9 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4817973262f4..607d40b59d96 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3173,6 +3173,9 @@ os.chmod mode: int Operating-system mode bitfield. + Be careful when using number literals for *mode*. The conventional UNIX notation for + numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in + Python. * @@ -3198,7 +3201,7 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=989081551c00293b]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=674a14bc998de09d]*/ { int result; @@ -3328,7 +3331,12 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, os.fchmod fd: int + The file descriptor of the file to be modified. mode: int + Operating-system mode bitfield. + Be careful when using number literals for *mode*. The conventional UNIX notation for + numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in + Python. Change the access permissions of the file given by file descriptor fd. @@ -3337,7 +3345,7 @@ Equivalent to os.chmod(fd, mode). static PyObject * os_fchmod_impl(PyObject *module, int fd, int mode) -/*[clinic end generated code: output=afd9bc05b4e426b3 input=8ab11975ca01ee5b]*/ +/*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ { int res; int async_err = 0; From webhook-mailer at python.org Wed Dec 21 13:02:12 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 18:02:12 -0000 Subject: [Python-checkins] gh-78878: Fix crash when creating an instance of `_ctypes.CField` (GH-14837) Message-ID: <mailman.3009.1671645733.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3e715e0cc811ebccce4234c3e25ef30787151d00 commit: 3e715e0cc811ebccce4234c3e25ef30787151d00 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T10:02:05-08:00 summary: gh-78878: Fix crash when creating an instance of `_ctypes.CField` (GH-14837) (cherry picked from commit d713c54ac8a2eba0616a5a07714696d935f1062e) Co-authored-by: Hai Shi <shihai1992 at gmail.com> files: A Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst M Lib/ctypes/test/test_struct_fields.py M Modules/_ctypes/cfield.c M Modules/_ctypes/stgdict.c diff --git a/Lib/ctypes/test/test_struct_fields.py b/Lib/ctypes/test/test_struct_fields.py index fefeaea1496a..e444f5e1f779 100644 --- a/Lib/ctypes/test/test_struct_fields.py +++ b/Lib/ctypes/test/test_struct_fields.py @@ -54,6 +54,12 @@ class X(Structure): x.char = b'a\0b\0' self.assertEqual(bytes(x), b'a\x00###') + def test_6(self): + class X(Structure): + _fields_ = [("x", c_int)] + CField = type(X.x) + self.assertRaises(TypeError, CField) + def test_gh99275(self): class BrokenStructure(Structure): def __init_subclass__(cls, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst new file mode 100644 index 000000000000..8b455fd2ef7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst @@ -0,0 +1 @@ +Fix crash when creating an instance of :class:`!_ctypes.CField`. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 13ed8b7eda65..9bdf1db856e7 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -30,13 +30,6 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ -static PyObject * -PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CFieldObject *obj; - obj = (CFieldObject *)type->tp_alloc(type, 0); - return (PyObject *)obj; -} /* * Expects the size, index and offset for the current field in *psize and @@ -68,7 +61,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, #define CONT_BITFIELD 2 #define EXPAND_BITFIELD 3 - self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + self = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (self == NULL) return NULL; dict = PyType_stgdict(desc); @@ -343,7 +336,7 @@ PyTypeObject PyCField_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - PyCField_new, /* tp_new */ + 0, /* tp_new */ 0, /* tp_free */ }; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 88999b8dff18..b7134ce2571b 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -258,7 +258,7 @@ MakeFields(PyObject *type, CFieldObject *descr, } continue; } - new_descr = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + new_descr = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (new_descr == NULL) { Py_DECREF(fdescr); Py_DECREF(fieldlist); From webhook-mailer at python.org Wed Dec 21 19:41:24 2022 From: webhook-mailer at python.org (brandtbucher) Date: Thu, 22 Dec 2022 00:41:24 -0000 Subject: [Python-checkins] GH-99554: Pack location tables more effectively (GH-99556) Message-ID: <mailman.3010.1671669684.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3c033a2e6fbde56f904aeca138141be6564341cf commit: 3c033a2e6fbde56f904aeca138141be6564341cf branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: brandtbucher <brandtbucher at gmail.com> date: 2022-12-21T16:41:18-08:00 summary: GH-99554: Pack location tables more effectively (GH-99556) files: A Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst M Programs/test_frozenmain.h M Python/compile.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst new file mode 100644 index 000000000000..96ec47db461d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst @@ -0,0 +1 @@ +Pack debugging location tables more efficiently during bytecode compilation. diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 96be3ce3c25c..95f78b19e65e 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -28,15 +28,12 @@ unsigned char M_test_frozenmain[] = { 115,114,3,0,0,0,218,3,107,101,121,169,0,243,0,0, 0,0,250,18,116,101,115,116,95,102,114,111,122,101,110,109, 97,105,110,46,112,121,250,8,60,109,111,100,117,108,101,62, - 114,18,0,0,0,1,0,0,0,115,154,0,0,0,240,3, - 1,1,1,240,8,0,1,11,128,10,128,10,128,10,216,0, - 24,208,0,24,208,0,24,208,0,24,224,0,5,128,5,208, - 6,26,212,0,27,208,0,27,216,0,5,128,5,128,106,144, - 35,151,40,145,40,212,0,27,208,0,27,216,9,38,208,9, - 26,215,9,38,209,9,38,212,9,40,168,24,212,9,50,128, - 6,240,2,6,12,2,240,0,7,1,42,241,0,7,1,42, - 128,67,240,14,0,5,10,128,69,208,10,40,144,67,208,10, - 40,208,10,40,152,54,160,35,156,59,208,10,40,208,10,40, - 212,4,41,208,4,41,208,4,41,240,15,7,1,42,240,0, - 7,1,42,240,0,7,1,42,114,16,0,0,0, + 114,18,0,0,0,1,0,0,0,115,100,0,0,0,240,3, + 1,1,1,243,8,0,1,11,219,0,24,225,0,5,208,6, + 26,213,0,27,217,0,5,128,106,144,35,151,40,145,40,213, + 0,27,216,9,38,208,9,26,215,9,38,209,9,38,212,9, + 40,168,24,212,9,50,128,6,240,2,6,12,2,242,0,7, + 1,42,128,67,241,14,0,5,10,208,10,40,144,67,209,10, + 40,152,54,160,35,156,59,209,10,40,214,4,41,242,15,7, + 1,42,114,16,0,0,0, }; diff --git a/Python/compile.c b/Python/compile.c index 09eb4016940d..023c13507d6a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -150,6 +150,15 @@ location_is_after(location loc1, location loc2) { (loc1.col_offset > loc2.end_col_offset)); } +static inline bool +same_location(location a, location b) +{ + return a.lineno == b.lineno && + a.end_lineno == b.end_lineno && + a.col_offset == b.col_offset && + a.end_col_offset == b.end_col_offset; +} + #define LOC(x) SRC_LOCATION_FROM_AST(x) typedef struct jump_target_label_ { @@ -7722,15 +7731,15 @@ write_location_info_oneline_form(struct assembler* a, int length, int line_delta } static void -write_location_info_long_form(struct assembler* a, struct instr* i, int length) +write_location_info_long_form(struct assembler* a, location loc, int length) { assert(length > 0 && length <= 8); write_location_first_byte(a, PY_CODE_LOCATION_INFO_LONG, length); - write_location_signed_varint(a, i->i_loc.lineno - a->a_lineno); - assert(i->i_loc.end_lineno >= i->i_loc.lineno); - write_location_varint(a, i->i_loc.end_lineno - i->i_loc.lineno); - write_location_varint(a, i->i_loc.col_offset + 1); - write_location_varint(a, i->i_loc.end_col_offset + 1); + write_location_signed_varint(a, loc.lineno - a->a_lineno); + assert(loc.end_lineno >= loc.lineno); + write_location_varint(a, loc.end_lineno - loc.lineno); + write_location_varint(a, loc.col_offset + 1); + write_location_varint(a, loc.end_col_offset + 1); } static void @@ -7749,7 +7758,7 @@ write_location_info_no_column(struct assembler* a, int length, int line_delta) #define THEORETICAL_MAX_ENTRY_SIZE 25 /* 1 + 6 + 6 + 6 + 6 */ static int -write_location_info_entry(struct assembler* a, struct instr* i, int isize) +write_location_info_entry(struct assembler* a, location loc, int isize) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_linetable); if (a->a_location_off + THEORETICAL_MAX_ENTRY_SIZE >= len) { @@ -7758,49 +7767,51 @@ write_location_info_entry(struct assembler* a, struct instr* i, int isize) return -1; } } - if (i->i_loc.lineno < 0) { + if (loc.lineno < 0) { write_location_info_none(a, isize); return 0; } - int line_delta = i->i_loc.lineno - a->a_lineno; - int column = i->i_loc.col_offset; - int end_column = i->i_loc.end_col_offset; + int line_delta = loc.lineno - a->a_lineno; + int column = loc.col_offset; + int end_column = loc.end_col_offset; assert(column >= -1); assert(end_column >= -1); if (column < 0 || end_column < 0) { - if (i->i_loc.end_lineno == i->i_loc.lineno || i->i_loc.end_lineno == -1) { + if (loc.end_lineno == loc.lineno || loc.end_lineno == -1) { write_location_info_no_column(a, isize, line_delta); - a->a_lineno = i->i_loc.lineno; + a->a_lineno = loc.lineno; return 0; } } - else if (i->i_loc.end_lineno == i->i_loc.lineno) { + else if (loc.end_lineno == loc.lineno) { if (line_delta == 0 && column < 80 && end_column - column < 16 && end_column >= column) { write_location_info_short_form(a, isize, column, end_column); return 0; } if (line_delta >= 0 && line_delta < 3 && column < 128 && end_column < 128) { write_location_info_oneline_form(a, isize, line_delta, column, end_column); - a->a_lineno = i->i_loc.lineno; + a->a_lineno = loc.lineno; return 0; } } - write_location_info_long_form(a, i, isize); - a->a_lineno = i->i_loc.lineno; + write_location_info_long_form(a, loc, isize); + a->a_lineno = loc.lineno; return 0; } static int -assemble_emit_location(struct assembler* a, struct instr* i) +assemble_emit_location(struct assembler* a, location loc, int isize) { - int isize = instr_size(i); + if (isize == 0) { + return 0; + } while (isize > 8) { - if (write_location_info_entry(a, i, 8) < 0) { + if (write_location_info_entry(a, loc, 8)) { return -1; } isize -= 8; } - return write_location_info_entry(a, i, isize); + return write_location_info_entry(a, loc, isize); } /* assemble_emit() @@ -8860,13 +8871,23 @@ assemble(struct compiler *c, int addNone) /* Emit location info */ a.a_lineno = c->u->u_firstlineno; + location loc = NO_LOCATION; + int size = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int j = 0; j < b->b_iused; j++) { - if (assemble_emit_location(&a, &b->b_instr[j]) < 0) { - goto error; + if (!same_location(loc, b->b_instr[j].i_loc)) { + if (assemble_emit_location(&a, loc, size)) { + goto error; + } + loc = b->b_instr[j].i_loc; + size = 0; } + size += instr_size(&b->b_instr[j]); } } + if (assemble_emit_location(&a, loc, size)) { + goto error; + } if (assemble_exception_table(&a, g->g_entryblock) < 0) { goto error; From webhook-mailer at python.org Thu Dec 22 06:21:03 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Thu, 22 Dec 2022 11:21:03 -0000 Subject: [Python-checkins] Correct typo in typing.py (#100423) Message-ID: <mailman.3011.1671708064.3313.python-checkins@python.org> https://github.com/python/cpython/commit/68981578eceee763da4163e93cf653cc6b1b6d35 commit: 68981578eceee763da4163e93cf653cc6b1b6d35 branch: main author: david-why <david_why at outlook.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-22T11:20:52Z summary: Correct typo in typing.py (#100423) In the docstring of `ParamSpec`, the name of `P = ParamSpec('P')` was mistakenly written as `'T'`. files: M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index d9d6fbcdb8f0..8bc38f98c867 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1194,7 +1194,7 @@ def add_two(x: float, y: float) -> float: Parameter specification variables can be introspected. e.g.: - P.__name__ == 'T' + P.__name__ == 'P' P.__bound__ == None P.__covariant__ == False P.__contravariant__ == False From webhook-mailer at python.org Thu Dec 22 06:30:28 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 22 Dec 2022 11:30:28 -0000 Subject: [Python-checkins] gh-99761: Add `_PyLong_IsPositiveSingleDigit` function to check for single digit integers (#100064) Message-ID: <mailman.3012.1671708629.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b82c36f17ada471e734c3ad93e6eff8b36a5ad9 commit: 2b82c36f17ada471e734c3ad93e6eff8b36a5ad9 branch: main author: Pieter Eendebak <pieter.eendebak at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-22T17:00:18+05:30 summary: gh-99761: Add `_PyLong_IsPositiveSingleDigit` function to check for single digit integers (#100064) files: M Include/internal/pycore_long.h M Python/bytecodes.c M Python/generated_cases.c.h diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 30c97b7edc98..8c1d017bb95e 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -110,6 +110,25 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); +/* Return 1 if the argument is positive single digit int */ +static inline int +_PyLong_IsPositiveSingleDigit(PyObject* sub) { + /* For a positive single digit int, the value of Py_SIZE(sub) is 0 or 1. + + We perform a fast check using a single comparison by casting from int + to uint which casts negative numbers to large positive numbers. + For details see Section 14.2 "Bounds Checking" in the Agner Fog + optimization manual found at: + https://www.agner.org/optimize/optimizing_cpp.pdf + + The function is not affected by -fwrapv, -fno-wrapv and -ftrapv + compiler options of GCC and clang + */ + assert(PyLong_CheckExact(sub)); + Py_ssize_t signed_size = Py_SIZE(sub); + return ((size_t)signed_size) <= 1; +} + #ifdef __cplusplus } #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b29e16e080e9..c0b625bd662c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -391,8 +391,7 @@ dummy_func( DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); @@ -410,8 +409,7 @@ dummy_func( DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); @@ -508,7 +506,7 @@ dummy_func( DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; // Ensure index < len(list) DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6d84a643b457..42b7ca086705 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -491,8 +491,7 @@ DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); @@ -517,8 +516,7 @@ DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); @@ -642,7 +640,7 @@ DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; // Ensure index < len(list) DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); From webhook-mailer at python.org Thu Dec 22 06:44:06 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 11:44:06 -0000 Subject: [Python-checkins] Correct typo in typing.py (GH-100423) Message-ID: <mailman.3013.1671709447.3313.python-checkins@python.org> https://github.com/python/cpython/commit/58967d1abec9a88b1657c50b421795b4eb519e56 commit: 58967d1abec9a88b1657c50b421795b4eb519e56 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T03:43:55-08:00 summary: Correct typo in typing.py (GH-100423) In the docstring of `ParamSpec`, the name of `P = ParamSpec('P')` was mistakenly written as `'T'`. (cherry picked from commit 68981578eceee763da4163e93cf653cc6b1b6d35) Co-authored-by: david-why <david_why at outlook.com> files: M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 9f8710b9f773..03cf243366db 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -901,7 +901,7 @@ def add_two(x: float, y: float) -> float: Parameter specification variables can be introspected. e.g.: - P.__name__ == 'T' + P.__name__ == 'P' P.__bound__ == None P.__covariant__ == False P.__contravariant__ == False From webhook-mailer at python.org Thu Dec 22 06:49:24 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 11:49:24 -0000 Subject: [Python-checkins] Correct typo in typing.py (GH-100423) Message-ID: <mailman.3014.1671709765.3313.python-checkins@python.org> https://github.com/python/cpython/commit/6afc389e1d1fa969a6cbfe5d6e51d6c2f8637d7d commit: 6afc389e1d1fa969a6cbfe5d6e51d6c2f8637d7d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T03:49:18-08:00 summary: Correct typo in typing.py (GH-100423) In the docstring of `ParamSpec`, the name of `P = ParamSpec('P')` was mistakenly written as `'T'`. (cherry picked from commit 68981578eceee763da4163e93cf653cc6b1b6d35) Co-authored-by: david-why <david_why at outlook.com> files: M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 9818237de435..b287446169c2 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1174,7 +1174,7 @@ def add_two(x: float, y: float) -> float: Parameter specification variables can be introspected. e.g.: - P.__name__ == 'T' + P.__name__ == 'P' P.__bound__ == None P.__covariant__ == False P.__contravariant__ == False From webhook-mailer at python.org Thu Dec 22 07:45:05 2022 From: webhook-mailer at python.org (markshannon) Date: Thu, 22 Dec 2022 12:45:05 -0000 Subject: [Python-checkins] GH-99770: Make the correct call specialization fail kind show up in the stats (GH-99771) Message-ID: <mailman.3015.1671713105.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a02161286a67692758cac38d9dbe42625c211605 commit: a02161286a67692758cac38d9dbe42625c211605 branch: main author: penguin_wwy <940375606 at qq.com> committer: markshannon <mark at hotpy.org> date: 2022-12-22T12:44:59Z summary: GH-99770: Make the correct call specialization fail kind show up in the stats (GH-99771) files: M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Python/specialize.c b/Python/specialize.c index c6c502716478..69a7bf98cad0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -309,7 +309,8 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_OUT_OF_RANGE 4 #define SPEC_FAIL_EXPECTED_ERROR 5 #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6 -#define SPEC_FAIL_NOT_PY_FUNCTION 7 +#define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7 +#define SPEC_FAIL_CODE_NOT_OPTIMIZED 8 #define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17 @@ -317,18 +318,18 @@ _PyCode_Quicken(PyCodeObject *code) /* Attributes */ -#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 8 -#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 9 -#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 10 -#define SPEC_FAIL_ATTR_METHOD 11 -#define SPEC_FAIL_ATTR_MUTABLE_CLASS 12 -#define SPEC_FAIL_ATTR_PROPERTY 13 -#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 14 -#define SPEC_FAIL_ATTR_READ_ONLY 15 -#define SPEC_FAIL_ATTR_AUDITED_SLOT 16 -#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 17 -#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 18 -#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 19 +#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9 +#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10 +#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11 +#define SPEC_FAIL_ATTR_METHOD 12 +#define SPEC_FAIL_ATTR_MUTABLE_CLASS 13 +#define SPEC_FAIL_ATTR_PROPERTY 14 +#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15 +#define SPEC_FAIL_ATTR_READ_ONLY 16 +#define SPEC_FAIL_ATTR_AUDITED_SLOT 17 +#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18 +#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19 +#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20 #define SPEC_FAIL_ATTR_SHADOWED 21 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 @@ -346,12 +347,12 @@ _PyCode_Quicken(PyCodeObject *code) /* Binary subscr and store subscr */ -#define SPEC_FAIL_SUBSCR_ARRAY_INT 8 -#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 9 -#define SPEC_FAIL_SUBSCR_LIST_SLICE 10 -#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 11 -#define SPEC_FAIL_SUBSCR_STRING_INT 12 -#define SPEC_FAIL_SUBSCR_STRING_SLICE 13 +#define SPEC_FAIL_SUBSCR_ARRAY_INT 9 +#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10 +#define SPEC_FAIL_SUBSCR_LIST_SLICE 11 +#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12 +#define SPEC_FAIL_SUBSCR_STRING_INT 13 +#define SPEC_FAIL_SUBSCR_STRING_SLICE 14 #define SPEC_FAIL_SUBSCR_BUFFER_INT 15 #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16 #define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17 @@ -366,49 +367,48 @@ _PyCode_Quicken(PyCodeObject *code) /* Binary op */ -#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 8 -#define SPEC_FAIL_BINARY_OP_ADD_OTHER 9 -#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 10 -#define SPEC_FAIL_BINARY_OP_AND_INT 11 -#define SPEC_FAIL_BINARY_OP_AND_OTHER 12 -#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 13 -#define SPEC_FAIL_BINARY_OP_LSHIFT 14 -#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 15 -#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 16 -#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 17 -#define SPEC_FAIL_BINARY_OP_OR 18 -#define SPEC_FAIL_BINARY_OP_POWER 19 -#define SPEC_FAIL_BINARY_OP_REMAINDER 20 -#define SPEC_FAIL_BINARY_OP_RSHIFT 21 -#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 22 -#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 23 -#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24 -#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 25 -#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 26 -#define SPEC_FAIL_BINARY_OP_XOR 27 +#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 9 +#define SPEC_FAIL_BINARY_OP_ADD_OTHER 10 +#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 11 +#define SPEC_FAIL_BINARY_OP_AND_INT 12 +#define SPEC_FAIL_BINARY_OP_AND_OTHER 13 +#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 14 +#define SPEC_FAIL_BINARY_OP_LSHIFT 15 +#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 16 +#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 17 +#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 18 +#define SPEC_FAIL_BINARY_OP_OR 19 +#define SPEC_FAIL_BINARY_OP_POWER 20 +#define SPEC_FAIL_BINARY_OP_REMAINDER 21 +#define SPEC_FAIL_BINARY_OP_RSHIFT 22 +#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 23 +#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 24 +#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25 +#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26 +#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27 +#define SPEC_FAIL_BINARY_OP_XOR 28 /* Calls */ -#define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9 -#define SPEC_FAIL_CALL_CO_NOT_OPTIMIZED 10 -/* SPEC_FAIL_METHOD defined as 11 above */ #define SPEC_FAIL_CALL_INSTANCE_METHOD 11 #define SPEC_FAIL_CALL_CMETHOD 12 #define SPEC_FAIL_CALL_CFUNC_VARARGS 13 #define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14 -#define SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS 15 -#define SPEC_FAIL_CALL_CFUNC_NOARGS 16 -#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17 -#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 18 -#define SPEC_FAIL_CALL_PYTHON_CLASS 19 -#define SPEC_FAIL_CALL_PEP_523 20 -#define SPEC_FAIL_CALL_BOUND_METHOD 21 -#define SPEC_FAIL_CALL_STR 22 -#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23 -#define SPEC_FAIL_CALL_CLASS_MUTABLE 24 -#define SPEC_FAIL_CALL_KWNAMES 25 -#define SPEC_FAIL_CALL_METHOD_WRAPPER 26 -#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27 +#define SPEC_FAIL_CALL_CFUNC_NOARGS 15 +#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16 +#define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17 +#define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18 +#define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19 +#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20 +#define SPEC_FAIL_CALL_PYTHON_CLASS 21 +#define SPEC_FAIL_CALL_PEP_523 22 +#define SPEC_FAIL_CALL_BOUND_METHOD 23 +#define SPEC_FAIL_CALL_STR 24 +#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 25 +#define SPEC_FAIL_CALL_CLASS_MUTABLE 26 +#define SPEC_FAIL_CALL_KWNAMES 27 +#define SPEC_FAIL_CALL_METHOD_WRAPPER 28 +#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -449,8 +449,8 @@ _PyCode_Quicken(PyCodeObject *code) // UNPACK_SEQUENCE -#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 8 -#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 9 +#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9 +#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10 static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); @@ -1251,10 +1251,10 @@ static int function_kind(PyCodeObject *code) { int flags = code->co_flags; if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) { - return SPEC_FAIL_CALL_COMPLEX_PARAMETERS; + return SPEC_FAIL_CODE_COMPLEX_PARAMETERS; } if ((flags & CO_OPTIMIZED) == 0) { - return SPEC_FAIL_CALL_CO_NOT_OPTIMIZED; + return SPEC_FAIL_CODE_NOT_OPTIMIZED; } return SIMPLE_FUNCTION; } @@ -1526,8 +1526,6 @@ builtin_call_fail_kind(int ml_flags) return SPEC_FAIL_CALL_CFUNC_VARARGS; case METH_VARARGS | METH_KEYWORDS: return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS; case METH_NOARGS: return SPEC_FAIL_CALL_CFUNC_NOARGS; case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: @@ -1535,6 +1533,29 @@ builtin_call_fail_kind(int ml_flags) /* These cases should be optimized, but return "other" just in case */ case METH_O: case METH_FASTCALL: + case METH_FASTCALL | METH_KEYWORDS: + return SPEC_FAIL_OTHER; + default: + return SPEC_FAIL_CALL_BAD_CALL_FLAGS; + } +} + +static int +meth_descr_call_fail_kind(int ml_flags) +{ + switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | + METH_KEYWORDS | METH_METHOD)) { + case METH_VARARGS: + return SPEC_FAIL_CALL_METH_DESCR_VARARGS; + case METH_VARARGS | METH_KEYWORDS: + return SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + return SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS; + /* These cases should be optimized, but return "other" just in case */ + case METH_NOARGS: + case METH_O: + case METH_FASTCALL: + case METH_FASTCALL | METH_KEYWORDS: return SPEC_FAIL_OTHER; default: return SPEC_FAIL_CALL_BAD_CALL_FLAGS; @@ -1583,12 +1604,12 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); return 0; } - case METH_FASTCALL|METH_KEYWORDS: { + case METH_FASTCALL | METH_KEYWORDS: { _py_set_opcode(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } - SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags)); + SPECIALIZATION_FAIL(CALL, meth_descr_call_fail_kind(descr->d_method->ml_flags)); return -1; } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 81b06f9f7469..1c8d10f70277 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -224,7 +224,7 @@ def pretty(defname): return defname.replace("_", " ").lower() def kind_to_text(kind, defines, opname): - if kind <= 7: + if kind <= 8: return pretty(defines[kind][0]) if opname.endswith("ATTR"): opname = "ATTR" From webhook-mailer at python.org Thu Dec 22 08:28:31 2022 From: webhook-mailer at python.org (FFY00) Date: Thu, 22 Dec 2022 13:28:31 -0000 Subject: [Python-checkins] gh-78997: fix bad rebase of moved test file (#100424) Message-ID: <mailman.3016.1671715712.3313.python-checkins@python.org> https://github.com/python/cpython/commit/aa878f086b7ba8bdd7006d9d509c671167a5fb1e commit: aa878f086b7ba8bdd7006d9d509c671167a5fb1e branch: main author: Eli Schwartz <eschwartz93 at gmail.com> committer: FFY00 <filipe.lains at gmail.com> date: 2022-12-22T13:28:25Z summary: gh-78997: fix bad rebase of moved test file (#100424) files: D Lib/ctypes/test/test_loading.py M Lib/test/test_ctypes/test_loading.py diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py deleted file mode 100644 index b61d6fa2912a..000000000000 --- a/Lib/ctypes/test/test_loading.py +++ /dev/null @@ -1,188 +0,0 @@ -from ctypes import * -import os -import shutil -import subprocess -import sys -import unittest -import test.support -from test.support import import_helper -from test.support import os_helper -from ctypes.util import find_library - -libc_name = None - -def setUpModule(): - global libc_name - if os.name == "nt": - libc_name = find_library("c") - elif sys.platform == "cygwin": - libc_name = "cygwin1.dll" - else: - libc_name = find_library("c") - - if test.support.verbose: - print("libc_name is", libc_name) - -class LoaderTest(unittest.TestCase): - - unknowndll = "xxrandomnamexx" - - def test_load(self): - if libc_name is None: - self.skipTest('could not find libc') - CDLL(libc_name) - CDLL(os.path.basename(libc_name)) - self.assertRaises(OSError, CDLL, self.unknowndll) - - def test_load_version(self): - if libc_name is None: - self.skipTest('could not find libc') - if os.path.basename(libc_name) != 'libc.so.6': - self.skipTest('wrong libc path for test') - cdll.LoadLibrary("libc.so.6") - # linux uses version, libc 9 should not exist - self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") - self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) - - def test_find(self): - for name in ("c", "m"): - lib = find_library(name) - if lib: - cdll.LoadLibrary(lib) - CDLL(lib) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_library(self): - # CRT is no longer directly loadable. See issue23606 for the - # discussion about alternative approaches. - #self.assertIsNotNone(libc_name) - if test.support.verbose: - print(find_library("kernel32")) - print(find_library("user32")) - - if os.name == "nt": - windll.kernel32.GetModuleHandleW - windll["kernel32"].GetModuleHandleW - windll.LoadLibrary("kernel32").GetModuleHandleW - WinDLL("kernel32").GetModuleHandleW - # embedded null character - self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_ordinal_functions(self): - import _ctypes_test - dll = WinDLL(_ctypes_test.__file__) - # We load the same function both via ordinal and name - func_ord = dll[2] - func_name = dll.GetString - # addressof gets the address where the function pointer is stored - a_ord = addressof(func_ord) - a_name = addressof(func_name) - f_ord_addr = c_void_p.from_address(a_ord).value - f_name_addr = c_void_p.from_address(a_name).value - self.assertEqual(hex(f_ord_addr), hex(f_name_addr)) - - self.assertRaises(AttributeError, dll.__getitem__, 1234) - - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_A(self): - from _ctypes import LoadLibrary, FreeLibrary - # On winXP 64-bit, advapi32 loads at an address that does - # NOT fit into a 32-bit integer. FreeLibrary must be able - # to accept this address. - - # These are tests for https://www.python.org/sf/1703286 - handle = LoadLibrary("advapi32") - FreeLibrary(handle) - - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_B(self): - # Since on winXP 64-bit advapi32 loads like described - # above, the (arbitrarily selected) CloseEventLog function - # also has a high address. 'call_function' should accept - # addresses so large. - from _ctypes import call_function - advapi32 = windll.advapi32 - # Calling CloseEventLog with a NULL argument should fail, - # but the call should not segfault or so. - self.assertEqual(0, advapi32.CloseEventLog(None)) - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p - proc = windll.kernel32.GetProcAddress(advapi32._handle, - b"CloseEventLog") - self.assertTrue(proc) - # This is the real test: call the function via 'call_function' - self.assertEqual(0, call_function(proc, (None,))) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_hasattr(self): - # bpo-34816: shouldn't raise OSError - self.assertFalse(hasattr(windll, 'test')) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_dll_with_flags(self): - _sqlite3 = import_helper.import_module("_sqlite3") - src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): - ext = "_d.dll" - else: - ext = ".dll" - - with os_helper.temp_dir() as tmp: - # We copy two files and load _sqlite3.dll (formerly .pyd), - # which has a dependency on sqlite3.dll. Then we test - # loading it in subprocesses to avoid it starting in memory - # for each test. - target = os.path.join(tmp, "_sqlite3.dll") - shutil.copy(src, target) - shutil.copy(os.path.join(os.path.dirname(src), "sqlite3" + ext), - os.path.join(tmp, "sqlite3" + ext)) - - def should_pass(command): - with self.subTest(command): - subprocess.check_output( - [sys.executable, "-c", - "from ctypes import *; import nt;" + command], - cwd=tmp - ) - - def should_fail(command): - with self.subTest(command): - with self.assertRaises(subprocess.CalledProcessError): - subprocess.check_output( - [sys.executable, "-c", - "from ctypes import *; import nt;" + command], - cwd=tmp, stderr=subprocess.STDOUT, - ) - - # Default load should not find this in CWD - should_fail("WinDLL('_sqlite3.dll')") - - # Relative path (but not just filename) should succeed - should_pass("WinDLL('./_sqlite3.dll')") - - # Insecure load flags should succeed - # Clear the DLL directory to avoid safe search settings propagating - should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)") - - # Full path load without DLL_LOAD_DIR shouldn't find dependency - should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + - "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32)") - - # Full path load with DLL_LOAD_DIR should succeed - should_pass("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + - "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32|" + - "nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)") - - # User-specified directory should succeed - should_pass("import os; p = os.add_dll_directory(os.getcwd());" + - "WinDLL('_sqlite3.dll'); p.close()") - - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 8d8632a4eb64..15e365ed267d 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -116,6 +116,12 @@ def test_1703286_B(self): # This is the real test: call the function via 'call_function' self.assertEqual(0, call_function(proc, (None,))) + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_hasattr(self): + # bpo-34816: shouldn't raise OSError + self.assertFalse(hasattr(windll, 'test')) + @unittest.skipUnless(os.name == "nt", 'test specific to Windows') def test_load_dll_with_flags(self): From webhook-mailer at python.org Thu Dec 22 08:38:19 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 22 Dec 2022 13:38:19 -0000 Subject: [Python-checkins] gh-100344: Add C implementation for `asyncio.current_task` (#100345) Message-ID: <mailman.3017.1671716301.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4cc63e0d4e4cf3299dcc0ea81616ba072ae5589d commit: 4cc63e0d4e4cf3299dcc0ea81616ba072ae5589d branch: main author: Itamar Ostricher <itamarost at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-22T19:08:12+05:30 summary: gh-100344: Add C implementation for `asyncio.current_task` (#100345) Co-authored-by: pranavtbhat files: A Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst M Doc/whatsnew/3.12.rst M Lib/asyncio/tasks.py M Lib/test/test_asyncio/test_tasks.py M Modules/_asynciomodule.c M Modules/clinic/_asynciomodule.c.h diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 0cc4471364b6..d480be27542c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -225,6 +225,9 @@ asyncio a custom event loop factory. (Contributed by Kumar Aditya in :gh:`99388`.) +* Add C implementation of :func:`asyncio.current_task` for 4x-6x speedup. + (Contributed by Itamar Ostricher and Pranav Thulasiram Bhat in :gh:`100344`.) + inspect ------- diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fa853283c0c5..e78719de216f 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -964,6 +964,7 @@ def _unregister_task(task): _all_tasks.discard(task) +_py_current_task = current_task _py_register_task = _register_task _py_unregister_task = _unregister_task _py_enter_task = _enter_task @@ -973,10 +974,12 @@ def _unregister_task(task): try: from _asyncio import (_register_task, _unregister_task, _enter_task, _leave_task, - _all_tasks, _current_tasks) + _all_tasks, _current_tasks, + current_task) except ImportError: pass else: + _c_current_task = current_task _c_register_task = _register_task _c_unregister_task = _unregister_task _c_enter_task = _enter_task diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 5168b8250ef0..e533d5273e9f 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2804,6 +2804,7 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): class BaseCurrentLoopTests: + current_task = None def setUp(self): super().setUp() @@ -2814,33 +2815,39 @@ def new_task(self, coro): raise NotImplementedError def test_current_task_no_running_loop(self): - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): with self.assertRaisesRegex(RuntimeError, 'no running event loop'): - asyncio.current_task() + self.current_task() def test_current_task_with_implicit_loop(self): async def coro(): - self.assertIs(asyncio.current_task(loop=self.loop), task) + self.assertIs(self.current_task(loop=self.loop), task) - self.assertIs(asyncio.current_task(None), task) - self.assertIs(asyncio.current_task(), task) + self.assertIs(self.current_task(None), task) + self.assertIs(self.current_task(), task) task = self.new_task(coro()) self.loop.run_until_complete(task) - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) class PyCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + current_task = staticmethod(tasks._py_current_task) def new_task(self, coro): return tasks._PyTask(coro, loop=self.loop) - at unittest.skipUnless(hasattr(tasks, '_CTask'), + at unittest.skipUnless(hasattr(tasks, '_CTask') and + hasattr(tasks, '_c_current_task'), 'requires the C _asyncio module') class CCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + if hasattr(tasks, '_c_current_task'): + current_task = staticmethod(tasks._c_current_task) + else: + current_task = None def new_task(self, coro): return getattr(tasks, '_CTask')(coro, loop=self.loop) diff --git a/Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst b/Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst new file mode 100644 index 000000000000..d55f6888dbde --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst @@ -0,0 +1,2 @@ +Provide C implementation for :func:`asyncio.current_task` for a 4x-6x +speedup. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 32be537c00a5..6fe4ca469475 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3314,6 +3314,44 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) } +/*[clinic input] +_asyncio.current_task + + loop: object = None + +Return a currently executed task. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_current_task_impl(PyObject *module, PyObject *loop) +/*[clinic end generated code: output=fe15ac331a7f981a input=58910f61a5627112]*/ +{ + PyObject *ret; + asyncio_state *state = get_asyncio_state(module); + + if (loop == Py_None) { + loop = _asyncio_get_running_loop_impl(module); + if (loop == NULL) { + return NULL; + } + } else { + Py_INCREF(loop); + } + + ret = PyDict_GetItemWithError(state->current_tasks, loop); + Py_DECREF(loop); + if (ret == NULL && PyErr_Occurred()) { + return NULL; + } + else if (ret == NULL) { + Py_RETURN_NONE; + } + Py_INCREF(ret); + return ret; +} + + /*********************** Module **************************/ @@ -3494,6 +3532,7 @@ module_init(asyncio_state *state) PyDoc_STRVAR(module_doc, "Accelerator module for asyncio"); static PyMethodDef asyncio_methods[] = { + _ASYNCIO_CURRENT_TASK_METHODDEF _ASYNCIO_GET_EVENT_LOOP_METHODDEF _ASYNCIO_GET_RUNNING_LOOP_METHODDEF _ASYNCIO__GET_RUNNING_LOOP_METHODDEF diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index f2fbb352c2c6..43c5d7717986 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -1242,4 +1242,64 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=83580c190031241c input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_asyncio_current_task__doc__, +"current_task($module, /, loop=None)\n" +"--\n" +"\n" +"Return a currently executed task."); + +#define _ASYNCIO_CURRENT_TASK_METHODDEF \ + {"current_task", _PyCFunction_CAST(_asyncio_current_task), METH_FASTCALL|METH_KEYWORDS, _asyncio_current_task__doc__}, + +static PyObject * +_asyncio_current_task_impl(PyObject *module, PyObject *loop); + +static PyObject * +_asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(loop), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"loop", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "current_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *loop = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + loop = args[0]; +skip_optional_pos: + return_value = _asyncio_current_task_impl(module, loop); + +exit: + return return_value; +} +/*[clinic end generated code: output=00f494214f2fd008 input=a9049054013a1b77]*/ From webhook-mailer at python.org Thu Dec 22 15:13:57 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 20:13:57 -0000 Subject: [Python-checkins] GH-99554: Trim trailing whitespace (GH-100435) Message-ID: <mailman.3018.1671740038.3313.python-checkins@python.org> https://github.com/python/cpython/commit/09edde95f4841d5dffa584b1c963eb7ceab3f16a commit: 09edde95f4841d5dffa584b1c963eb7ceab3f16a branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T12:13:51-08:00 summary: GH-99554: Trim trailing whitespace (GH-100435) Automerge-Triggered-By: GH:brandtbucher files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 023c13507d6a..cbbdfb9e9467 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -153,7 +153,7 @@ location_is_after(location loc1, location loc2) { static inline bool same_location(location a, location b) { - return a.lineno == b.lineno && + return a.lineno == b.lineno && a.end_lineno == b.end_lineno && a.col_offset == b.col_offset && a.end_col_offset == b.end_col_offset; From webhook-mailer at python.org Thu Dec 22 16:57:23 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Thu, 22 Dec 2022 21:57:23 -0000 Subject: [Python-checkins] gh-85432: Harmonise parameter names between C and pure-Python implementations of `datetime.time.strftime`, `datetime.datetime.fromtimestamp` (#99993) Message-ID: <mailman.3019.1671746244.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9cdb6429971cd8b874ceaeeb04ae2ecdbba42bdb commit: 9cdb6429971cd8b874ceaeeb04ae2ecdbba42bdb branch: main author: Alex Waygood <Alex.Waygood at Gmail.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-22T15:57:18-06:00 summary: gh-85432: Harmonise parameter names between C and pure-Python implementations of `datetime.time.strftime`, `datetime.datetime.fromtimestamp` (#99993) files: A Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst M Lib/datetime.py M Lib/test/datetimetester.py diff --git a/Lib/datetime.py b/Lib/datetime.py index 1b0c5cb2d1c6..68746de1cabf 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1553,8 +1553,7 @@ def fromisoformat(cls, time_string): except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') - - def strftime(self, fmt): + def strftime(self, format): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. """ @@ -1563,7 +1562,7 @@ def strftime(self, fmt): timetuple = (1900, 1, 1, self._hour, self._minute, self._second, 0, 1, -1) - return _wrap_strftime(self, fmt, timetuple) + return _wrap_strftime(self, format, timetuple) def __format__(self, fmt): if not isinstance(fmt, str): @@ -1787,14 +1786,14 @@ def _fromtimestamp(cls, t, utc, tz): return result @classmethod - def fromtimestamp(cls, t, tz=None): + def fromtimestamp(cls, timestamp, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) - return cls._fromtimestamp(t, tz is not None, tz) + return cls._fromtimestamp(timestamp, tz is not None, tz) @classmethod def utcfromtimestamp(cls, t): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 121d973b6d5f..6a1df174a1b9 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2426,6 +2426,12 @@ def test_fromtimestamp(self): got = self.theclass.fromtimestamp(ts) self.verify_field_equality(expected, got) + def test_fromtimestamp_keyword_arg(self): + import time + + # gh-85432: The parameter was named "t" in the pure-Python impl. + self.theclass.fromtimestamp(timestamp=time.time()) + def test_utcfromtimestamp(self): import time @@ -3528,6 +3534,9 @@ def test_strftime(self): except UnicodeEncodeError: pass + # gh-85432: The parameter was named "fmt" in the pure-Python impl. + t.strftime(format="%f") + def test_format(self): t = self.theclass(1, 2, 3, 4) self.assertEqual(t.__format__(''), str(t)) diff --git a/Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst b/Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst new file mode 100644 index 000000000000..68f5d7c942f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst @@ -0,0 +1,5 @@ +Rename the *fmt* parameter of the pure-Python implementation of +:meth:`datetime.time.strftime` to *format*. Rename the *t* parameter of +:meth:`datetime.datetime.fromtimestamp` to *timestamp*. These changes mean +the parameter names in the pure-Python implementation now match the +parameter names in the C implementation. Patch by Alex Waygood. From webhook-mailer at python.org Thu Dec 22 17:35:41 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Thu, 22 Dec 2022 22:35:41 -0000 Subject: [Python-checkins] gh-57762: fix misleading tkinter.Tk docstring (#98837) Message-ID: <mailman.3020.1671748542.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ad23da0e77c4c4a3185df64d1c199b8c17e08188 commit: ad23da0e77c4c4a3185df64d1c199b8c17e08188 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-22T16:35:32-06:00 summary: gh-57762: fix misleading tkinter.Tk docstring (#98837) Mentioned as a desired change by terryjreedy on the corresponding issue, since Tk is not a subclass of Toplevel. files: M Lib/tkinter/__init__.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index a8e7bf490ad4..7565e0f7e460 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2305,7 +2305,7 @@ class Tk(Misc, Wm): def __init__(self, screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None): - """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will be created. BASENAME will be used for the identification of the profile file (see readprofile). It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME From webhook-mailer at python.org Thu Dec 22 18:01:44 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:01:44 -0000 Subject: [Python-checkins] gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Message-ID: <mailman.3021.1671750105.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1e0631e5ba7fcb4882a08d2abdc7614130c07660 commit: 1e0631e5ba7fcb4882a08d2abdc7614130c07660 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:01:38-08:00 summary: gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Mentioned as a desired change by terryjreedy on the corresponding issue, since Tk is not a subclass of Toplevel. (cherry picked from commit ad23da0e77c4c4a3185df64d1c199b8c17e08188) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Lib/tkinter/__init__.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index a8e7bf490ad4..7565e0f7e460 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2305,7 +2305,7 @@ class Tk(Misc, Wm): def __init__(self, screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None): - """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will be created. BASENAME will be used for the identification of the profile file (see readprofile). It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME From webhook-mailer at python.org Thu Dec 22 18:03:43 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:03:43 -0000 Subject: [Python-checkins] gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Message-ID: <mailman.3022.1671750224.3313.python-checkins@python.org> https://github.com/python/cpython/commit/de74d49b21f7671e2cbacf686e63efaee51959f5 commit: de74d49b21f7671e2cbacf686e63efaee51959f5 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:03:38-08:00 summary: gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Mentioned as a desired change by terryjreedy on the corresponding issue, since Tk is not a subclass of Toplevel. (cherry picked from commit ad23da0e77c4c4a3185df64d1c199b8c17e08188) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Lib/tkinter/__init__.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index d42d9a013dd6..7b8dc7bdcd8c 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2278,7 +2278,7 @@ class Tk(Misc, Wm): def __init__(self, screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None): - """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will be created. BASENAME will be used for the identification of the profile file (see readprofile). It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME From webhook-mailer at python.org Thu Dec 22 18:23:30 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Thu, 22 Dec 2022 23:23:30 -0000 Subject: [Python-checkins] gh-48496: Added example and link to faq for UnboundLocalError in reference (#93068) Message-ID: <mailman.3023.1671751411.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f3db68e6e66ebb36e1b9cb30daba913ecc736169 commit: f3db68e6e66ebb36e1b9cb30daba913ecc736169 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-22T17:23:25-06:00 summary: gh-48496: Added example and link to faq for UnboundLocalError in reference (#93068) files: M Doc/faq/programming.rst M Doc/reference/executionmodel.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 584d33e9622e..c396e2b081fc 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -113,6 +113,8 @@ Yes. The coding style required for standard library modules is documented as Core Language ============= +.. _faq-unboundlocalerror: + Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 3f01180e13f7..a264015cbf40 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -128,6 +128,8 @@ lead to errors when a name is used within a block before it is bound. This rule is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. +See :ref:`the FAQ entry on UnboundLocalError <faq-unboundlocalerror>` +for examples. If the :keyword:`global` statement occurs within a block, all uses of the names specified in the statement refer to the bindings of those names in the top-level From webhook-mailer at python.org Thu Dec 22 18:30:15 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:30:15 -0000 Subject: [Python-checkins] gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) Message-ID: <mailman.3024.1671751815.3313.python-checkins@python.org> https://github.com/python/cpython/commit/86cdfaa885eedefe3a33fccc0bbccd0b6a388260 commit: 86cdfaa885eedefe3a33fccc0bbccd0b6a388260 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:30:09-08:00 summary: gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) (cherry picked from commit f3db68e6e66ebb36e1b9cb30daba913ecc736169) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: M Doc/faq/programming.rst M Doc/reference/executionmodel.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index f8140481b120..bd75801cc242 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -113,6 +113,8 @@ Yes. The coding style required for standard library modules is documented as Core Language ============= +.. _faq-unboundlocalerror: + Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index d9183561820b..081f71cf2e7d 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -128,6 +128,8 @@ lead to errors when a name is used within a block before it is bound. This rule is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. +See :ref:`the FAQ entry on UnboundLocalError <faq-unboundlocalerror>` +for examples. If the :keyword:`global` statement occurs within a block, all uses of the names specified in the statement refer to the bindings of those names in the top-level From webhook-mailer at python.org Thu Dec 22 18:30:18 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:30:18 -0000 Subject: [Python-checkins] gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) Message-ID: <mailman.3025.1671751819.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1fa4c6bd1f895df1ca6205a8bd3f60fdd287405c commit: 1fa4c6bd1f895df1ca6205a8bd3f60fdd287405c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:30:12-08:00 summary: gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) (cherry picked from commit f3db68e6e66ebb36e1b9cb30daba913ecc736169) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: M Doc/faq/programming.rst M Doc/reference/executionmodel.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 584d33e9622e..c396e2b081fc 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -113,6 +113,8 @@ Yes. The coding style required for standard library modules is documented as Core Language ============= +.. _faq-unboundlocalerror: + Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 3f01180e13f7..a264015cbf40 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -128,6 +128,8 @@ lead to errors when a name is used within a block before it is bound. This rule is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. +See :ref:`the FAQ entry on UnboundLocalError <faq-unboundlocalerror>` +for examples. If the :keyword:`global` statement occurs within a block, all uses of the names specified in the statement refer to the bindings of those names in the top-level From webhook-mailer at python.org Fri Dec 23 01:22:37 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Fri, 23 Dec 2022 06:22:37 -0000 Subject: [Python-checkins] Fix typo in 3.12 What's New (#100449) Message-ID: <mailman.3026.1671776557.3313.python-checkins@python.org> https://github.com/python/cpython/commit/73c08eeaffadb73184808e462792b6793ce9f82d commit: 73c08eeaffadb73184808e462792b6793ce9f82d branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T00:22:31-06:00 summary: Fix typo in 3.12 What's New (#100449) files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d480be27542c..617fadd83f8f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -939,7 +939,7 @@ Removed internals. (Contributed by Victor Stinner in :gh:`92651`.) -* Leagcy Unicode APIs has been removed. See :pep:`623` for detail. +* Legacy Unicode APIs has been removed. See :pep:`623` for detail. * :c:macro:`PyUnicode_WCHAR_KIND` * :c:func:`PyUnicode_AS_UNICODE` From webhook-mailer at python.org Fri Dec 23 03:23:44 2022 From: webhook-mailer at python.org (mdickinson) Date: Fri, 23 Dec 2022 08:23:44 -0000 Subject: [Python-checkins] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) Message-ID: <mailman.3027.1671783825.3313.python-checkins@python.org> https://github.com/python/cpython/commit/84bc6a4f25fcf467813ee12b74118f7b1b54e285 commit: 84bc6a4f25fcf467813ee12b74118f7b1b54e285 branch: main author: Eric Wieser <wieser.eric at gmail.com> committer: mdickinson <dickinsm at gmail.com> date: 2022-12-23T08:23:19Z summary: gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) The itemsize returned in a memoryview of a ctypes array is now computed from the item type, instead of dividing the total size by the length and assuming that the length is not zero. files: A Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst M Lib/test/test_ctypes/test_pep3118.py M Modules/_ctypes/_ctypes.c diff --git a/Lib/test/test_ctypes/test_pep3118.py b/Lib/test/test_ctypes/test_pep3118.py index 81e8ca7638fd..efffc80a66fc 100644 --- a/Lib/test/test_ctypes/test_pep3118.py +++ b/Lib/test/test_ctypes/test_pep3118.py @@ -176,7 +176,9 @@ class Complete(Structure): ## arrays and pointers (c_double * 4, "<d", (4,), c_double), + (c_double * 0, "<d", (0,), c_double), (c_float * 4 * 3 * 2, "<f", (2,3,4), c_float), + (c_float * 4 * 0 * 2, "<f", (2,0,4), c_float), (POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)), (POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)), (POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)), diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst new file mode 100644 index 000000000000..841740130bd1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst @@ -0,0 +1,3 @@ +``ctypes`` arrays of length 0 now report a correct itemsize when a +``memoryview`` is constructed from them, rather than always giving a value +of 0. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b9092d3981f3..f69a37709963 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2731,11 +2731,33 @@ static PyMemberDef PyCData_members[] = { { NULL }, }; -static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) +/* Find the innermost type of an array type, returning a borrowed reference */ +static PyObject * +PyCData_item_type(PyObject *type) +{ + if (PyCArrayTypeObject_Check(type)) { + StgDictObject *stg_dict; + PyObject *elem_type; + + /* asserts used here as these are all guaranteed by construction */ + stg_dict = PyType_stgdict(type); + assert(stg_dict); + elem_type = stg_dict->proto; + assert(elem_type); + return PyCData_item_type(elem_type); + } + else { + return type; + } +} + +static int +PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; StgDictObject *dict = PyObject_stgdict(myself); - Py_ssize_t i; + PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); + StgDictObject *item_dict = PyType_stgdict(item_type); if (view == NULL) return 0; @@ -2747,12 +2769,7 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) view->format = dict->format ? dict->format : "B"; view->ndim = dict->ndim; view->shape = dict->shape; - view->itemsize = self->b_size; - if (view->itemsize) { - for (i = 0; i < view->ndim; ++i) { - view->itemsize /= dict->shape[i]; - } - } + view->itemsize = item_dict->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; From webhook-mailer at python.org Fri Dec 23 03:56:01 2022 From: webhook-mailer at python.org (mdickinson) Date: Fri, 23 Dec 2022 08:56:01 -0000 Subject: [Python-checkins] [3.10] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (#100451) Message-ID: <mailman.3028.1671785761.3313.python-checkins@python.org> https://github.com/python/cpython/commit/95c55a69b3935e36108059ea89efd372d0902c6e commit: 95c55a69b3935e36108059ea89efd372d0902c6e branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: mdickinson <dickinsm at gmail.com> date: 2022-12-23T08:55:55Z summary: [3.10] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (#100451) gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) The itemsize returned in a memoryview of a ctypes array is now computed from the item type, instead of dividing the total size by the length and assuming that the length is not zero. (cherry picked from commit 84bc6a4f25fcf467813ee12b74118f7b1b54e285) Co-authored-by: Eric Wieser <wieser.eric at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst M Lib/ctypes/test/test_pep3118.py M Modules/_ctypes/_ctypes.c diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py index 81e8ca7638fd..efffc80a66fc 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/ctypes/test/test_pep3118.py @@ -176,7 +176,9 @@ class Complete(Structure): ## arrays and pointers (c_double * 4, "<d", (4,), c_double), + (c_double * 0, "<d", (0,), c_double), (c_float * 4 * 3 * 2, "<f", (2,3,4), c_float), + (c_float * 4 * 0 * 2, "<f", (2,0,4), c_float), (POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)), (POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)), (POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)), diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst new file mode 100644 index 000000000000..841740130bd1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst @@ -0,0 +1,3 @@ +``ctypes`` arrays of length 0 now report a correct itemsize when a +``memoryview`` is constructed from them, rather than always giving a value +of 0. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 84378c40357b..a534a828d1ee 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2795,11 +2795,33 @@ static PyMemberDef PyCData_members[] = { { NULL }, }; -static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) +/* Find the innermost type of an array type, returning a borrowed reference */ +static PyObject * +PyCData_item_type(PyObject *type) +{ + if (PyCArrayTypeObject_Check(type)) { + StgDictObject *stg_dict; + PyObject *elem_type; + + /* asserts used here as these are all guaranteed by construction */ + stg_dict = PyType_stgdict(type); + assert(stg_dict); + elem_type = stg_dict->proto; + assert(elem_type); + return PyCData_item_type(elem_type); + } + else { + return type; + } +} + +static int +PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; StgDictObject *dict = PyObject_stgdict(myself); - Py_ssize_t i; + PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); + StgDictObject *item_dict = PyType_stgdict(item_type); if (view == NULL) return 0; @@ -2812,12 +2834,7 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) view->format = dict->format ? dict->format : "B"; view->ndim = dict->ndim; view->shape = dict->shape; - view->itemsize = self->b_size; - if (view->itemsize) { - for (i = 0; i < view->ndim; ++i) { - view->itemsize /= dict->shape[i]; - } - } + view->itemsize = item_dict->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; From webhook-mailer at python.org Fri Dec 23 03:56:26 2022 From: webhook-mailer at python.org (mdickinson) Date: Fri, 23 Dec 2022 08:56:26 -0000 Subject: [Python-checkins] [3.11] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (GH-100452) Message-ID: <mailman.3029.1671785786.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f6fe4bb75cb1c7423cfd08d0bf10f9843761c2ac commit: f6fe4bb75cb1c7423cfd08d0bf10f9843761c2ac branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: mdickinson <dickinsm at gmail.com> date: 2022-12-23T08:56:20Z summary: [3.11] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (GH-100452) gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) The itemsize returned in a memoryview of a ctypes array is now computed from the item type, instead of dividing the total size by the length and assuming that the length is not zero. (cherry picked from commit 84bc6a4f25fcf467813ee12b74118f7b1b54e285) Co-authored-by: Eric Wieser <wieser.eric at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst M Lib/ctypes/test/test_pep3118.py M Modules/_ctypes/_ctypes.c diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py index 81e8ca7638fd..efffc80a66fc 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/ctypes/test/test_pep3118.py @@ -176,7 +176,9 @@ class Complete(Structure): ## arrays and pointers (c_double * 4, "<d", (4,), c_double), + (c_double * 0, "<d", (0,), c_double), (c_float * 4 * 3 * 2, "<f", (2,3,4), c_float), + (c_float * 4 * 0 * 2, "<f", (2,0,4), c_float), (POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)), (POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)), (POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)), diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst new file mode 100644 index 000000000000..841740130bd1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst @@ -0,0 +1,3 @@ +``ctypes`` arrays of length 0 now report a correct itemsize when a +``memoryview`` is constructed from them, rather than always giving a value +of 0. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d6fa11d34813..b5eafe49e4ee 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2775,11 +2775,33 @@ static PyMemberDef PyCData_members[] = { { NULL }, }; -static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) +/* Find the innermost type of an array type, returning a borrowed reference */ +static PyObject * +PyCData_item_type(PyObject *type) +{ + if (PyCArrayTypeObject_Check(type)) { + StgDictObject *stg_dict; + PyObject *elem_type; + + /* asserts used here as these are all guaranteed by construction */ + stg_dict = PyType_stgdict(type); + assert(stg_dict); + elem_type = stg_dict->proto; + assert(elem_type); + return PyCData_item_type(elem_type); + } + else { + return type; + } +} + +static int +PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; StgDictObject *dict = PyObject_stgdict(myself); - Py_ssize_t i; + PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); + StgDictObject *item_dict = PyType_stgdict(item_type); if (view == NULL) return 0; @@ -2792,12 +2814,7 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) view->format = dict->format ? dict->format : "B"; view->ndim = dict->ndim; view->shape = dict->shape; - view->itemsize = self->b_size; - if (view->itemsize) { - for (i = 0; i < view->ndim; ++i) { - view->itemsize /= dict->shape[i]; - } - } + view->itemsize = item_dict->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; From webhook-mailer at python.org Fri Dec 23 09:42:30 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 23 Dec 2022 14:42:30 -0000 Subject: [Python-checkins] GH-100459: fix copy-paste errors in specialization stats (GH-100460) Message-ID: <mailman.3030.1671806551.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2659036c757a11235c4abd21f02c3a548a344fe7 commit: 2659036c757a11235c4abd21f02c3a548a344fe7 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: markshannon <mark at hotpy.org> date: 2022-12-23T14:42:24Z summary: GH-100459: fix copy-paste errors in specialization stats (GH-100460) files: M Python/specialize.c diff --git a/Python/specialize.c b/Python/specialize.c index 69a7bf98cad0..d9af7b742d54 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -860,7 +860,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) // We *might* not really need this check, but we inherited it from // PyObject_GenericSetAttr and friends... and this way we still do the // right thing if someone forgets to call PyType_Ready(type): - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER); goto fail; } if (PyModule_CheckExact(owner)) { @@ -915,16 +915,16 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; case BUILTIN_CLASSMETHOD: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); goto fail; case PYTHON_CLASSMETHOD: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); goto fail; case NON_OVERRIDING: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR); goto fail; case NON_DESCRIPTOR: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); goto fail; case ABSENT: if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, From webhook-mailer at python.org Fri Dec 23 09:45:59 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 23 Dec 2022 14:45:59 -0000 Subject: [Python-checkins] gh-99110: Initialize `frame->previous` in init_frame to fix segmentation fault when accessing `frame.f_back` (#100182) Message-ID: <mailman.3031.1671806761.3313.python-checkins@python.org> https://github.com/python/cpython/commit/88d565f32a709140664444c6dea20ecd35a10e94 commit: 88d565f32a709140664444c6dea20ecd35a10e94 branch: main author: Bill Fisher <william.w.fisher at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-23T20:15:53+05:30 summary: gh-99110: Initialize `frame->previous` in init_frame to fix segmentation fault when accessing `frame.f_back` (#100182) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst M Include/internal/pycore_frame.h M Lib/test/test_frame.py M Modules/_testcapimodule.c M Objects/frameobject.c diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 7fa410d288c3..f18723b30322 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -96,7 +96,10 @@ static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); -/* Consumes reference to func and locals */ +/* Consumes reference to func and locals. + Does not initialize frame->previous, which happens + when frame is linked into the frame stack. + */ static inline void _PyFrame_InitializeSpecials( _PyInterpreterFrame *frame, PyFunctionObject *func, diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index ed413f105e5b..40c734b6e33a 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -408,6 +408,15 @@ def test_frame_get_generator(self): frame = next(gen) self.assertIs(gen, _testcapi.frame_getgenerator(frame)) + def test_frame_fback_api(self): + """Test that accessing `f_back` does not cause a segmentation fault on + a frame created with `PyFrame_New` (GH-99110).""" + def dummy(): + pass + + frame = _testcapi.frame_new(dummy.__code__, globals(), locals()) + # The following line should not cause a segmentation fault. + self.assertIsNone(frame.f_back) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst new file mode 100644 index 000000000000..175740dfca07 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst @@ -0,0 +1,2 @@ +Initialize frame->previous in frameobject.c to fix a segmentation fault when +accessing frames created by :c:func:`PyFrame_New`. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 35c895d9ceb2..c32fdb5f5fbe 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -20,6 +20,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "frameobject.h" // PyFrame_New #include "marshal.h" // PyMarshal_WriteLongToFile #include "structmember.h" // for offsetof(), T_OBJECT #include <float.h> // FLT_MAX @@ -2839,6 +2840,22 @@ frame_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(lasti); } +static PyObject * +frame_new(PyObject *self, PyObject *args) +{ + PyObject *code, *globals, *locals; + if (!PyArg_ParseTuple(args, "OOO", &code, &globals, &locals)) { + return NULL; + } + if (!PyCode_Check(code)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); + return NULL; + } + PyThreadState *tstate = PyThreadState_Get(); + + return (PyObject *)PyFrame_New(tstate, (PyCodeObject *)code, globals, locals); +} + static PyObject * test_frame_getvar(PyObject *self, PyObject *args) { @@ -3277,6 +3294,7 @@ static PyMethodDef TestMethods[] = { {"frame_getgenerator", frame_getgenerator, METH_O, NULL}, {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL}, {"frame_getlasti", frame_getlasti, METH_O, NULL}, + {"frame_new", frame_new, METH_VARARGS, NULL}, {"frame_getvar", test_frame_getvar, METH_VARARGS, NULL}, {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL}, {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 74c26d8d4d96..eab85c08fc01 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1013,6 +1013,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) PyCodeObject *code = (PyCodeObject *)func->func_code; _PyFrame_InitializeSpecials(frame, (PyFunctionObject*)Py_NewRef(func), Py_XNewRef(locals), code); + frame->previous = NULL; for (Py_ssize_t i = 0; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = NULL; } From webhook-mailer at python.org Fri Dec 23 10:00:33 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Fri, 23 Dec 2022 15:00:33 -0000 Subject: [Python-checkins] gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (#98710) Message-ID: <mailman.3032.1671807634.3313.python-checkins@python.org> https://github.com/python/cpython/commit/49f6ff719c4e0beeafd6c42edd696601acf72764 commit: 49f6ff719c4e0beeafd6c42edd696601acf72764 branch: main author: Petr Viktorin <encukou at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-23T07:00:21-08:00 summary: gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (#98710) files: M Doc/c-api/arg.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index c5be453c1533..9713431688d4 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -34,24 +34,39 @@ These formats allow accessing an object as a contiguous chunk of memory. You don't have to provide raw storage for the returned unicode or bytes area. -In general, when a format sets a pointer to a buffer, the buffer is -managed by the corresponding Python object, and the buffer shares -the lifetime of this object. You won't have to release any memory yourself. -The only exceptions are ``es``, ``es#``, ``et`` and ``et#``. - -However, when a :c:type:`Py_buffer` structure gets filled, the underlying -buffer is locked so that the caller can subsequently use the buffer even -inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data -being resized or destroyed. As a result, **you have to call** -:c:func:`PyBuffer_Release` after you have finished processing the data (or -in any early abort case). - Unless otherwise stated, buffers are not NUL-terminated. -Some formats require a read-only :term:`bytes-like object`, and set a -pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, -which disallows mutable objects such as :class:`bytearray`. +There are three ways strings and buffers can be converted to C: + +* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure. + This locks the underlying buffer so that the caller can subsequently use + the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS` + block without the risk of mutable data being resized or destroyed. + As a result, **you have to call** :c:func:`PyBuffer_Release` after you have + finished processing the data (or in any early abort case). + +* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer. + **You have to call** :c:func:`PyMem_Free` after you have finished + processing the data (or in any early abort case). + +* .. _c-arg-borrowed-buffer: + + Other formats take a :class:`str` or a read-only :term:`bytes-like object`, + such as :class:`bytes`, and provide a ``const char *`` pointer to + its buffer. + In this case the buffer is "borrowed": it is managed by the corresponding + Python object, and shares the lifetime of this object. + You won't have to release any memory yourself. + + To ensure that the underlying buffer may be safely borrowed, the object's + :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``. + This disallows common mutable objects such as :class:`bytearray`, + but also some read-only objects such as :class:`memoryview` of + :class:`bytes`. + + Besides this ``bf_releasebuffer`` requirement, there is no check to verify + whether the input object is immutable (e.g. whether it would honor a request + for a writable buffer, or whether another thread can mutate the data). .. note:: @@ -89,7 +104,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode objects are converted to C strings using ``'utf-8'`` encoding. ``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable objects. + Like ``s*``, except that it provides a :ref:`borrowed buffer <c-arg-borrowed-buffer>`. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. The string may contain embedded null bytes. Unicode objects are converted @@ -108,8 +123,9 @@ which disallows mutable objects such as :class:`bytearray`. pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a character - string; it does not accept Unicode objects. The bytes buffer must not + This format converts a bytes-like object to a C pointer to a + :ref:`borrowed <c-arg-borrowed-buffer>` character string; + it does not accept Unicode objects. The bytes buffer must not contain embedded null bytes; if it does, a :exc:`ValueError` exception is raised. From webhook-mailer at python.org Fri Dec 23 10:08:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 15:08:17 -0000 Subject: [Python-checkins] gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) Message-ID: <mailman.3033.1671808098.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4b3b6423c3667d8c9350fb920a5a69824c5250ff commit: 4b3b6423c3667d8c9350fb920a5a69824c5250ff branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T07:08:11-08:00 summary: gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) (cherry picked from commit 49f6ff719c4e0beeafd6c42edd696601acf72764) Co-authored-by: Petr Viktorin <encukou at gmail.com> files: M Doc/c-api/arg.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 85f9eda17a20..6a53c79bd3be 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -34,24 +34,39 @@ These formats allow accessing an object as a contiguous chunk of memory. You don't have to provide raw storage for the returned unicode or bytes area. -In general, when a format sets a pointer to a buffer, the buffer is -managed by the corresponding Python object, and the buffer shares -the lifetime of this object. You won't have to release any memory yourself. -The only exceptions are ``es``, ``es#``, ``et`` and ``et#``. - -However, when a :c:type:`Py_buffer` structure gets filled, the underlying -buffer is locked so that the caller can subsequently use the buffer even -inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data -being resized or destroyed. As a result, **you have to call** -:c:func:`PyBuffer_Release` after you have finished processing the data (or -in any early abort case). - Unless otherwise stated, buffers are not NUL-terminated. -Some formats require a read-only :term:`bytes-like object`, and set a -pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, -which disallows mutable objects such as :class:`bytearray`. +There are three ways strings and buffers can be converted to C: + +* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure. + This locks the underlying buffer so that the caller can subsequently use + the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS` + block without the risk of mutable data being resized or destroyed. + As a result, **you have to call** :c:func:`PyBuffer_Release` after you have + finished processing the data (or in any early abort case). + +* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer. + **You have to call** :c:func:`PyMem_Free` after you have finished + processing the data (or in any early abort case). + +* .. _c-arg-borrowed-buffer: + + Other formats take a :class:`str` or a read-only :term:`bytes-like object`, + such as :class:`bytes`, and provide a ``const char *`` pointer to + its buffer. + In this case the buffer is "borrowed": it is managed by the corresponding + Python object, and shares the lifetime of this object. + You won't have to release any memory yourself. + + To ensure that the underlying buffer may be safely borrowed, the object's + :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``. + This disallows common mutable objects such as :class:`bytearray`, + but also some read-only objects such as :class:`memoryview` of + :class:`bytes`. + + Besides this ``bf_releasebuffer`` requirement, there is no check to verify + whether the input object is immutable (e.g. whether it would honor a request + for a writable buffer, or whether another thread can mutate the data). .. note:: @@ -89,7 +104,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode objects are converted to C strings using ``'utf-8'`` encoding. ``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable objects. + Like ``s*``, except that it provides a :ref:`borrowed buffer <c-arg-borrowed-buffer>`. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. The string may contain embedded null bytes. Unicode objects are converted @@ -108,8 +123,9 @@ which disallows mutable objects such as :class:`bytearray`. pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a character - string; it does not accept Unicode objects. The bytes buffer must not + This format converts a bytes-like object to a C pointer to a + :ref:`borrowed <c-arg-borrowed-buffer>` character string; + it does not accept Unicode objects. The bytes buffer must not contain embedded null bytes; if it does, a :exc:`ValueError` exception is raised. From webhook-mailer at python.org Fri Dec 23 10:09:40 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 15:09:40 -0000 Subject: [Python-checkins] gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) Message-ID: <mailman.3034.1671808181.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bd472198c6ab7d69657aa3d70df1320584cf98b2 commit: bd472198c6ab7d69657aa3d70df1320584cf98b2 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T07:09:34-08:00 summary: gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) (cherry picked from commit 49f6ff719c4e0beeafd6c42edd696601acf72764) Co-authored-by: Petr Viktorin <encukou at gmail.com> files: M Doc/c-api/arg.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 85f9eda17a20..6a53c79bd3be 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -34,24 +34,39 @@ These formats allow accessing an object as a contiguous chunk of memory. You don't have to provide raw storage for the returned unicode or bytes area. -In general, when a format sets a pointer to a buffer, the buffer is -managed by the corresponding Python object, and the buffer shares -the lifetime of this object. You won't have to release any memory yourself. -The only exceptions are ``es``, ``es#``, ``et`` and ``et#``. - -However, when a :c:type:`Py_buffer` structure gets filled, the underlying -buffer is locked so that the caller can subsequently use the buffer even -inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data -being resized or destroyed. As a result, **you have to call** -:c:func:`PyBuffer_Release` after you have finished processing the data (or -in any early abort case). - Unless otherwise stated, buffers are not NUL-terminated. -Some formats require a read-only :term:`bytes-like object`, and set a -pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, -which disallows mutable objects such as :class:`bytearray`. +There are three ways strings and buffers can be converted to C: + +* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure. + This locks the underlying buffer so that the caller can subsequently use + the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS` + block without the risk of mutable data being resized or destroyed. + As a result, **you have to call** :c:func:`PyBuffer_Release` after you have + finished processing the data (or in any early abort case). + +* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer. + **You have to call** :c:func:`PyMem_Free` after you have finished + processing the data (or in any early abort case). + +* .. _c-arg-borrowed-buffer: + + Other formats take a :class:`str` or a read-only :term:`bytes-like object`, + such as :class:`bytes`, and provide a ``const char *`` pointer to + its buffer. + In this case the buffer is "borrowed": it is managed by the corresponding + Python object, and shares the lifetime of this object. + You won't have to release any memory yourself. + + To ensure that the underlying buffer may be safely borrowed, the object's + :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``. + This disallows common mutable objects such as :class:`bytearray`, + but also some read-only objects such as :class:`memoryview` of + :class:`bytes`. + + Besides this ``bf_releasebuffer`` requirement, there is no check to verify + whether the input object is immutable (e.g. whether it would honor a request + for a writable buffer, or whether another thread can mutate the data). .. note:: @@ -89,7 +104,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode objects are converted to C strings using ``'utf-8'`` encoding. ``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable objects. + Like ``s*``, except that it provides a :ref:`borrowed buffer <c-arg-borrowed-buffer>`. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. The string may contain embedded null bytes. Unicode objects are converted @@ -108,8 +123,9 @@ which disallows mutable objects such as :class:`bytearray`. pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a character - string; it does not accept Unicode objects. The bytes buffer must not + This format converts a bytes-like object to a C pointer to a + :ref:`borrowed <c-arg-borrowed-buffer>` character string; + it does not accept Unicode objects. The bytes buffer must not contain embedded null bytes; if it does, a :exc:`ValueError` exception is raised. From webhook-mailer at python.org Fri Dec 23 10:23:41 2022 From: webhook-mailer at python.org (Fidget-Spinner) Date: Fri, 23 Dec 2022 15:23:41 -0000 Subject: [Python-checkins] gh-92216: improve performance of `hasattr` for type objects (GH-99979) Message-ID: <mailman.3035.1671809022.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7fc7909677759fd0cb9d30cd7ff839afd3973325 commit: 7fc7909677759fd0cb9d30cd7ff839afd3973325 branch: main author: Pieter Eendebak <pieter.eendebak at gmail.com> committer: Fidget-Spinner <kenjin at python.org> date: 2022-12-23T23:23:36+08:00 summary: gh-92216: improve performance of `hasattr` for type objects (GH-99979) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst M Include/internal/pycore_typeobject.h M Objects/object.c M Objects/typeobject.c diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c207ce615c6f..4d705740a9a6 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -74,6 +74,10 @@ extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *); extern void _PyStaticType_ClearWeakRefs(PyTypeObject *type); extern void _PyStaticType_Dealloc(PyTypeObject *type); +PyObject * +_Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); +PyObject * +_Py_type_getattro(PyTypeObject *type, PyObject *name); PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst new file mode 100644 index 000000000000..f7ef52d97c27 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst @@ -0,0 +1 @@ +Improve the performance of :func:`hasattr` for type objects with a missing attribute. diff --git a/Objects/object.c b/Objects/object.c index 028b0edc9111..fae508cae3d6 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -939,7 +939,15 @@ _PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result) } return 0; } - if (tp->tp_getattro != NULL) { + if (tp->tp_getattro == (getattrofunc)_Py_type_getattro) { + int supress_missing_attribute_exception = 0; + *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); + if (supress_missing_attribute_exception) { + // return 0 without having to clear the exception + return 0; + } + } + else if (tp->tp_getattro != NULL) { *result = (*tp->tp_getattro)(v, name); } else if (tp->tp_getattr != NULL) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a96f993e99dd..16b1a3035d56 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4219,9 +4219,19 @@ _PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ -static PyObject * -type_getattro(PyTypeObject *type, PyObject *name) + but uses _PyType_Lookup() instead of just looking in type->tp_dict. + + The argument suppress_missing_attribute is used to provide a + fast path for hasattr. The possible values are: + + * NULL: do not suppress the exception + * Non-zero pointer: suppress the PyExc_AttributeError and + set *suppress_missing_attribute to 1 to signal we are returning NULL while + having suppressed the exception (other exceptions are not suppressed) + + */ +PyObject * +_Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missing_attribute) { PyTypeObject *metatype = Py_TYPE(type); PyObject *meta_attribute, *attribute; @@ -4301,12 +4311,25 @@ type_getattro(PyTypeObject *type, PyObject *name) } /* Give up */ - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - type->tp_name, name); + if (suppress_missing_attribute == NULL) { + PyErr_Format(PyExc_AttributeError, + "type object '%.50s' has no attribute '%U'", + type->tp_name, name); + } else { + // signal the caller we have not set an PyExc_AttributeError and gave up + *suppress_missing_attribute = 1; + } return NULL; } +/* This is similar to PyObject_GenericGetAttr(), + but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ +PyObject * +_Py_type_getattro(PyTypeObject *type, PyObject *name) +{ + return _Py_type_getattro_impl(type, name, NULL); +} + static int type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) { @@ -4798,7 +4821,7 @@ PyTypeObject PyType_Type = { 0, /* tp_hash */ (ternaryfunc)type_call, /* tp_call */ 0, /* tp_str */ - (getattrofunc)type_getattro, /* tp_getattro */ + (getattrofunc)_Py_type_getattro, /* tp_getattro */ (setattrofunc)type_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | From webhook-mailer at python.org Fri Dec 23 11:26:48 2022 From: webhook-mailer at python.org (Fidget-Spinner) Date: Fri, 23 Dec 2022 16:26:48 -0000 Subject: [Python-checkins] gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289) Message-ID: <mailman.3036.1671812808.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c3c7848a48b74a321632202e4bdcf2f465fb1cc6 commit: c3c7848a48b74a321632202e4bdcf2f465fb1cc6 branch: main author: Ken Jin <kenjin at python.org> committer: Fidget-Spinner <kenjin at python.org> date: 2022-12-24T00:26:42+08:00 summary: gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst M Include/internal/pycore_opcode.h M Include/opcode.h M Lib/opcode.py M Python/bytecodes.c M Python/generated_cases.c.h M Python/opcode_targets.h M Python/specialize.c diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index da8a272f2fa2..d702580d4980 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -152,6 +152,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_MANAGED_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, @@ -313,12 +314,12 @@ static const char *const _PyOpcode_OpName[263] = { [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_MANAGED_DICT] = "LOAD_ATTR_METHOD_MANAGED_DICT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -345,7 +346,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -353,7 +354,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -373,9 +374,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -385,27 +386,27 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", - [173] = "<173>", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [174] = "<174>", [175] = "<175>", [176] = "<176>", @@ -499,7 +500,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 173: \ case 174: \ case 175: \ case 176: \ diff --git a/Include/opcode.h b/Include/opcode.h index 888250ed37e8..088c1a5d8ca5 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -174,23 +174,24 @@ extern "C" { #define LOAD_ATTR_WITH_HINT 78 #define LOAD_ATTR_METHOD_LAZY_DICT 79 #define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_WITH_DICT 81 -#define LOAD_ATTR_METHOD_WITH_VALUES 86 -#define LOAD_CONST__LOAD_FAST 113 -#define LOAD_FAST__LOAD_CONST 121 -#define LOAD_FAST__LOAD_FAST 141 -#define LOAD_GLOBAL_BUILTIN 143 -#define LOAD_GLOBAL_MODULE 153 -#define STORE_ATTR_INSTANCE_VALUE 154 -#define STORE_ATTR_SLOT 158 -#define STORE_ATTR_WITH_HINT 159 -#define STORE_FAST__LOAD_FAST 160 -#define STORE_FAST__STORE_FAST 161 -#define STORE_SUBSCR_DICT 166 -#define STORE_SUBSCR_LIST_INT 167 -#define UNPACK_SEQUENCE_LIST 168 -#define UNPACK_SEQUENCE_TUPLE 169 -#define UNPACK_SEQUENCE_TWO_TUPLE 170 +#define LOAD_ATTR_METHOD_MANAGED_DICT 81 +#define LOAD_ATTR_METHOD_WITH_DICT 86 +#define LOAD_ATTR_METHOD_WITH_VALUES 113 +#define LOAD_CONST__LOAD_FAST 121 +#define LOAD_FAST__LOAD_CONST 141 +#define LOAD_FAST__LOAD_FAST 143 +#define LOAD_GLOBAL_BUILTIN 153 +#define LOAD_GLOBAL_MODULE 154 +#define STORE_ATTR_INSTANCE_VALUE 158 +#define STORE_ATTR_SLOT 159 +#define STORE_ATTR_WITH_HINT 160 +#define STORE_FAST__LOAD_FAST 161 +#define STORE_FAST__STORE_FAST 166 +#define STORE_SUBSCR_DICT 167 +#define STORE_SUBSCR_LIST_INT 168 +#define UNPACK_SEQUENCE_LIST 169 +#define UNPACK_SEQUENCE_TUPLE 170 +#define UNPACK_SEQUENCE_TWO_TUPLE 173 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index fc57affbac58..59f368c0d245 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -336,6 +336,7 @@ def pseudo_op(name, op, real_ops): # These will always push [unbound method, self] onto the stack. "LOAD_ATTR_METHOD_LAZY_DICT", "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_MANAGED_DICT", "LOAD_ATTR_METHOD_WITH_DICT", "LOAD_ATTR_METHOD_WITH_VALUES", ], diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst new file mode 100644 index 000000000000..45182108b1a4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst @@ -0,0 +1 @@ +Specialize method loading for objects with ``Py_TPFLAGS_MANAGED_DICT`` set. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c0b625bd662c..c2f6a7add53a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2717,6 +2717,31 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } + // error: LOAD_ATTR has irregular stack effect + inst(LOAD_ATTR_METHOD_MANAGED_DICT) { + assert(cframe.use_tracing == 0); + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + uint32_t type_version = read_u32(cache->type_version); + assert(type_version != 0); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) + assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyObject *dict = _PyDictOrValues_GetDict(dorv); + PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; + // Note: cache->keys_version can be 0 when dict is NULL. + DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + SET_TOP(Py_NewRef(res)); + PUSH(self); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 42b7ca086705..0012ba82e948 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2964,6 +2964,31 @@ DISPATCH(); } + TARGET(LOAD_ATTR_METHOD_MANAGED_DICT) { + assert(cframe.use_tracing == 0); + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + uint32_t type_version = read_u32(cache->type_version); + assert(type_version != 0); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) + assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyObject *dict = _PyDictOrValues_GetDict(dorv); + PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; + // Note: cache->keys_version can be 0 when dict is NULL. + DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + SET_TOP(Py_NewRef(res)); + PUSH(self); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + DISPATCH(); + } + TARGET(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ assert(cframe.use_tracing == 0); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index be3ad01c151c..67f5bacecc8c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -80,12 +80,12 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_MANAGED_DICT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,27 +152,27 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&_unknown_opcode, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index d9af7b742d54..53215748c99b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -335,7 +335,6 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 -#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -1036,11 +1035,14 @@ PyObject *descr, DescriptorClassification kind) PyDictKeysObject *keys; if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; if (_PyDictOrValues_IsValues(dorv)) { + keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; dictkind = MANAGED_VALUES; } else { + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + keys = dict != NULL ? dict->ma_keys : NULL; + // User has directly accessed __dict__. dictkind = MANAGED_DICT; } } @@ -1067,7 +1069,7 @@ PyObject *descr, DescriptorClassification kind) } } } - if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT) { + if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT || (dictkind == MANAGED_DICT && keys != NULL)) { Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != DKIX_EMPTY) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); @@ -1088,8 +1090,11 @@ PyObject *descr, DescriptorClassification kind) _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); - goto fail; + if (keys == NULL) { + write_u32(cache->keys_version, 0); + } + _py_set_opcode(instr, LOAD_ATTR_METHOD_MANAGED_DICT); + break; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_DICT); From webhook-mailer at python.org Fri Dec 23 12:48:52 2022 From: webhook-mailer at python.org (Fidget-Spinner) Date: Fri, 23 Dec 2022 17:48:52 -0000 Subject: [Python-checkins] Revert "gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289)" (#100468) Message-ID: <mailman.3037.1671817733.3313.python-checkins@python.org> https://github.com/python/cpython/commit/36d358348de8efad75ebcf55dad8ed4a4f6dcda9 commit: 36d358348de8efad75ebcf55dad8ed4a4f6dcda9 branch: main author: Ken Jin <kenjin at python.org> committer: Fidget-Spinner <kenjin at python.org> date: 2022-12-24T01:48:43+08:00 summary: Revert "gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289)" (#100468) This reverts commit c3c7848a48b74a321632202e4bdcf2f465fb1cc6. files: D Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst M Include/internal/pycore_opcode.h M Include/opcode.h M Lib/opcode.py M Python/bytecodes.c M Python/generated_cases.c.h M Python/opcode_targets.h M Python/specialize.c diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index d702580d4980..da8a272f2fa2 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -152,7 +152,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_MANAGED_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, @@ -314,12 +313,12 @@ static const char *const _PyOpcode_OpName[263] = { [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_MANAGED_DICT] = "LOAD_ATTR_METHOD_MANAGED_DICT", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -346,7 +345,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -354,7 +353,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -374,9 +373,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -386,27 +385,27 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [173] = "<173>", [174] = "<174>", [175] = "<175>", [176] = "<176>", @@ -500,6 +499,7 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 173: \ case 174: \ case 175: \ case 176: \ diff --git a/Include/opcode.h b/Include/opcode.h index 088c1a5d8ca5..888250ed37e8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -174,24 +174,23 @@ extern "C" { #define LOAD_ATTR_WITH_HINT 78 #define LOAD_ATTR_METHOD_LAZY_DICT 79 #define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_MANAGED_DICT 81 -#define LOAD_ATTR_METHOD_WITH_DICT 86 -#define LOAD_ATTR_METHOD_WITH_VALUES 113 -#define LOAD_CONST__LOAD_FAST 121 -#define LOAD_FAST__LOAD_CONST 141 -#define LOAD_FAST__LOAD_FAST 143 -#define LOAD_GLOBAL_BUILTIN 153 -#define LOAD_GLOBAL_MODULE 154 -#define STORE_ATTR_INSTANCE_VALUE 158 -#define STORE_ATTR_SLOT 159 -#define STORE_ATTR_WITH_HINT 160 -#define STORE_FAST__LOAD_FAST 161 -#define STORE_FAST__STORE_FAST 166 -#define STORE_SUBSCR_DICT 167 -#define STORE_SUBSCR_LIST_INT 168 -#define UNPACK_SEQUENCE_LIST 169 -#define UNPACK_SEQUENCE_TUPLE 170 -#define UNPACK_SEQUENCE_TWO_TUPLE 173 +#define LOAD_ATTR_METHOD_WITH_DICT 81 +#define LOAD_ATTR_METHOD_WITH_VALUES 86 +#define LOAD_CONST__LOAD_FAST 113 +#define LOAD_FAST__LOAD_CONST 121 +#define LOAD_FAST__LOAD_FAST 141 +#define LOAD_GLOBAL_BUILTIN 143 +#define LOAD_GLOBAL_MODULE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 160 +#define STORE_FAST__STORE_FAST 161 +#define STORE_SUBSCR_DICT 166 +#define STORE_SUBSCR_LIST_INT 167 +#define UNPACK_SEQUENCE_LIST 168 +#define UNPACK_SEQUENCE_TUPLE 169 +#define UNPACK_SEQUENCE_TWO_TUPLE 170 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 59f368c0d245..fc57affbac58 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -336,7 +336,6 @@ def pseudo_op(name, op, real_ops): # These will always push [unbound method, self] onto the stack. "LOAD_ATTR_METHOD_LAZY_DICT", "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_MANAGED_DICT", "LOAD_ATTR_METHOD_WITH_DICT", "LOAD_ATTR_METHOD_WITH_VALUES", ], diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst deleted file mode 100644 index 45182108b1a4..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst +++ /dev/null @@ -1 +0,0 @@ -Specialize method loading for objects with ``Py_TPFLAGS_MANAGED_DICT`` set. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c2f6a7add53a..c0b625bd662c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2717,31 +2717,6 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_METHOD_MANAGED_DICT) { - assert(cframe.use_tracing == 0); - PyObject *self = TOP(); - PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); - assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyObject *dict = _PyDictOrValues_GetDict(dorv); - PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; - // Note: cache->keys_version can be 0 when dict is NULL. - DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0012ba82e948..42b7ca086705 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2964,31 +2964,6 @@ DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_MANAGED_DICT) { - assert(cframe.use_tracing == 0); - PyObject *self = TOP(); - PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); - assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyObject *dict = _PyDictOrValues_GetDict(dorv); - PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; - // Note: cache->keys_version can be 0 when dict is NULL. - DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - DISPATCH(); - } - TARGET(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ assert(cframe.use_tracing == 0); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 67f5bacecc8c..be3ad01c151c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -80,12 +80,12 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_MANAGED_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,27 +152,27 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 53215748c99b..d9af7b742d54 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -335,6 +335,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 +#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -1035,14 +1036,11 @@ PyObject *descr, DescriptorClassification kind) PyDictKeysObject *keys; if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; if (_PyDictOrValues_IsValues(dorv)) { - keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; dictkind = MANAGED_VALUES; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - keys = dict != NULL ? dict->ma_keys : NULL; - // User has directly accessed __dict__. dictkind = MANAGED_DICT; } } @@ -1069,7 +1067,7 @@ PyObject *descr, DescriptorClassification kind) } } } - if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT || (dictkind == MANAGED_DICT && keys != NULL)) { + if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT) { Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != DKIX_EMPTY) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); @@ -1090,11 +1088,8 @@ PyObject *descr, DescriptorClassification kind) _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: - if (keys == NULL) { - write_u32(cache->keys_version, 0); - } - _py_set_opcode(instr, LOAD_ATTR_METHOD_MANAGED_DICT); - break; + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); + goto fail; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_DICT); From webhook-mailer at python.org Fri Dec 23 13:15:53 2022 From: webhook-mailer at python.org (sweeneyde) Date: Fri, 23 Dec 2022 18:15:53 -0000 Subject: [Python-checkins] gh-94155: Reduce hash collisions for code objects (#100183) Message-ID: <mailman.3038.1671819354.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a98d9ea56e7b473af54438ecc487a6bf1b4d6530 commit: a98d9ea56e7b473af54438ecc487a6bf1b4d6530 branch: main author: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com> committer: sweeneyde <36520290+sweeneyde at users.noreply.github.com> date: 2022-12-23T13:15:47-05:00 summary: gh-94155: Reduce hash collisions for code objects (#100183) * Uses a better hashing algorithm to get better dispersion and remove commutativity. * Incorporates `co_firstlineno`, `Py_SIZE(co)`, and bytecode instructions. * This is now the entire set of criteria used in `code_richcompare`, except for `_PyCode_ConstantKey` (which would incorporate the types of `co_consts` rather than just their values). files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst M Lib/test/test_code.py M Objects/codeobject.c diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 02ab8fbcdb07..b13d5770abe8 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -465,6 +465,32 @@ def f(): self.assertNotEqual(code_b, code_d) self.assertNotEqual(code_c, code_d) + def test_code_hash_uses_firstlineno(self): + c1 = (lambda: 1).__code__ + c2 = (lambda: 1).__code__ + self.assertNotEqual(c1, c2) + self.assertNotEqual(hash(c1), hash(c2)) + c3 = c1.replace(co_firstlineno=17) + self.assertNotEqual(c1, c3) + self.assertNotEqual(hash(c1), hash(c3)) + + def test_code_hash_uses_order(self): + # Swapping posonlyargcount and kwonlyargcount should change the hash. + c = (lambda x, y, *, z=1, w=1: 1).__code__ + self.assertEqual(c.co_argcount, 2) + self.assertEqual(c.co_posonlyargcount, 0) + self.assertEqual(c.co_kwonlyargcount, 2) + swapped = c.replace(co_posonlyargcount=2, co_kwonlyargcount=0) + self.assertNotEqual(c, swapped) + self.assertNotEqual(hash(c), hash(swapped)) + + def test_code_hash_uses_bytecode(self): + c = (lambda x, y: x + y).__code__ + d = (lambda x, y: x * y).__code__ + c1 = c.replace(co_code=d.co_code) + self.assertNotEqual(c, c1) + self.assertNotEqual(hash(c), hash(c1)) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst new file mode 100644 index 000000000000..e7c7ed2fad0e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst @@ -0,0 +1 @@ +Improved the hashing algorithm for code objects, mitigating some hash collisions. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 1e5a92270be8..e174c6fee9cc 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1842,28 +1842,41 @@ code_richcompare(PyObject *self, PyObject *other, int op) static Py_hash_t code_hash(PyCodeObject *co) { - Py_hash_t h, h0, h1, h2, h3; - h0 = PyObject_Hash(co->co_name); - if (h0 == -1) return -1; - h1 = PyObject_Hash(co->co_consts); - if (h1 == -1) return -1; - h2 = PyObject_Hash(co->co_names); - if (h2 == -1) return -1; - h3 = PyObject_Hash(co->co_localsplusnames); - if (h3 == -1) return -1; - Py_hash_t h4 = PyObject_Hash(co->co_linetable); - if (h4 == -1) { - return -1; + Py_uhash_t uhash = 20221211; + #define SCRAMBLE_IN(H) do { \ + uhash ^= (Py_uhash_t)(H); \ + uhash *= _PyHASH_MULTIPLIER; \ + } while (0) + #define SCRAMBLE_IN_HASH(EXPR) do { \ + Py_hash_t h = PyObject_Hash(EXPR); \ + if (h == -1) { \ + return -1; \ + } \ + SCRAMBLE_IN(h); \ + } while (0) + + SCRAMBLE_IN_HASH(co->co_name); + SCRAMBLE_IN_HASH(co->co_consts); + SCRAMBLE_IN_HASH(co->co_names); + SCRAMBLE_IN_HASH(co->co_localsplusnames); + SCRAMBLE_IN_HASH(co->co_linetable); + SCRAMBLE_IN_HASH(co->co_exceptiontable); + SCRAMBLE_IN(co->co_argcount); + SCRAMBLE_IN(co->co_posonlyargcount); + SCRAMBLE_IN(co->co_kwonlyargcount); + SCRAMBLE_IN(co->co_flags); + SCRAMBLE_IN(co->co_firstlineno); + SCRAMBLE_IN(Py_SIZE(co)); + for (int i = 0; i < Py_SIZE(co); i++) { + int deop = _PyOpcode_Deopt[_Py_OPCODE(_PyCode_CODE(co)[i])]; + SCRAMBLE_IN(deop); + SCRAMBLE_IN(_Py_OPARG(_PyCode_CODE(co)[i])); + i += _PyOpcode_Caches[deop]; } - Py_hash_t h5 = PyObject_Hash(co->co_exceptiontable); - if (h5 == -1) { - return -1; + if ((Py_hash_t)uhash == -1) { + return -2; } - h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ - co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^ - co->co_flags; - if (h == -1) h = -2; - return h; + return (Py_hash_t)uhash; } From webhook-mailer at python.org Fri Dec 23 14:41:43 2022 From: webhook-mailer at python.org (cjw296) Date: Fri, 23 Dec 2022 19:41:43 -0000 Subject: [Python-checkins] gh-83076: 3.8x speed improvement in (Async)Mock instantiation (#100252) Message-ID: <mailman.3039.1671824504.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c5726b727e26b81a267933654cf26b760a90d9aa commit: c5726b727e26b81a267933654cf26b760a90d9aa branch: main author: Carl Meyer <carl at oddbird.net> committer: cjw296 <chris at withers.org> date: 2022-12-23T19:41:37Z summary: gh-83076: 3.8x speed improvement in (Async)Mock instantiation (#100252) files: A Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst M Lib/test/test_unittest/testmock/testasync.py M Lib/unittest/mock.py diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index e05a22861d47..52a3b71be1ef 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -300,6 +300,19 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_async_attributes_instance(self): + async_instance = AsyncClass() + async_instance.async_func_attr = async_func + async_instance.later_async_func_attr = normal_func + + mock_async_instance = Mock(spec_set=async_instance) + + async_instance.later_async_func_attr = async_func + + self.assertIsInstance(mock_async_instance.async_func_attr, AsyncMock) + # only the shape of the spec at the time of mock construction matters + self.assertNotIsInstance(mock_async_instance.later_async_func_attr, AsyncMock) + def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index a273753d6a0a..583ab74a8255 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -411,15 +411,18 @@ class NonCallableMock(Base): # necessary. _lock = RLock() - def __new__(cls, /, *args, **kw): + def __new__( + cls, spec=None, wraps=None, name=None, spec_set=None, + parent=None, _spec_state=None, _new_name='', _new_parent=None, + _spec_as_instance=False, _eat_self=None, unsafe=False, **kwargs + ): # every instance has its own class # so we can create magic methods on the # class without stomping on other mocks bases = (cls,) if not issubclass(cls, AsyncMockMixin): # Check if spec is an async object or function - bound_args = _MOCK_SIG.bind_partial(cls, *args, **kw).arguments - spec_arg = bound_args.get('spec_set', bound_args.get('spec')) + spec_arg = spec_set or spec if spec_arg is not None and _is_async_obj(spec_arg): bases = (AsyncMockMixin, cls) new = type(cls.__name__, bases, {'__doc__': cls.__doc__}) @@ -505,10 +508,6 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_signature = None _spec_asyncs = [] - for attr in dir(spec): - if iscoroutinefunction(getattr(spec, attr, None)): - _spec_asyncs.append(attr) - if spec is not None and not _is_list(spec): if isinstance(spec, type): _spec_class = spec @@ -518,7 +517,13 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_as_instance, _eat_self) _spec_signature = res and res[1] - spec = dir(spec) + spec_list = dir(spec) + + for attr in spec_list: + if iscoroutinefunction(getattr(spec, attr, None)): + _spec_asyncs.append(attr) + + spec = spec_list __dict__ = self.__dict__ __dict__['_spec_class'] = _spec_class @@ -1057,9 +1062,6 @@ def _calls_repr(self, prefix="Calls"): return f"\n{prefix}: {safe_repr(self.mock_calls)}." -_MOCK_SIG = inspect.signature(NonCallableMock.__init__) - - class _AnyComparer(list): """A list which checks if it contains a call which may have an argument of ANY, flipping the components of item and self from @@ -2138,10 +2140,8 @@ def mock_add_spec(self, spec, spec_set=False): class AsyncMagicMixin(MagicMixin): - def __init__(self, /, *args, **kw): - self._mock_set_magics() # make magic work for kwargs in init - _safe_super(AsyncMagicMixin, self).__init__(*args, **kw) - self._mock_set_magics() # fix magic broken by upper level init + pass + class MagicMock(MagicMixin, Mock): """ @@ -2183,6 +2183,10 @@ def __get__(self, obj, _type=None): return self.create_mock() +_CODE_ATTRS = dir(CodeType) +_CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + + class AsyncMockMixin(Base): await_count = _delegating_property('await_count') await_args = _delegating_property('await_args') @@ -2200,7 +2204,9 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=CodeType) + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG code_mock.co_flags = inspect.CO_COROUTINE self.__dict__['__code__'] = code_mock self.__dict__['__name__'] = 'AsyncMock' diff --git a/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst new file mode 100644 index 000000000000..a4984e695b43 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst @@ -0,0 +1 @@ +Instantiation of ``Mock()`` and ``AsyncMock()`` is now 3.8x faster. From webhook-mailer at python.org Fri Dec 23 15:17:30 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Fri, 23 Dec 2022 20:17:30 -0000 Subject: [Python-checkins] gh-99482: remove `jython` compatibility parts from stdlib and tests (#99484) Message-ID: <mailman.3040.1671826652.3313.python-checkins@python.org> https://github.com/python/cpython/commit/745545b5bb847023f90505bf9caa983463413780 commit: 745545b5bb847023f90505bf9caa983463413780 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T14:17:24-06:00 summary: gh-99482: remove `jython` compatibility parts from stdlib and tests (#99484) files: A Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst M Doc/whatsnew/3.12.rst M Lib/copy.py M Lib/pickle.py M Lib/platform.py M Lib/site.py M Lib/test/support/__init__.py M Lib/test/support/os_helper.py M Lib/test/test___all__.py M Lib/test/test_codeop.py M Lib/test/test_exceptions.py M Lib/test/test_import/__init__.py M Lib/test/test_platform.py M Lib/test/test_strftime.py M Lib/test/test_unicode.py M Lib/types.py M Lib/unittest/loader.py M Lib/xml/sax/__init__.py M Lib/xml/sax/_exceptions.py M Lib/xml/sax/expatreader.py diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 617fadd83f8f..69f97debd694 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -955,6 +955,9 @@ Removed ``SSTATE_INTERNED_IMMORTAL`` macro. (Contributed by Victor Stinner in :gh:`85858`.) +* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. + (Contributed by Nikita Sobolev in :gh:`99482`.) + * Remove ``_use_broken_old_ctypes_structure_semantics_`` flag from :mod:`ctypes` module. (Contributed by Nikita Sobolev in :gh:`99285`.) diff --git a/Lib/copy.py b/Lib/copy.py index 1b276afe0812..6e8c19bc652d 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -56,11 +56,6 @@ class Error(Exception): pass error = Error # backward compatibility -try: - from org.python.core import PyStringMap -except ImportError: - PyStringMap = None - __all__ = ["Error", "copy", "deepcopy"] def copy(x): @@ -120,9 +115,6 @@ def _copy_immutable(x): d[set] = set.copy d[bytearray] = bytearray.copy -if PyStringMap is not None: - d[PyStringMap] = PyStringMap.copy - del d, t def deepcopy(x, memo=None, _nil=[]): @@ -231,8 +223,6 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy): y[deepcopy(key, memo)] = deepcopy(value, memo) return y d[dict] = _deepcopy_dict -if PyStringMap is not None: - d[PyStringMap] = _deepcopy_dict def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) @@ -301,4 +291,4 @@ def _reconstruct(x, memo, func, args, y[key] = value return y -del types, weakref, PyStringMap +del types, weakref diff --git a/Lib/pickle.py b/Lib/pickle.py index f027e0432045..15fa5f6e5799 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -98,12 +98,6 @@ class _Stop(Exception): def __init__(self, value): self.value = value -# Jython has PyStringMap; it's a dict subclass with string keys -try: - from org.python.core import PyStringMap -except ImportError: - PyStringMap = None - # Pickle opcodes. See pickletools.py for extensive docs. The listing # here is in kind-of alphabetical order of 1-character pickle code. # pickletools groups them by purpose. @@ -972,8 +966,6 @@ def save_dict(self, obj): self._batch_setitems(obj.items()) dispatch[dict] = save_dict - if PyStringMap is not None: - dispatch[PyStringMap] = save_dict def _batch_setitems(self, items): # Helper to batch up SETITEMS sequences; proto >= 1 only diff --git a/Lib/platform.py b/Lib/platform.py index 6745321e31c2..b018046f5268 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1290,7 +1290,7 @@ def platform(aliased=0, terse=0): else: platform = _platform(system, release, version, csd) - elif system in ('Linux',): + elif system == 'Linux': # check for libc vs. glibc libcname, libcversion = libc_ver() platform = _platform(system, release, machine, processor, diff --git a/Lib/site.py b/Lib/site.py index 69670d9d7f22..7faf1c6f6af2 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -404,12 +404,7 @@ def setquit(): def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) - if sys.platform[:4] == 'java': - builtins.credits = _sitebuiltins._Printer( - "credits", - "Jython is maintained by the Jython developers (www.jython.org).") - else: - builtins.credits = _sitebuiltins._Printer("credits", """\ + builtins.credits = _sitebuiltins._Printer("credits", """\ Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands for supporting Python development. See www.python.org for more information.""") files, dirs = [], [] diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b7186057990a..ff736f1c2db8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -505,6 +505,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): requires_legacy_unicode_capi = unittest.skipUnless(unicode_legacy_string, 'requires legacy Unicode C API') +# Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') is_android = hasattr(sys, 'getandroidapilevel') @@ -736,8 +737,6 @@ def gc_collect(): """ import gc gc.collect() - if is_jython: - time.sleep(0.1) gc.collect() gc.collect() diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index f37a442aa0e6..2d4356a1191b 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -11,11 +11,7 @@ # Filename used for testing -if os.name == 'java': - # Jython disallows @ in module names - TESTFN_ASCII = '$test' -else: - TESTFN_ASCII = '@test' +TESTFN_ASCII = '@test' # Disambiguate TESTFN for parallel testing, while letting it remain a valid # module name. diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 1ec83cb0b144..ecf73b3ad1be 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -100,10 +100,9 @@ def test_all(self): '__future__', ]) - if not sys.platform.startswith('java'): - # In case _socket fails to build, make this test fail more gracefully - # than an AttributeError somewhere deep in CGIHTTPServer. - import _socket + # In case _socket fails to build, make this test fail more gracefully + # than an AttributeError somewhere deep in CGIHTTPServer. + import _socket ignored = [] failed_imports = [] diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index d7b51be642e4..6966c2ffd811 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -2,47 +2,18 @@ Test cases for codeop.py Nick Mathewson """ -import sys import unittest import warnings -from test import support from test.support import warnings_helper from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT -import io - -if support.is_jython: - - def unify_callables(d): - for n,v in d.items(): - if hasattr(v, '__call__'): - d[n] = True - return d class CodeopTests(unittest.TestCase): def assertValid(self, str, symbol='single'): '''succeed iff str is a valid piece of code''' - if support.is_jython: - code = compile_command(str, "<input>", symbol) - self.assertTrue(code) - if symbol == "single": - d,r = {},{} - saved_stdout = sys.stdout - sys.stdout = io.StringIO() - try: - exec(code, d) - exec(compile(str,"<input>","single"), r) - finally: - sys.stdout = saved_stdout - elif symbol == 'eval': - ctx = {'a': 2} - d = { 'value': eval(code,ctx) } - r = { 'value': eval(str,ctx) } - self.assertEqual(unify_callables(r),unify_callables(d)) - else: - expected = compile(str, "<input>", symbol, PyCF_DONT_IMPLY_DEDENT) - self.assertEqual(compile_command(str, "<input>", symbol), expected) + expected = compile(str, "<input>", symbol, PyCF_DONT_IMPLY_DEDENT) + self.assertEqual(compile_command(str, "<input>", symbol), expected) def assertIncomplete(self, str, symbol='single'): '''succeed iff str is the start of a valid piece of code''' @@ -62,16 +33,12 @@ def test_valid(self): av = self.assertValid # special case - if not support.is_jython: - self.assertEqual(compile_command(""), - compile("pass", "<input>", 'single', - PyCF_DONT_IMPLY_DEDENT)) - self.assertEqual(compile_command("\n"), - compile("pass", "<input>", 'single', - PyCF_DONT_IMPLY_DEDENT)) - else: - av("") - av("\n") + self.assertEqual(compile_command(""), + compile("pass", "<input>", 'single', + PyCF_DONT_IMPLY_DEDENT)) + self.assertEqual(compile_command("\n"), + compile("pass", "<input>", 'single', + PyCF_DONT_IMPLY_DEDENT)) av("a = 1") av("\na = 1") diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 65a3a8a48a88..f629321458d8 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -360,10 +360,9 @@ def test_capi3(): self.assertRaises(SystemError, _testcapi.raise_exception, InvalidException, 1) - if not sys.platform.startswith('java'): - test_capi1() - test_capi2() - test_capi3() + test_capi1() + test_capi2() + test_capi3() def test_WindowsError(self): try: diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 6c5b80bcee6c..1e4429ed7efe 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -20,7 +20,7 @@ from test.support import os_helper from test.support import ( - STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten, + STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, is_wasi) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) @@ -163,10 +163,7 @@ def test_import(self): def test_with_extension(ext): # The extension is normally ".py", perhaps ".pyw". source = TESTFN + ext - if is_jython: - pyc = TESTFN + "$py.class" - else: - pyc = TESTFN + ".pyc" + pyc = TESTFN + ".pyc" with open(source, "w", encoding='utf-8') as f: print("# This tests Python's ability to import a", diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 3992faf8e5cd..72942dda3424 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -329,7 +329,7 @@ def raises_oserror(*a): def test_java_ver(self): res = platform.java_ver() - if sys.platform == 'java': + if sys.platform == 'java': # Is never actually checked in CI self.assertTrue(all(res)) def test_win32_ver(self): diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index be43c49e40aa..cebfc8927862 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -54,14 +54,10 @@ def _update_variables(self, now): self.now = now def setUp(self): - try: - import java - java.util.Locale.setDefault(java.util.Locale.US) - except ImportError: - from locale import setlocale, LC_TIME - saved_locale = setlocale(LC_TIME) - setlocale(LC_TIME, 'C') - self.addCleanup(setlocale, LC_TIME, saved_locale) + from locale import setlocale, LC_TIME + saved_locale = setlocale(LC_TIME) + setlocale(LC_TIME, 'C') + self.addCleanup(setlocale, LC_TIME, saved_locale) def test_strftime(self): now = time.time() diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 5465d35534f1..79360f5549e4 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -94,88 +94,86 @@ def test_literals(self): self.assertNotEqual(r"\u0020", " ") def test_ascii(self): - if not sys.platform.startswith('java'): - # Test basic sanity of repr() - self.assertEqual(ascii('abc'), "'abc'") - self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") - self.assertEqual(ascii('ab\\'), "'ab\\\\'") - self.assertEqual(ascii('\\c'), "'\\\\c'") - self.assertEqual(ascii('\\'), "'\\\\'") - self.assertEqual(ascii('\n'), "'\\n'") - self.assertEqual(ascii('\r'), "'\\r'") - self.assertEqual(ascii('\t'), "'\\t'") - self.assertEqual(ascii('\b'), "'\\x08'") - self.assertEqual(ascii("'\""), """'\\'"'""") - self.assertEqual(ascii("'\""), """'\\'"'""") - self.assertEqual(ascii("'"), '''"'"''') - self.assertEqual(ascii('"'), """'"'""") - latin1repr = ( - "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" - "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" - "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" - "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" - "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" - "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" - "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9" - "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7" - "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5" - "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3" - "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1" - "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef" - "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd" - "\\xfe\\xff'") - testrepr = ascii(''.join(map(chr, range(256)))) - self.assertEqual(testrepr, latin1repr) - # Test ascii works on wide unicode escapes without overflow. - self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), - ascii("\U00010000" * 39 + "\uffff" * 4096)) - - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, ascii, WrongRepr()) + # Test basic sanity of repr() + self.assertEqual(ascii('abc'), "'abc'") + self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") + self.assertEqual(ascii('ab\\'), "'ab\\\\'") + self.assertEqual(ascii('\\c'), "'\\\\c'") + self.assertEqual(ascii('\\'), "'\\\\'") + self.assertEqual(ascii('\n'), "'\\n'") + self.assertEqual(ascii('\r'), "'\\r'") + self.assertEqual(ascii('\t'), "'\\t'") + self.assertEqual(ascii('\b'), "'\\x08'") + self.assertEqual(ascii("'\""), """'\\'"'""") + self.assertEqual(ascii("'\""), """'\\'"'""") + self.assertEqual(ascii("'"), '''"'"''') + self.assertEqual(ascii('"'), """'"'""") + latin1repr = ( + "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" + "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" + "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" + "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" + "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" + "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" + "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9" + "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7" + "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5" + "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3" + "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1" + "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef" + "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd" + "\\xfe\\xff'") + testrepr = ascii(''.join(map(chr, range(256)))) + self.assertEqual(testrepr, latin1repr) + # Test ascii works on wide unicode escapes without overflow. + self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), + ascii("\U00010000" * 39 + "\uffff" * 4096)) + + class WrongRepr: + def __repr__(self): + return b'byte-repr' + self.assertRaises(TypeError, ascii, WrongRepr()) def test_repr(self): - if not sys.platform.startswith('java'): - # Test basic sanity of repr() - self.assertEqual(repr('abc'), "'abc'") - self.assertEqual(repr('ab\\c'), "'ab\\\\c'") - self.assertEqual(repr('ab\\'), "'ab\\\\'") - self.assertEqual(repr('\\c'), "'\\\\c'") - self.assertEqual(repr('\\'), "'\\\\'") - self.assertEqual(repr('\n'), "'\\n'") - self.assertEqual(repr('\r'), "'\\r'") - self.assertEqual(repr('\t'), "'\\t'") - self.assertEqual(repr('\b'), "'\\x08'") - self.assertEqual(repr("'\""), """'\\'"'""") - self.assertEqual(repr("'\""), """'\\'"'""") - self.assertEqual(repr("'"), '''"'"''') - self.assertEqual(repr('"'), """'"'""") - latin1repr = ( - "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" - "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" - "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" - "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" - "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" - "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" - "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" - "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" - "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5" - "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3" - "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1" - "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" - "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd" - "\xfe\xff'") - testrepr = repr(''.join(map(chr, range(256)))) - self.assertEqual(testrepr, latin1repr) - # Test repr works on wide unicode escapes without overflow. - self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), - repr("\U00010000" * 39 + "\uffff" * 4096)) - - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, repr, WrongRepr()) + # Test basic sanity of repr() + self.assertEqual(repr('abc'), "'abc'") + self.assertEqual(repr('ab\\c'), "'ab\\\\c'") + self.assertEqual(repr('ab\\'), "'ab\\\\'") + self.assertEqual(repr('\\c'), "'\\\\c'") + self.assertEqual(repr('\\'), "'\\\\'") + self.assertEqual(repr('\n'), "'\\n'") + self.assertEqual(repr('\r'), "'\\r'") + self.assertEqual(repr('\t'), "'\\t'") + self.assertEqual(repr('\b'), "'\\x08'") + self.assertEqual(repr("'\""), """'\\'"'""") + self.assertEqual(repr("'\""), """'\\'"'""") + self.assertEqual(repr("'"), '''"'"''') + self.assertEqual(repr('"'), """'"'""") + latin1repr = ( + "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" + "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" + "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" + "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" + "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" + "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" + "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" + "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5" + "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3" + "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1" + "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" + "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd" + "\xfe\xff'") + testrepr = repr(''.join(map(chr, range(256)))) + self.assertEqual(testrepr, latin1repr) + # Test repr works on wide unicode escapes without overflow. + self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), + repr("\U00010000" * 39 + "\uffff" * 4096)) + + class WrongRepr: + def __repr__(self): + return b'byte-repr' + self.assertRaises(TypeError, repr, WrongRepr()) def test_iterators(self): # Make sure unicode objects have an __iter__ method @@ -684,8 +682,7 @@ def test_islower(self): def test_isupper(self): super().test_isupper() - if not sys.platform.startswith('java'): - self.checkequalnofix(False, '\u1FFc', 'isupper') + self.checkequalnofix(False, '\u1FFc', 'isupper') self.assertTrue('\u2167'.isupper()) self.assertFalse('\u2177'.isupper()) # non-BMP, uppercase @@ -1473,10 +1470,9 @@ def test_formatting(self): self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.5), 'abc, abc, -1, -2.000000, 3.50') self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.57), 'abc, abc, -1, -2.000000, 3.57') self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 1003.57), 'abc, abc, -1, -2.000000, 1003.57') - if not sys.platform.startswith('java'): - self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'") - self.assertEqual("%r" % ("\u1234",), "'\u1234'") - self.assertEqual("%a" % ("\u1234",), "'\\u1234'") + self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'") + self.assertEqual("%r" % ("\u1234",), "'\u1234'") + self.assertEqual("%a" % ("\u1234",), "'\\u1234'") self.assertEqual("%(x)s, %(y)s" % {'x':"abc", 'y':"def"}, 'abc, def') self.assertEqual("%(x)s, %(\xfc)s" % {'x':"abc", '\xfc':"def"}, 'abc, def') @@ -1671,29 +1667,27 @@ def __str__(self): # unicode(obj, encoding, error) tests (this maps to # PyUnicode_FromEncodedObject() at C level) - if not sys.platform.startswith('java'): - self.assertRaises( - TypeError, - str, - 'decoding unicode is not supported', - 'utf-8', - 'strict' - ) + self.assertRaises( + TypeError, + str, + 'decoding unicode is not supported', + 'utf-8', + 'strict' + ) self.assertEqual( str(b'strings are decoded to unicode', 'utf-8', 'strict'), 'strings are decoded to unicode' ) - if not sys.platform.startswith('java'): - self.assertEqual( - str( - memoryview(b'character buffers are decoded to unicode'), - 'utf-8', - 'strict' - ), - 'character buffers are decoded to unicode' - ) + self.assertEqual( + str( + memoryview(b'character buffers are decoded to unicode'), + 'utf-8', + 'strict' + ), + 'character buffers are decoded to unicode' + ) self.assertRaises(TypeError, str, 42, 42, 42) diff --git a/Lib/types.py b/Lib/types.py index f8353126cb52..aa8a1c847223 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -56,7 +56,6 @@ def _m(self): pass TracebackType = type(exc.__traceback__) FrameType = type(exc.__traceback__.tb_frame) -# For Jython, the following two types are identical GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index eb18cd0b49cd..80d4fbdd8e36 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -57,9 +57,7 @@ def testSkipped(self): TestClass = type("ModuleSkipped", (case.TestCase,), attrs) return suiteClass((TestClass(methodname),)) -def _jython_aware_splitext(path): - if path.lower().endswith('$py.class'): - return path[:-9] +def _splitext(path): return os.path.splitext(path)[0] @@ -315,7 +313,7 @@ def _get_directory_containing_module(self, module_name): def _get_name_from_path(self, path): if path == self._top_level_dir: return '.' - path = _jython_aware_splitext(os.path.normpath(path)) + path = _splitext(os.path.normpath(path)) _relpath = os.path.relpath(path, self._top_level_dir) assert not os.path.isabs(_relpath), "Path must be within the project" @@ -393,13 +391,13 @@ def _find_test_path(self, full_path, pattern): else: mod_file = os.path.abspath( getattr(module, '__file__', full_path)) - realpath = _jython_aware_splitext( + realpath = _splitext( os.path.realpath(mod_file)) - fullpath_noext = _jython_aware_splitext( + fullpath_noext = _splitext( os.path.realpath(full_path)) if realpath.lower() != fullpath_noext.lower(): module_dir = os.path.dirname(realpath) - mod_name = _jython_aware_splitext( + mod_name = _splitext( os.path.basename(full_path)) expected_dir = os.path.dirname(full_path) msg = ("%r module incorrectly imported from %r. Expected " diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index 17b75879ebaa..b657310207cf 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -60,11 +60,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: default_parser_list = os.environ["PY_SAX_PARSER"].split(",") -del os - -_key = "python.xml.sax.parser" -if sys.platform[:4] == "java" and sys.registry.containsKey(_key): - default_parser_list = sys.registry.getProperty(_key).split(",") +del os, sys def make_parser(parser_list=()): @@ -93,15 +89,6 @@ def make_parser(parser_list=()): # --- Internal utility methods used by make_parser -if sys.platform[ : 4] == "java": - def _create_parser(parser_name): - from org.python.core import imp - drv_module = imp.importName(parser_name, 0, globals()) - return drv_module.create_parser() - -else: - def _create_parser(parser_name): - drv_module = __import__(parser_name,{},{},['create_parser']) - return drv_module.create_parser() - -del sys +def _create_parser(parser_name): + drv_module = __import__(parser_name,{},{},['create_parser']) + return drv_module.create_parser() diff --git a/Lib/xml/sax/_exceptions.py b/Lib/xml/sax/_exceptions.py index a9b2ba35c6a2..f292dc3a8e50 100644 --- a/Lib/xml/sax/_exceptions.py +++ b/Lib/xml/sax/_exceptions.py @@ -1,8 +1,4 @@ """Different kinds of SAX Exceptions""" -import sys -if sys.platform[:4] == "java": - from java.lang import Exception -del sys # ===== SAXEXCEPTION ===== diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index e334ac9fea0d..b9ad52692db8 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -12,12 +12,6 @@ from xml.sax.handler import feature_string_interning from xml.sax.handler import property_xml_string, property_interning_dict -# xml.parsers.expat does not raise ImportError in Jython -import sys -if sys.platform[:4] == "java": - raise SAXReaderNotAvailable("expat not available in Java", None) -del sys - try: from xml.parsers import expat except ImportError: diff --git a/Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst b/Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst new file mode 100644 index 000000000000..dd2c925478c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst @@ -0,0 +1 @@ +Remove ``Jython`` partial compatibility code from several stdlib modules. From webhook-mailer at python.org Fri Dec 23 15:19:21 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Fri, 23 Dec 2022 20:19:21 -0000 Subject: [Python-checkins] bpo-40447: accept all path-like objects in compileall.compile_file (#19883) Message-ID: <mailman.3041.1671826763.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e commit: 1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e branch: main author: Filipe La?ns <lains at riseup.net> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T14:19:16-06:00 summary: bpo-40447: accept all path-like objects in compileall.compile_file (#19883) Signed-off-by: Filipe La?ns <lains at archlinux.org> Signed-off-by: Filipe La?ns <lains at riseup.net> Co-authored-by: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst M Lib/compileall.py M Lib/test/test_compileall.py diff --git a/Lib/compileall.py b/Lib/compileall.py index 330a90786efc..a388931fb5a9 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -154,8 +154,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 73c83c9bf1ef..05154c8f1c60 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -167,6 +167,20 @@ def test_compile_file_pathlike_ddir(self): quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_file_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_file_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_path(self): with test.test_importlib.util.import_state(path=[self.directory]): self.assertTrue(compileall.compile_path(quiet=2)) @@ -219,6 +233,20 @@ def test_compile_dir_pathlike(self): self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_dir_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_dir_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + @skipUnless(_have_multiprocessing, "requires multiprocessing") @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): diff --git a/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst new file mode 100644 index 000000000000..941038d095b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst @@ -0,0 +1,2 @@ +Accept :class:`os.PathLike` (such as :class:`pathlib.Path`) in the ``stripdir`` arguments of +:meth:`compileall.compile_file` and :meth:`compileall.compile_dir`. From webhook-mailer at python.org Fri Dec 23 15:40:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 20:40:00 -0000 Subject: [Python-checkins] bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) Message-ID: <mailman.3042.1671828001.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9c4b8cca09342734eb04fc1093d5952a2317dfb7 commit: 9c4b8cca09342734eb04fc1093d5952a2317dfb7 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T12:39:55-08:00 summary: bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) (cherry picked from commit 1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e) Co-authored-by: Filipe La?ns <lains at riseup.net> Signed-off-by: Filipe La?ns <lains at archlinux.org> Signed-off-by: Filipe La?ns <lains at riseup.net> Co-authored-by: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst M Lib/compileall.py M Lib/test/test_compileall.py diff --git a/Lib/compileall.py b/Lib/compileall.py index 3755e76ba813..50183ea85468 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -154,8 +154,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index a7318bf404c3..554e2b27eea2 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -167,6 +167,20 @@ def test_compile_file_pathlike_ddir(self): quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_file_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_file_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_path(self): with test.test_importlib.util.import_state(path=[self.directory]): self.assertTrue(compileall.compile_path(quiet=2)) @@ -219,6 +233,20 @@ def test_compile_dir_pathlike(self): self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_dir_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_dir_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + @skipUnless(_have_multiprocessing, "requires multiprocessing") @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): diff --git a/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst new file mode 100644 index 000000000000..941038d095b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst @@ -0,0 +1,2 @@ +Accept :class:`os.PathLike` (such as :class:`pathlib.Path`) in the ``stripdir`` arguments of +:meth:`compileall.compile_file` and :meth:`compileall.compile_dir`. From webhook-mailer at python.org Fri Dec 23 15:43:19 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 20:43:19 -0000 Subject: [Python-checkins] bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) Message-ID: <mailman.3043.1671828199.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d5eb2f4747cd9afbc7f32fd6373a5f73aa59f61c commit: d5eb2f4747cd9afbc7f32fd6373a5f73aa59f61c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T12:43:13-08:00 summary: bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) (cherry picked from commit 1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e) Co-authored-by: Filipe La?ns <lains at riseup.net> Signed-off-by: Filipe La?ns <lains at archlinux.org> Signed-off-by: Filipe La?ns <lains at riseup.net> Co-authored-by: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst M Lib/compileall.py M Lib/test/test_compileall.py diff --git a/Lib/compileall.py b/Lib/compileall.py index 330a90786efc..a388931fb5a9 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -154,8 +154,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 73c83c9bf1ef..05154c8f1c60 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -167,6 +167,20 @@ def test_compile_file_pathlike_ddir(self): quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_file_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_file_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_path(self): with test.test_importlib.util.import_state(path=[self.directory]): self.assertTrue(compileall.compile_path(quiet=2)) @@ -219,6 +233,20 @@ def test_compile_dir_pathlike(self): self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_dir_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_dir_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + @skipUnless(_have_multiprocessing, "requires multiprocessing") @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): diff --git a/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst new file mode 100644 index 000000000000..941038d095b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst @@ -0,0 +1,2 @@ +Accept :class:`os.PathLike` (such as :class:`pathlib.Path`) in the ``stripdir`` arguments of +:meth:`compileall.compile_file` and :meth:`compileall.compile_dir`. From webhook-mailer at python.org Fri Dec 23 17:36:04 2022 From: webhook-mailer at python.org (rhettinger) Date: Fri, 23 Dec 2022 22:36:04 -0000 Subject: [Python-checkins] GH-100425: Improve accuracy of builtin sum() for float inputs (GH-100426) Message-ID: <mailman.3044.1671834965.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5d84966cce6c596da22922a07f49bde959ff5201 commit: 5d84966cce6c596da22922a07f49bde959ff5201 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-23T14:35:58-08:00 summary: GH-100425: Improve accuracy of builtin sum() for float inputs (GH-100426) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst M Doc/library/functions.rst M Doc/library/math.rst M Doc/tutorial/floatingpoint.rst M Lib/test/test_builtin.py M Python/bltinmodule.c diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2110990d1889..817c1f858aae 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1733,6 +1733,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.8 The *start* parameter can be specified as a keyword argument. + .. versionchanged:: 3.12 Summation of floats switched to an algorithm + that gives higher accuracy on most builds. + + .. class:: super() super(type, object_or_type=None) diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 559c6ec5dd9d..aeebcaf6ab08 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -108,12 +108,7 @@ Number-theoretic and representation functions .. function:: fsum(iterable) Return an accurate floating point sum of values in the iterable. Avoids - loss of precision by tracking multiple intermediate partial sums: - - >>> sum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 0.9999999999999999 - >>> fsum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 1.0 + loss of precision by tracking multiple intermediate partial sums. The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the typical case where the rounding mode is half-even. On some non-Windows diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index e1cd7f9ece75..cedade6e3366 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -192,7 +192,7 @@ added onto a running total. That can make a difference in overall accuracy so that the errors do not accumulate to the point where they affect the final total: - >>> sum([0.1] * 10) == 1.0 + >>> 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 == 1.0 False >>> math.fsum([0.1] * 10) == 1.0 True diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index eb1c389257cc..c65600483258 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -9,6 +9,7 @@ import gc import io import locale +import math import os import pickle import platform @@ -31,6 +32,7 @@ from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok from test.support.warnings_helper import check_warnings +from test.support import requires_IEEE_754 from unittest.mock import MagicMock, patch try: import pty, signal @@ -38,6 +40,12 @@ pty = signal = None +# Detect evidence of double-rounding: sum() does not always +# get improved accuracy on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + + class Squares: def __init__(self, max): @@ -1617,6 +1625,8 @@ def test_sum(self): self.assertEqual(repr(sum([-0.0])), '0.0') self.assertEqual(repr(sum([-0.0], -0.0)), '-0.0') self.assertEqual(repr(sum([], -0.0)), '-0.0') + self.assertTrue(math.isinf(sum([float("inf"), float("inf")]))) + self.assertTrue(math.isinf(sum([1e308, 1e308]))) self.assertRaises(TypeError, sum) self.assertRaises(TypeError, sum, 42) @@ -1641,6 +1651,14 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sum accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sum_accuracy(self): + self.assertEqual(sum([0.1] * 10), 1.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + def test_type(self): self.assertEqual(type(''), type('123')) self.assertNotEqual(type(''), type(())) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst new file mode 100644 index 000000000000..5559020b11d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst @@ -0,0 +1 @@ +Improve the accuracy of ``sum()`` with compensated summation. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ff96c25da5eb..2d4822e6d468 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2532,6 +2532,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) if (PyFloat_CheckExact(result)) { double f_result = PyFloat_AS_DOUBLE(result); + double c = 0.0; Py_SETREF(result, NULL); while(result == NULL) { item = PyIter_Next(iter); @@ -2539,10 +2540,25 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) Py_DECREF(iter); if (PyErr_Occurred()) return NULL; + /* Avoid losing the sign on a negative result, + and don't let adding the compensation convert + an infinite or overflowed sum to a NaN. */ + if (c && Py_IS_FINITE(c)) { + f_result += c; + } return PyFloat_FromDouble(f_result); } if (PyFloat_CheckExact(item)) { - f_result += PyFloat_AS_DOUBLE(item); + // Improved Kahan?Babu?ka algorithm by Arnold Neumaier + // https://www.mat.univie.ac.at/~neum/scan/01.pdf + double x = PyFloat_AS_DOUBLE(item); + double t = f_result + x; + if (fabs(f_result) >= fabs(x)) { + c += (f_result - t) + x; + } else { + c += (x - t) + f_result; + } + f_result = t; _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); continue; } @@ -2556,6 +2572,9 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) continue; } } + if (c && Py_IS_FINITE(c)) { + f_result += c; + } result = PyFloat_FromDouble(f_result); if (result == NULL) { Py_DECREF(item); From webhook-mailer at python.org Fri Dec 23 17:52:37 2022 From: webhook-mailer at python.org (brettcannon) Date: Fri, 23 Dec 2022 22:52:37 -0000 Subject: [Python-checkins] gh-68320, gh-88302 - Allow for private `pathlib.Path` subclassing (GH-31691) Message-ID: <mailman.3045.1671835958.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a68e585c8b7b27323f67905868467ce0588a1dae commit: a68e585c8b7b27323f67905868467ce0588a1dae branch: main author: Barney Gale <barney.gale at gmail.com> committer: brettcannon <brett at python.org> date: 2022-12-23T14:52:23-08:00 summary: gh-68320, gh-88302 - Allow for private `pathlib.Path` subclassing (GH-31691) Users may wish to define subclasses of `pathlib.Path` to add or modify existing methods. Before this change, attempting to instantiate a subclass raised an exception like: AttributeError: type object 'PPath' has no attribute '_flavour' Previously the `_flavour` attribute was assigned as follows: PurePath._flavour = xxx not set!! xxx PurePosixPath._flavour = _PosixFlavour() PureWindowsPath._flavour = _WindowsFlavour() This change replaces it with a `_pathmod` attribute, set as follows: PurePath._pathmod = os.path PurePosixPath._pathmod = posixpath PureWindowsPath._pathmod = ntpath Functionality from `_PosixFlavour` and `_WindowsFlavour` is moved into `PurePath` as underscored-prefixed classmethods. Flavours are removed. Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> Co-authored-by: Brett Cannon <brett at python.org> Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Eryk Sun <eryksun at gmail.com> files: A Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst M Lib/pathlib.py M Lib/test/test_pathlib.py diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 7890fdade420..b959e85d1840 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -30,6 +30,14 @@ # Internals # +# Reference for Windows paths can be found at +# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . +_WIN_RESERVED_NAMES = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + _WINERROR_NOT_READY = 21 # drive exists but is not accessible _WINERROR_INVALID_NAME = 123 # fix for bpo-35306 _WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself @@ -52,150 +60,6 @@ def _is_wildcard_pattern(pat): # be looked up directly as a file. return "*" in pat or "?" in pat or "[" in pat - -class _Flavour(object): - """A flavour implements a particular (platform-specific) set of path - semantics.""" - - def __init__(self): - self.join = self.sep.join - - def parse_parts(self, parts): - if not parts: - return '', '', [] - sep = self.sep - altsep = self.altsep - path = self.pathmod.join(*parts) - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = self.splitroot(path) - unfiltered_parsed = [drv + root] + rel.split(sep) - parsed = [sys.intern(x) for x in unfiltered_parsed if x and x != '.'] - return drv, root, parsed - - def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2): - """ - Join the two paths represented by the respective - (drive, root, parts) tuples. Return a new (drive, root, parts) tuple. - """ - if root2: - if not drv2 and drv: - return drv, root2, [drv + root2] + parts2[1:] - elif drv2: - if drv2 == drv or self.casefold(drv2) == self.casefold(drv): - # Same drive => second path is relative to the first - return drv, root, parts + parts2[1:] - else: - # Second path is non-anchored (common case) - return drv, root, parts + parts2 - return drv2, root2, parts2 - - -class _WindowsFlavour(_Flavour): - # Reference for Windows paths can be found at - # http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx - - sep = '\\' - altsep = '/' - has_drv = True - pathmod = ntpath - - is_supported = (os.name == 'nt') - - reserved_names = ( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {'COM%s' % c for c in '123456789\xb9\xb2\xb3'} | - {'LPT%s' % c for c in '123456789\xb9\xb2\xb3'} - ) - - def splitroot(self, part, sep=sep): - drv, rest = self.pathmod.splitdrive(part) - if drv[:1] == sep or rest[:1] == sep: - return drv, sep, rest.lstrip(sep) - else: - return drv, '', rest - - def casefold(self, s): - return s.lower() - - def casefold_parts(self, parts): - return [p.lower() for p in parts] - - def compile_pattern(self, pattern): - return re.compile(fnmatch.translate(pattern), re.IGNORECASE).fullmatch - - def is_reserved(self, parts): - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if not parts: - return False - if parts[0].startswith('\\\\'): - # UNC paths are never reserved - return False - name = parts[-1].partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in self.reserved_names - - def make_uri(self, path): - # Under Windows, file URIs use the UTF-8 encoding. - drive = path.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - rest = path.as_posix()[2:].lstrip('/') - return 'file:///%s/%s' % ( - drive, urlquote_from_bytes(rest.encode('utf-8'))) - else: - # It's a path on a network drive => 'file://host/share/a/b' - return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8')) - - -class _PosixFlavour(_Flavour): - sep = '/' - altsep = '' - has_drv = False - pathmod = posixpath - - is_supported = (os.name != 'nt') - - def splitroot(self, part, sep=sep): - if part and part[0] == sep: - stripped_part = part.lstrip(sep) - # According to POSIX path resolution: - # http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap04.html#tag_04_11 - # "A pathname that begins with two successive slashes may be - # interpreted in an implementation-defined manner, although more - # than two leading slashes shall be treated as a single slash". - if len(part) - len(stripped_part) == 2: - return '', sep * 2, stripped_part - else: - return '', sep, stripped_part - else: - return '', '', part - - def casefold(self, s): - return s - - def casefold_parts(self, parts): - return parts - - def compile_pattern(self, pattern): - return re.compile(fnmatch.translate(pattern)).fullmatch - - def is_reserved(self, parts): - return False - - def make_uri(self, path): - # We represent the path using the local filesystem encoding, - # for portability to other applications. - bpath = bytes(path) - return 'file://' + urlquote_from_bytes(bpath) - - -_windows_flavour = _WindowsFlavour() -_posix_flavour = _PosixFlavour() - - # # Globbing helpers # @@ -237,14 +101,15 @@ def select_from(self, parent_path): is_dir = path_cls.is_dir exists = path_cls.exists scandir = path_cls._scandir + normcase = path_cls._flavour.normcase if not is_dir(parent_path): return iter([]) - return self._select_from(parent_path, is_dir, exists, scandir) + return self._select_from(parent_path, is_dir, exists, scandir, normcase) class _TerminatingSelector: - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): yield parent_path @@ -254,11 +119,11 @@ def __init__(self, name, child_parts, flavour): self.name = name _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): try: path = parent_path._make_child_relpath(self.name) if (is_dir if self.dironly else exists)(path): - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): yield p except PermissionError: return @@ -267,10 +132,10 @@ def _select_from(self, parent_path, is_dir, exists, scandir): class _WildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): - self.match = flavour.compile_pattern(pat) + self.match = re.compile(fnmatch.translate(flavour.normcase(pat))).fullmatch _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): try: # We must close the scandir() object before proceeding to # avoid exhausting file descriptors when globbing deep trees. @@ -289,9 +154,9 @@ def _select_from(self, parent_path, is_dir, exists, scandir): raise continue name = entry.name - if self.match(name): + if self.match(normcase(name)): path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): yield p except PermissionError: return @@ -323,13 +188,13 @@ def _iterate_directories(self, parent_path, is_dir, scandir): except PermissionError: return - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): try: yielded = set() try: successor_select = self.successor._select_from for starting_point in self._iterate_directories(parent_path, is_dir, scandir): - for p in successor_select(starting_point, is_dir, exists, scandir): + for p in successor_select(starting_point, is_dir, exists, scandir, normcase): if p not in yielded: yield p yielded.add(p) @@ -387,8 +252,9 @@ class PurePath(object): """ __slots__ = ( '_drv', '_root', '_parts', - '_str', '_hash', '_pparts', '_cached_cparts', + '_str', '_hash', '_parts_tuple', '_parts_normcase_cached', ) + _flavour = os.path def __new__(cls, *args): """Construct a PurePath from one or several strings and or existing @@ -405,6 +271,33 @@ def __reduce__(self): # when pickling related paths. return (self.__class__, tuple(self._parts)) + @classmethod + def _split_root(cls, part): + sep = cls._flavour.sep + rel = cls._flavour.splitdrive(part)[1].lstrip(sep) + anchor = part.removesuffix(rel) + if anchor: + anchor = cls._flavour.normpath(anchor) + drv, root = cls._flavour.splitdrive(anchor) + if drv.startswith(sep): + # UNC paths always have a root. + root = sep + return drv, root, rel + + @classmethod + def _parse_parts(cls, parts): + if not parts: + return '', '', [] + sep = cls._flavour.sep + altsep = cls._flavour.altsep + path = cls._flavour.join(*parts) + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls._split_root(path) + unfiltered_parsed = [drv + root] + rel.split(sep) + parsed = [sys.intern(x) for x in unfiltered_parsed if x and x != '.'] + return drv, root, parsed + @classmethod def _parse_args(cls, args): # This is useful when you don't want to create an instance, just @@ -423,7 +316,7 @@ def _parse_args(cls, args): "argument should be a str object or an os.PathLike " "object returning str, not %r" % type(a)) - return cls._flavour.parse_parts(parts) + return cls._parse_parts(parts) @classmethod def _from_parts(cls, args): @@ -447,15 +340,9 @@ def _from_parsed_parts(cls, drv, root, parts): @classmethod def _format_parsed_parts(cls, drv, root, parts): if drv or root: - return drv + root + cls._flavour.join(parts[1:]) + return drv + root + cls._flavour.sep.join(parts[1:]) else: - return cls._flavour.join(parts) - - def _make_child(self, args): - drv, root, parts = self._parse_args(args) - drv, root, parts = self._flavour.join_parsed_parts( - self._drv, self._root, self._parts, drv, root, parts) - return self._from_parsed_parts(drv, root, parts) + return cls._flavour.sep.join(parts) def __str__(self): """Return the string representation of the path, suitable for @@ -488,48 +375,62 @@ def as_uri(self): """Return the path as a 'file' URI.""" if not self.is_absolute(): raise ValueError("relative path can't be expressed as a file URI") - return self._flavour.make_uri(self) + + drive = self._drv + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + return prefix + urlquote_from_bytes(os.fsencode(path)) @property - def _cparts(self): - # Cached casefolded parts, for hashing and comparison + def _parts_normcase(self): + # Cached parts with normalized case, for hashing and comparison. try: - return self._cached_cparts + return self._parts_normcase_cached except AttributeError: - self._cached_cparts = self._flavour.casefold_parts(self._parts) - return self._cached_cparts + self._parts_normcase_cached = [self._flavour.normcase(p) for p in self._parts] + return self._parts_normcase_cached def __eq__(self, other): if not isinstance(other, PurePath): return NotImplemented - return self._cparts == other._cparts and self._flavour is other._flavour + return self._parts_normcase == other._parts_normcase and self._flavour is other._flavour def __hash__(self): try: return self._hash except AttributeError: - self._hash = hash(tuple(self._cparts)) + self._hash = hash(tuple(self._parts_normcase)) return self._hash def __lt__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts < other._cparts + return self._parts_normcase < other._parts_normcase def __le__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts <= other._cparts + return self._parts_normcase <= other._parts_normcase def __gt__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts > other._cparts + return self._parts_normcase > other._parts_normcase def __ge__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts >= other._cparts + return self._parts_normcase >= other._parts_normcase drive = property(attrgetter('_drv'), doc="""The drive prefix (letter or UNC path), if any.""") @@ -592,7 +493,7 @@ def with_name(self, name): """Return a new path with the file name changed.""" if not self.name: raise ValueError("%r has an empty name" % (self,)) - drv, root, parts = self._flavour.parse_parts((name,)) + drv, root, parts = self._parse_parts((name,)) if (not name or name[-1] in [self._flavour.sep, self._flavour.altsep] or drv or root or len(parts) != 1): raise ValueError("Invalid name %r" % (name)) @@ -669,10 +570,10 @@ def parts(self): # We cache the tuple to avoid building a new one each time .parts # is accessed. XXX is this necessary? try: - return self._pparts + return self._parts_tuple except AttributeError: - self._pparts = tuple(self._parts) - return self._pparts + self._parts_tuple = tuple(self._parts) + return self._parts_tuple def joinpath(self, *args): """Combine this path with one or several arguments, and return a @@ -680,11 +581,26 @@ def joinpath(self, *args): paths) or a totally different path (if one of the arguments is anchored). """ - return self._make_child(args) + drv1, root1, parts1 = self._drv, self._root, self._parts + drv2, root2, parts2 = self._parse_args(args) + if root2: + if not drv2 and drv1: + return self._from_parsed_parts(drv1, root2, [drv1 + root2] + parts2[1:]) + else: + return self._from_parsed_parts(drv2, root2, parts2) + elif drv2: + if drv2 == drv1 or self._flavour.normcase(drv2) == self._flavour.normcase(drv1): + # Same drive => second path is relative to the first. + return self._from_parsed_parts(drv1, root1, parts1 + parts2[1:]) + else: + return self._from_parsed_parts(drv2, root2, parts2) + else: + # Second path is non-anchored (common case). + return self._from_parsed_parts(drv1, root1, parts1 + parts2) def __truediv__(self, key): try: - return self._make_child((key,)) + return self.joinpath(key) except TypeError: return NotImplemented @@ -712,29 +628,40 @@ def parents(self): def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - if not self._root: - return False - return not self._flavour.has_drv or bool(self._drv) + # ntpath.isabs() is defective - see GH-44626 . + if self._flavour is ntpath: + return bool(self._drv and self._root) + return self._flavour.isabs(self) def is_reserved(self): """Return True if the path contains one of the special names reserved by the system, if any.""" - return self._flavour.is_reserved(self._parts) + if self._flavour is posixpath or not self._parts: + return False + + # NOTE: the rules for reserved names seem somewhat complicated + # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not + # exist). We err on the side of caution and return True for paths + # which are not considered reserved by Windows. + if self._parts[0].startswith('\\\\'): + # UNC paths are never reserved. + return False + name = self._parts[-1].partition('.')[0].partition(':')[0].rstrip(' ') + return name.upper() in _WIN_RESERVED_NAMES def match(self, path_pattern): """ Return True if this path matches the given pattern. """ - cf = self._flavour.casefold - path_pattern = cf(path_pattern) - drv, root, pat_parts = self._flavour.parse_parts((path_pattern,)) + path_pattern = self._flavour.normcase(path_pattern) + drv, root, pat_parts = self._parse_parts((path_pattern,)) if not pat_parts: raise ValueError("empty pattern") - if drv and drv != cf(self._drv): + elif drv and drv != self._flavour.normcase(self._drv): return False - if root and root != cf(self._root): + elif root and root != self._root: return False - parts = self._cparts + parts = self._parts_normcase if drv or root: if len(pat_parts) != len(parts): return False @@ -757,7 +684,7 @@ class PurePosixPath(PurePath): On a POSIX system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - _flavour = _posix_flavour + _flavour = posixpath __slots__ = () @@ -767,7 +694,7 @@ class PureWindowsPath(PurePath): On a Windows system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - _flavour = _windows_flavour + _flavour = ntpath __slots__ = () @@ -789,7 +716,7 @@ def __new__(cls, *args, **kwargs): if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath self = cls._from_parts(args) - if not self._flavour.is_supported: + if self._flavour is not os.path: raise NotImplementedError("cannot instantiate %r on your system" % (cls.__name__,)) return self @@ -842,7 +769,7 @@ def samefile(self, other_path): other_st = other_path.stat() except AttributeError: other_st = self.__class__(other_path).stat() - return os.path.samestat(st, other_st) + return self._flavour.samestat(st, other_st) def iterdir(self): """Yield path objects of the directory contents. @@ -866,7 +793,7 @@ def glob(self, pattern): sys.audit("pathlib.Path.glob", self, pattern) if not pattern: raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) + drv, root, pattern_parts = self._parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") if pattern[-1] in (self._flavour.sep, self._flavour.altsep): @@ -881,7 +808,7 @@ def rglob(self, pattern): this subtree. """ sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) + drv, root, pattern_parts = self._parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): @@ -912,7 +839,7 @@ def check_eloop(e): raise RuntimeError("Symlink loop from %r" % e.filename) try: - s = os.path.realpath(self, strict=strict) + s = self._flavour.realpath(self, strict=strict) except OSError as e: check_eloop(e) raise @@ -1184,7 +1111,7 @@ def is_mount(self): """ Check if this path is a mount point """ - return self._flavour.pathmod.ismount(self) + return self._flavour.ismount(self) def is_symlink(self): """ @@ -1205,7 +1132,7 @@ def is_junction(self): """ Whether this path is a junction. """ - return self._flavour.pathmod.isjunction(self) + return self._flavour.isjunction(self) def is_block_device(self): """ @@ -1277,7 +1204,7 @@ def expanduser(self): """ if (not (self._drv or self._root) and self._parts and self._parts[0][:1] == '~'): - homedir = os.path.expanduser(self._parts[0]) + homedir = self._flavour.expanduser(self._parts[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") return self._from_parts([homedir] + self._parts[1:]) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index fa6ea0ac63d8..7d4d782cf5f0 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -26,7 +26,7 @@ class _BaseFlavourTest(object): def _check_parse_parts(self, arg, expected): - f = self.flavour.parse_parts + f = self.cls._parse_parts sep = self.flavour.sep altsep = self.flavour.altsep actual = f([x.replace('/', sep) for x in arg]) @@ -65,7 +65,8 @@ def test_parse_parts_common(self): class PosixFlavourTest(_BaseFlavourTest, unittest.TestCase): - flavour = pathlib._posix_flavour + cls = pathlib.PurePosixPath + flavour = pathlib.PurePosixPath._flavour def test_parse_parts(self): check = self._check_parse_parts @@ -80,7 +81,7 @@ def test_parse_parts(self): check(['\\a'], ('', '', ['\\a'])) def test_splitroot(self): - f = self.flavour.splitroot + f = self.cls._split_root self.assertEqual(f(''), ('', '', '')) self.assertEqual(f('a'), ('', '', 'a')) self.assertEqual(f('a/b'), ('', '', 'a/b')) @@ -101,7 +102,8 @@ def test_splitroot(self): class NTFlavourTest(_BaseFlavourTest, unittest.TestCase): - flavour = pathlib._windows_flavour + cls = pathlib.PureWindowsPath + flavour = pathlib.PureWindowsPath._flavour def test_parse_parts(self): check = self._check_parse_parts @@ -142,7 +144,7 @@ def test_parse_parts(self): check(['c:/a/b', 'c:/x/y'], ('c:', '\\', ['c:\\', 'x', 'y'])) def test_splitroot(self): - f = self.flavour.splitroot + f = self.cls._split_root self.assertEqual(f(''), ('', '', '')) self.assertEqual(f('a'), ('', '', 'a')) self.assertEqual(f('a\\b'), ('', '', 'a\\b')) @@ -151,19 +153,12 @@ def test_splitroot(self): self.assertEqual(f('c:a\\b'), ('c:', '', 'a\\b')) self.assertEqual(f('c:\\a\\b'), ('c:', '\\', 'a\\b')) # Redundant slashes in the root are collapsed. - self.assertEqual(f('\\\\a'), ('', '\\', 'a')) - self.assertEqual(f('\\\\\\a/b'), ('', '\\', 'a/b')) self.assertEqual(f('c:\\\\a'), ('c:', '\\', 'a')) self.assertEqual(f('c:\\\\\\a/b'), ('c:', '\\', 'a/b')) # Valid UNC paths. self.assertEqual(f('\\\\a\\b'), ('\\\\a\\b', '\\', '')) self.assertEqual(f('\\\\a\\b\\'), ('\\\\a\\b', '\\', '')) self.assertEqual(f('\\\\a\\b\\c\\d'), ('\\\\a\\b', '\\', 'c\\d')) - # These are non-UNC paths (according to ntpath.py and test_ntpath). - # However, command.com says such paths are invalid, so it's - # difficult to know what the right semantics are. - self.assertEqual(f('\\\\\\a\\b'), ('', '\\', 'a\\b')) - self.assertEqual(f('\\\\a'), ('', '\\', 'a')) # @@ -182,8 +177,7 @@ class _BasePurePathTest(object): ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), ], '/b/c/d': [ - ('a', '/b/c', 'd'), ('a', '///b//c', 'd/'), - ('/a', '/b/c', 'd'), + ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), # Empty components get removed. ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), ], @@ -291,19 +285,26 @@ def test_as_uri_common(self): def test_repr_common(self): for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - p = self.cls(pathstr) - clsname = p.__class__.__name__ - r = repr(p) - # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) - inner = r[len(clsname) + 1 : -1] - self.assertEqual(eval(inner), p.as_posix()) - # The repr() roundtrips. - q = eval(r, pathlib.__dict__) - self.assertIs(q.__class__, p.__class__) - self.assertEqual(q, p) - self.assertEqual(repr(q), r) + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + clsname = p.__class__.__name__ + r = repr(p) + # The repr() is in the form ClassName("forward-slashes path"). + self.assertTrue(r.startswith(clsname + '('), r) + self.assertTrue(r.endswith(')'), r) + inner = r[len(clsname) + 1 : -1] + self.assertEqual(eval(inner), p.as_posix()) + + def test_repr_roundtrips(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + r = repr(p) + # The repr() roundtrips. + q = eval(r, pathlib.__dict__) + self.assertIs(q.__class__, p.__class__) + self.assertEqual(q, p) + self.assertEqual(repr(q), r) def test_eq_common(self): P = self.cls @@ -2412,9 +2413,9 @@ def test_is_symlink(self): def test_is_junction(self): P = self.cls(BASE) - with mock.patch.object(P._flavour, 'pathmod'): - self.assertEqual(P.is_junction(), P._flavour.pathmod.isjunction.return_value) - P._flavour.pathmod.isjunction.assert_called_once_with(P) + with mock.patch.object(P._flavour, 'isjunction'): + self.assertEqual(P.is_junction(), P._flavour.isjunction.return_value) + P._flavour.isjunction.assert_called_once_with(P) def test_is_fifo_false(self): P = self.cls(BASE) @@ -3072,6 +3073,22 @@ def check(): check() +class PurePathSubclassTest(_BasePurePathTest, unittest.TestCase): + class cls(pathlib.PurePath): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +class PathSubclassTest(_BasePathTest, unittest.TestCase): + class cls(pathlib.Path): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + class CompatiblePathTest(unittest.TestCase): """ Test that a type can be made compatible with PurePath diff --git a/Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst b/Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst new file mode 100644 index 000000000000..8ca5213fb23a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst @@ -0,0 +1,3 @@ +Make :class:`pathlib.PurePath` and :class:`~pathlib.Path` subclassable +(private to start). Previously, attempting to instantiate a subclass +resulted in an :exc:`AttributeError` being raised. Patch by Barney Gale. From webhook-mailer at python.org Fri Dec 23 18:43:35 2022 From: webhook-mailer at python.org (brettcannon) Date: Fri, 23 Dec 2022 23:43:35 -0000 Subject: [Python-checkins] gh-99947: Ensure unreported errors are chained for SystemError during import (GH-99946) Message-ID: <mailman.3046.1671839017.3313.python-checkins@python.org> https://github.com/python/cpython/commit/474220e3a58d739acc5154eb3e000461d2222d62 commit: 474220e3a58d739acc5154eb3e000461d2222d62 branch: main author: Sebastian Berg <sebastianb at nvidia.com> committer: brettcannon <brett at python.org> date: 2022-12-23T15:43:19-08:00 summary: gh-99947: Ensure unreported errors are chained for SystemError during import (GH-99946) files: A Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst M Lib/test/test_importlib/extension/test_loader.py M Objects/moduleobject.c M Python/importdl.c diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index d69192b56bac..3bf2bbdcdcc4 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -351,9 +351,14 @@ def test_bad_modules(self): ]: with self.subTest(name_base): name = self.name + '_' + name_base - with self.assertRaises(SystemError): + with self.assertRaises(SystemError) as cm: self.load_module_by_name(name) + # If there is an unreported exception, it should be chained + # with the `SystemError`. + if "unreported_exception" in name_base: + self.assertIsNotNone(cm.exception.__cause__) + def test_nonascii(self): # Test that modules with non-ASCII names can be loaded. # punycode behaves slightly differently in some-ASCII and no-ASCII diff --git a/Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst b/Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst new file mode 100644 index 000000000000..fbed192d317b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst @@ -0,0 +1 @@ +Raising SystemError on import will now have its cause be set to the original unexpected exception. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 8e03f2446f6f..24190e320ee6 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -327,9 +327,10 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } else { if (PyErr_Occurred()) { - PyErr_Format(PyExc_SystemError, - "creation of module %s raised unreported exception", - name); + _PyErr_FormatFromCause( + PyExc_SystemError, + "creation of module %s raised unreported exception", + name); goto error; } } @@ -431,7 +432,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) return -1; } if (PyErr_Occurred()) { - PyErr_Format( + _PyErr_FormatFromCause( PyExc_SystemError, "execution of module %s raised unreported exception", name); diff --git a/Python/importdl.c b/Python/importdl.c index 40227674ca47..91fa06f49c28 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -180,8 +180,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) } goto error; } else if (PyErr_Occurred()) { - PyErr_Clear(); - PyErr_Format( + _PyErr_FormatFromCause( PyExc_SystemError, "initialization of %s raised unreported exception", name_buf); From webhook-mailer at python.org Fri Dec 23 18:52:21 2022 From: webhook-mailer at python.org (rhettinger) Date: Fri, 23 Dec 2022 23:52:21 -0000 Subject: [Python-checkins] Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) Message-ID: <mailman.3047.1671839542.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f89de679ffec35e82548341cb23e675546602288 commit: f89de679ffec35e82548341cb23e675546602288 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-23T15:52:16-08:00 summary: Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9146ed1bfb62..9688aae68e6c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -834,7 +834,8 @@ which incur interpreter overhead. return chain.from_iterable(repeat(tuple(iterable), n)) def dotproduct(vec1, vec2): - return sum(map(operator.mul, vec1, vec2)) + "Compute a sum of products." + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) def convolve(signal, kernel): # See: https://betterexplained.com/articles/intuitive-convolution/ @@ -846,7 +847,7 @@ which incur interpreter overhead. window = collections.deque([0], maxlen=n) * n for x in chain(signal, repeat(0, n-1)): window.append(x) - yield sum(map(operator.mul, kernel, window)) + yield dotproduct(kernel, window) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. From webhook-mailer at python.org Fri Dec 23 19:00:27 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 00:00:27 -0000 Subject: [Python-checkins] Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) Message-ID: <mailman.3048.1671840028.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9477594374ba58bc00a95d99d0e2e22cccd02f5e commit: 9477594374ba58bc00a95d99d0e2e22cccd02f5e branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T16:00:21-08:00 summary: Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) (cherry picked from commit f89de679ffec35e82548341cb23e675546602288) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 8eb843ab0a67..371f7c1fdcd4 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -795,7 +795,8 @@ which incur interpreter overhead. return chain.from_iterable(repeat(tuple(iterable), n)) def dotproduct(vec1, vec2): - return sum(map(operator.mul, vec1, vec2)) + "Compute a sum of products." + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) def convolve(signal, kernel): # See: https://betterexplained.com/articles/intuitive-convolution/ @@ -807,7 +808,7 @@ which incur interpreter overhead. window = collections.deque([0], maxlen=n) * n for x in chain(signal, repeat(0, n-1)): window.append(x) - yield sum(map(operator.mul, kernel, window)) + yield dotproduct(kernel, window) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. From webhook-mailer at python.org Fri Dec 23 19:03:36 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 00:03:36 -0000 Subject: [Python-checkins] gh-94808: improve test coverage of number formatting (#99472) Message-ID: <mailman.3049.1671840217.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7ca45e5ddd493411e61706d07679ea54b954e41b commit: 7ca45e5ddd493411e61706d07679ea54b954e41b branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T18:03:31-06:00 summary: gh-94808: improve test coverage of number formatting (#99472) files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 79360f5549e4..41094a7e917d 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1307,6 +1307,20 @@ def __repr__(self): self.assertRaises(ValueError, ("{" + big + "}").format) self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) + # test number formatter errors: + self.assertRaises(ValueError, '{0:x}'.format, 1j) + self.assertRaises(ValueError, '{0:x}'.format, 1.0) + self.assertRaises(ValueError, '{0:X}'.format, 1j) + self.assertRaises(ValueError, '{0:X}'.format, 1.0) + self.assertRaises(ValueError, '{0:o}'.format, 1j) + self.assertRaises(ValueError, '{0:o}'.format, 1.0) + self.assertRaises(ValueError, '{0:u}'.format, 1j) + self.assertRaises(ValueError, '{0:u}'.format, 1.0) + self.assertRaises(ValueError, '{0:i}'.format, 1j) + self.assertRaises(ValueError, '{0:i}'.format, 1.0) + self.assertRaises(ValueError, '{0:d}'.format, 1j) + self.assertRaises(ValueError, '{0:d}'.format, 1.0) + # issue 6089 self.assertRaises(ValueError, "{0[0]x}".format, [None]) self.assertRaises(ValueError, "{0[0](10)}".format, [None]) @@ -1541,11 +1555,31 @@ def __int__(self): self.assertEqual('%X' % letter_m, '6D') self.assertEqual('%o' % letter_m, '155') self.assertEqual('%c' % letter_m, 'm') - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14), - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11), - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79), - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi), - self.assertRaises(TypeError, operator.mod, '%c', pi), + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) + self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) + self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) + self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) + self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + + class RaisingNumber: + def __int__(self): + raise RuntimeError('int') # should not be `TypeError` + def __index__(self): + raise RuntimeError('index') # should not be `TypeError` + + rn = RaisingNumber() + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) def test_formatting_with_enum(self): # issue18780 From webhook-mailer at python.org Fri Dec 23 21:04:26 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:04:26 -0000 Subject: [Python-checkins] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100456) Message-ID: <mailman.3050.1671847467.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a23cb72ac82372fac05ba36ce08923840ca0de06 commit: a23cb72ac82372fac05ba36ce08923840ca0de06 branch: main author: Illia Volochii <illia.volochii at gmail.com> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:04:20-08:00 summary: gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100456) files: A Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1bdfa0681e0..f798992d8af6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,7 +235,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1s, 3.0.7] + openssl_ver: [1.1.1s, 3.0.7, 3.1.0-beta1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst new file mode 100644 index 000000000000..8b08ca0dcef7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst @@ -0,0 +1 @@ +Start running SSL tests with OpenSSL 3.1.0-beta1. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 30d66964fd1d..5ad597c8347e 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -402,15 +402,15 @@ class BuildOpenSSL(AbstractBuilder): depend_target = 'depend' def _post_install(self): - if self.version.startswith("3.0"): - self._post_install_300() + if self.version.startswith("3."): + self._post_install_3xx() def _build_src(self, config_args=()): - if self.version.startswith("3.0"): + if self.version.startswith("3."): config_args += ("enable-fips",) super()._build_src(config_args) - def _post_install_300(self): + def _post_install_3xx(self): # create ssl/ subdir with example configs # Install FIPS module self._subprocess_call( From webhook-mailer at python.org Fri Dec 23 21:07:57 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:07:57 -0000 Subject: [Python-checkins] [3.11] Correct CVE-2020-10735 documentation (GH-100306). (#100476) Message-ID: <mailman.3051.1671847678.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a852c5f8ee19f996335841a78b73549d7d4117c4 commit: a852c5f8ee19f996335841a78b73549d7d4117c4 branch: 3.11 author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:07:50-08:00 summary: [3.11] Correct CVE-2020-10735 documentation (GH-100306). (#100476) (cherry picked from commit 1cf3d78c92eb07dc09d15cc2e773b0b1b9436825) Co-authored-by: Jeremy Paige <ucodery at gmail.com> files: M Doc/library/stdtypes.rst M Python/clinic/sysmodule.c.h M Python/sysmodule.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a1514f611d4f..2bc86e978387 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5476,7 +5476,7 @@ to mitigate denial of service attacks. This limit *only* applies to decimal or other non-power-of-two number bases. Hexadecimal, octal, and binary conversions are unlimited. The limit can be configured. -The :class:`int` type in CPython is an abitrary length number stored in binary +The :class:`int` type in CPython is an arbitrary length number stored in binary form (commonly known as a "bignum"). There exists no algorithm that can convert a string to a binary integer or a binary integer to a string in linear time, *unless* the base is a power of 2. Even the best known algorithms for base 10 @@ -5540,7 +5540,7 @@ and :class:`str` or :class:`bytes`: * ``int(string)`` with default base 10. * ``int(string, base)`` for all bases that are not a power of 2. * ``str(integer)``. -* ``repr(integer)`` +* ``repr(integer)``. * any other string conversion to base 10, for example ``f"{integer}"``, ``"{}".format(integer)``, or ``b"%d" % integer``. @@ -5568,7 +5568,7 @@ command line flag to configure the limit: :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. If both the env var and the ``-X`` option are set, the ``-X`` option takes precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initilization. + :data:`sys.int_info.default_max_str_digits` was used during initialization. From code, you can inspect the current limit and set a new one using these :mod:`sys` APIs: diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index b3ecab59a733..8ca3cd086059 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -673,7 +673,7 @@ PyDoc_STRVAR(sys_get_int_max_str_digits__doc__, "get_int_max_str_digits($module, /)\n" "--\n" "\n" -"Set the maximum string digits limit for non-binary int<->str conversions."); +"Return the maximum string digits limit for non-binary int<->str conversions."); #define SYS_GET_INT_MAX_STR_DIGITS_METHODDEF \ {"get_int_max_str_digits", (PyCFunction)sys_get_int_max_str_digits, METH_NOARGS, sys_get_int_max_str_digits__doc__}, @@ -1067,4 +1067,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=21a32aa71d36a98c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3cae0e0212d88bcd input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0ecfd77ba14c..68a835a30baa 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1627,12 +1627,12 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] sys.get_int_max_str_digits -Set the maximum string digits limit for non-binary int<->str conversions. +Return the maximum string digits limit for non-binary int<->str conversions. [clinic start generated code]*/ static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=8dab13e2023e60d5]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromSsize_t(interp->int_max_str_digits); From webhook-mailer at python.org Fri Dec 23 21:08:32 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:08:32 -0000 Subject: [Python-checkins] [3.10] Correct CVE-2020-10735 documentation (GH-100306). (#100477) Message-ID: <mailman.3052.1671847713.3313.python-checkins@python.org> https://github.com/python/cpython/commit/88fe8d701af3316c8869ea18ea1c7acec6f68c04 commit: 88fe8d701af3316c8869ea18ea1c7acec6f68c04 branch: 3.10 author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:08:27-08:00 summary: [3.10] Correct CVE-2020-10735 documentation (GH-100306). (#100477) (cherry picked from commit 1cf3d78c92eb07dc09d15cc2e773b0b1b9436825) Co-authored-by: Jeremy Paige <ucodery at gmail.com> files: M Doc/library/stdtypes.rst M Python/clinic/sysmodule.c.h M Python/sysmodule.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d71d6936c568..5750c7cb68c8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5427,7 +5427,7 @@ to mitigate denial of service attacks. This limit *only* applies to decimal or other non-power-of-two number bases. Hexadecimal, octal, and binary conversions are unlimited. The limit can be configured. -The :class:`int` type in CPython is an abitrary length number stored in binary +The :class:`int` type in CPython is an arbitrary length number stored in binary form (commonly known as a "bignum"). There exists no algorithm that can convert a string to a binary integer or a binary integer to a string in linear time, *unless* the base is a power of 2. Even the best known algorithms for base 10 @@ -5491,7 +5491,7 @@ and :class:`str` or :class:`bytes`: * ``int(string)`` with default base 10. * ``int(string, base)`` for all bases that are not a power of 2. * ``str(integer)``. -* ``repr(integer)`` +* ``repr(integer)``. * any other string conversion to base 10, for example ``f"{integer}"``, ``"{}".format(integer)``, or ``b"%d" % integer``. @@ -5519,7 +5519,7 @@ command line flag to configure the limit: :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. If both the env var and the ``-X`` option are set, the ``-X`` option takes precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initilization. + :data:`sys.int_info.default_max_str_digits` was used during initialization. From code, you can inspect the current limit and set a new one using these :mod:`sys` APIs: diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 2a6ad89137b0..63c88778555a 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -651,7 +651,7 @@ PyDoc_STRVAR(sys_get_int_max_str_digits__doc__, "get_int_max_str_digits($module, /)\n" "--\n" "\n" -"Set the maximum string digits limit for non-binary int<->str conversions."); +"Return the maximum string digits limit for non-binary int<->str conversions."); #define SYS_GET_INT_MAX_STR_DIGITS_METHODDEF \ {"get_int_max_str_digits", (PyCFunction)sys_get_int_max_str_digits, METH_NOARGS, sys_get_int_max_str_digits__doc__}, @@ -1036,4 +1036,4 @@ sys__deactivate_opcache(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=6230a1e3a4415744 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1d40b6af6e80cc71 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 2b5c9d3ebbe8..e740cf933d1d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1659,12 +1659,12 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] sys.get_int_max_str_digits -Set the maximum string digits limit for non-binary int<->str conversions. +Return the maximum string digits limit for non-binary int<->str conversions. [clinic start generated code]*/ static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=8dab13e2023e60d5]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromSsize_t(interp->int_max_str_digits); From webhook-mailer at python.org Fri Dec 23 21:14:20 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 02:14:20 -0000 Subject: [Python-checkins] gh-94808: improve test coverage of number formatting (GH-99472) Message-ID: <mailman.3053.1671848061.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c4aff6d273aea44265c35ff19b30cc6aea2e06d4 commit: c4aff6d273aea44265c35ff19b30cc6aea2e06d4 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T18:14:08-08:00 summary: gh-94808: improve test coverage of number formatting (GH-99472) (cherry picked from commit 7ca45e5ddd493411e61706d07679ea54b954e41b) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index b3c02c1b9984..c265438a86d5 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1306,6 +1306,20 @@ def __repr__(self): self.assertRaises(ValueError, ("{" + big + "}").format) self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) + # test number formatter errors: + self.assertRaises(ValueError, '{0:x}'.format, 1j) + self.assertRaises(ValueError, '{0:x}'.format, 1.0) + self.assertRaises(ValueError, '{0:X}'.format, 1j) + self.assertRaises(ValueError, '{0:X}'.format, 1.0) + self.assertRaises(ValueError, '{0:o}'.format, 1j) + self.assertRaises(ValueError, '{0:o}'.format, 1.0) + self.assertRaises(ValueError, '{0:u}'.format, 1j) + self.assertRaises(ValueError, '{0:u}'.format, 1.0) + self.assertRaises(ValueError, '{0:i}'.format, 1j) + self.assertRaises(ValueError, '{0:i}'.format, 1.0) + self.assertRaises(ValueError, '{0:d}'.format, 1j) + self.assertRaises(ValueError, '{0:d}'.format, 1.0) + # issue 6089 self.assertRaises(ValueError, "{0[0]x}".format, [None]) self.assertRaises(ValueError, "{0[0](10)}".format, [None]) @@ -1541,11 +1555,31 @@ def __int__(self): self.assertEqual('%X' % letter_m, '6D') self.assertEqual('%o' % letter_m, '155') self.assertEqual('%c' % letter_m, 'm') - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14), - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11), - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79), - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi), - self.assertRaises(TypeError, operator.mod, '%c', pi), + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) + self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) + self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) + self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) + self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + + class RaisingNumber: + def __int__(self): + raise RuntimeError('int') # should not be `TypeError` + def __index__(self): + raise RuntimeError('index') # should not be `TypeError` + + rn = RaisingNumber() + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) def test_formatting_with_enum(self): # issue18780 From webhook-mailer at python.org Fri Dec 23 21:14:20 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 02:14:20 -0000 Subject: [Python-checkins] gh-94808: improve test coverage of number formatting (GH-99472) Message-ID: <mailman.3054.1671848062.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b168b0e013854b9a7d896c72dfc4d220bd036461 commit: b168b0e013854b9a7d896c72dfc4d220bd036461 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T18:14:14-08:00 summary: gh-94808: improve test coverage of number formatting (GH-99472) (cherry picked from commit 7ca45e5ddd493411e61706d07679ea54b954e41b) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index f6a1651e76f7..1b31ae1e0560 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1266,6 +1266,20 @@ def __repr__(self): self.assertRaises(ValueError, ("{" + big + "}").format) self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) + # test number formatter errors: + self.assertRaises(ValueError, '{0:x}'.format, 1j) + self.assertRaises(ValueError, '{0:x}'.format, 1.0) + self.assertRaises(ValueError, '{0:X}'.format, 1j) + self.assertRaises(ValueError, '{0:X}'.format, 1.0) + self.assertRaises(ValueError, '{0:o}'.format, 1j) + self.assertRaises(ValueError, '{0:o}'.format, 1.0) + self.assertRaises(ValueError, '{0:u}'.format, 1j) + self.assertRaises(ValueError, '{0:u}'.format, 1.0) + self.assertRaises(ValueError, '{0:i}'.format, 1j) + self.assertRaises(ValueError, '{0:i}'.format, 1.0) + self.assertRaises(ValueError, '{0:d}'.format, 1j) + self.assertRaises(ValueError, '{0:d}'.format, 1.0) + # issue 6089 self.assertRaises(ValueError, "{0[0]x}".format, [None]) self.assertRaises(ValueError, "{0[0](10)}".format, [None]) @@ -1501,11 +1515,31 @@ def __int__(self): self.assertEqual('%X' % letter_m, '6D') self.assertEqual('%o' % letter_m, '155') self.assertEqual('%c' % letter_m, 'm') - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14), - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11), - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79), - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi), - self.assertRaises(TypeError, operator.mod, '%c', pi), + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) + self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) + self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) + self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) + self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + + class RaisingNumber: + def __int__(self): + raise RuntimeError('int') # should not be `TypeError` + def __index__(self): + raise RuntimeError('index') # should not be `TypeError` + + rn = RaisingNumber() + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) def test_formatting_with_enum(self): # issue18780 From webhook-mailer at python.org Fri Dec 23 21:30:33 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:30:33 -0000 Subject: [Python-checkins] gh-100268: Add is_integer method to int (#100439) Message-ID: <mailman.3055.1671849034.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3e46f9fe05b40ee42009878620f448d3a4b44cb5 commit: 3e46f9fe05b40ee42009878620f448d3a4b44cb5 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:30:27-08:00 summary: gh-100268: Add is_integer method to int (#100439) This improves the lives of type annotation users of `float` - which type checkers implicitly treat as `int|float` because that is what most code actually wants. Before this change a `.is_integer()` method could not be assumed to exist on things annotated as `: float` due to the method not existing on both types. files: A Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst M Doc/library/stdtypes.rst M Lib/test/test_long.py M Objects/clinic/longobject.c.h M Objects/longobject.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 624284f7092f..0ef03035a572 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -609,6 +609,12 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.8 +.. method:: int.is_integer() + + Returns ``True``. Exists for duck type compatibility with :meth:`float.is_integer`. + + .. versionadded:: 3.12 + Additional Methods on Float --------------------------- diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 77b37ca1fa4a..569ab15820e3 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1553,6 +1553,11 @@ def test_from_bytes_small(self): b = i.to_bytes(2, signed=True) self.assertIs(int.from_bytes(b, signed=True), i) + def test_is_integer(self): + self.assertTrue((-1).is_integer()) + self.assertTrue((0).is_integer()) + self.assertTrue((1).is_integer()) + def test_access_to_nonexistent_digit_0(self): # http://bugs.python.org/issue14630: A bug in _PyLong_Copy meant that # ob_digit[0] was being incorrectly accessed for instances of a diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst new file mode 100644 index 000000000000..73d04c19d1cc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst @@ -0,0 +1 @@ +Add :meth:`int.is_integer` to improve duck type compatibility between :class:`int` and :class:`float`. diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index dde49099cf95..206bffdd086a 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -467,4 +467,22 @@ int_from_bytes(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyOb exit: return return_value; } -/*[clinic end generated code: output=bf6074ecf2f32cf4 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(int_is_integer__doc__, +"is_integer($self, /)\n" +"--\n" +"\n" +"Returns True. Exists for duck type compatibility with float.is_integer."); + +#define INT_IS_INTEGER_METHODDEF \ + {"is_integer", (PyCFunction)int_is_integer, METH_NOARGS, int_is_integer__doc__}, + +static PyObject * +int_is_integer_impl(PyObject *self); + +static PyObject * +int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return int_is_integer_impl(self); +} +/*[clinic end generated code: output=e518fe2b5d519322 input=a9049054013a1b77]*/ diff --git a/Objects/longobject.c b/Objects/longobject.c index 8596ce9797b5..0df3b9a9d564 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6168,6 +6168,19 @@ long_long_meth(PyObject *self, PyObject *Py_UNUSED(ignored)) return long_long(self); } +/*[clinic input] +int.is_integer + +Returns True. Exists for duck type compatibility with float.is_integer. +[clinic start generated code]*/ + +static PyObject * +int_is_integer_impl(PyObject *self) +/*[clinic end generated code: output=90f8e794ce5430ef input=7e41c4d4416e05f2]*/ +{ + Py_RETURN_TRUE; +} + static PyMethodDef long_methods[] = { {"conjugate", long_long_meth, METH_NOARGS, "Returns self, the complex conjugate of any int."}, @@ -6186,6 +6199,7 @@ static PyMethodDef long_methods[] = { INT___GETNEWARGS___METHODDEF INT___FORMAT___METHODDEF INT___SIZEOF___METHODDEF + INT_IS_INTEGER_METHODDEF {NULL, NULL} /* sentinel */ }; From webhook-mailer at python.org Sat Dec 24 00:21:59 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 05:21:59 -0000 Subject: [Python-checkins] gh-77771: Add enterabs example in sched (#92716) Message-ID: <mailman.3056.1671859321.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0f6420640c0f3462e6b76b01a392844676de1fb9 commit: 0f6420640c0f3462e6b76b01a392844676de1fb9 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T23:21:52-06:00 summary: gh-77771: Add enterabs example in sched (#92716) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/sched.rst diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index a4ba2848f11d..a051c65b97b0 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -44,16 +44,22 @@ Example:: ... print(time.time()) ... s.enter(10, 1, print_time) ... s.enter(5, 2, print_time, argument=('positional',)) + ... # despite having higher priority, 'keyword' runs after 'positional' as enter() is relative ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) + ... s.enterabs(1_650_000_000, 10, print_time, argument=("first enterabs",)) + ... s.enterabs(1_650_000_000, 5, print_time, argument=("second enterabs",)) ... s.run() ... print(time.time()) ... >>> print_some_times() - 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default - 930343700.276 + 1652342830.3640375 + From print_time 1652342830.3642538 second enterabs + From print_time 1652342830.3643398 first enterabs + From print_time 1652342835.3694863 positional + From print_time 1652342835.3696074 keyword + From print_time 1652342840.369612 default + 1652342840.3697174 + .. _scheduler-objects: From webhook-mailer at python.org Sat Dec 24 00:30:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 05:30:11 -0000 Subject: [Python-checkins] gh-77771: Add enterabs example in sched (GH-92716) Message-ID: <mailman.3057.1671859812.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ecfe468a6ef10e78b1f9ea8418f04bc63d6b62f7 commit: ecfe468a6ef10e78b1f9ea8418f04bc63d6b62f7 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T21:30:06-08:00 summary: gh-77771: Add enterabs example in sched (GH-92716) (cherry picked from commit 0f6420640c0f3462e6b76b01a392844676de1fb9) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/sched.rst diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index a4ba2848f11d..a051c65b97b0 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -44,16 +44,22 @@ Example:: ... print(time.time()) ... s.enter(10, 1, print_time) ... s.enter(5, 2, print_time, argument=('positional',)) + ... # despite having higher priority, 'keyword' runs after 'positional' as enter() is relative ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) + ... s.enterabs(1_650_000_000, 10, print_time, argument=("first enterabs",)) + ... s.enterabs(1_650_000_000, 5, print_time, argument=("second enterabs",)) ... s.run() ... print(time.time()) ... >>> print_some_times() - 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default - 930343700.276 + 1652342830.3640375 + From print_time 1652342830.3642538 second enterabs + From print_time 1652342830.3643398 first enterabs + From print_time 1652342835.3694863 positional + From print_time 1652342835.3696074 keyword + From print_time 1652342840.369612 default + 1652342840.3697174 + .. _scheduler-objects: From webhook-mailer at python.org Sat Dec 24 00:31:24 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 05:31:24 -0000 Subject: [Python-checkins] gh-77771: Add enterabs example in sched (GH-92716) Message-ID: <mailman.3058.1671859886.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b914054d9de574f5b3489f8601482eac5a7699e0 commit: b914054d9de574f5b3489f8601482eac5a7699e0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T21:31:18-08:00 summary: gh-77771: Add enterabs example in sched (GH-92716) (cherry picked from commit 0f6420640c0f3462e6b76b01a392844676de1fb9) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/sched.rst diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index a4ba2848f11d..a051c65b97b0 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -44,16 +44,22 @@ Example:: ... print(time.time()) ... s.enter(10, 1, print_time) ... s.enter(5, 2, print_time, argument=('positional',)) + ... # despite having higher priority, 'keyword' runs after 'positional' as enter() is relative ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) + ... s.enterabs(1_650_000_000, 10, print_time, argument=("first enterabs",)) + ... s.enterabs(1_650_000_000, 5, print_time, argument=("second enterabs",)) ... s.run() ... print(time.time()) ... >>> print_some_times() - 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default - 930343700.276 + 1652342830.3640375 + From print_time 1652342830.3642538 second enterabs + From print_time 1652342830.3643398 first enterabs + From print_time 1652342835.3694863 positional + From print_time 1652342835.3696074 keyword + From print_time 1652342840.369612 default + 1652342840.3697174 + .. _scheduler-objects: From webhook-mailer at python.org Sat Dec 24 00:47:16 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 24 Dec 2022 05:47:16 -0000 Subject: [Python-checkins] [3.11] gh-99110: Initialize frame->previous in init_frame to fix segmentation fault (GH-100182) (#100478) Message-ID: <mailman.3059.1671860837.3313.python-checkins@python.org> https://github.com/python/cpython/commit/57e727af3fda446dc79d65e2d17297d1194892ed commit: 57e727af3fda446dc79d65e2d17297d1194892ed branch: 3.11 author: Bill Fisher <william.w.fisher at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-24T11:17:10+05:30 summary: [3.11] gh-99110: Initialize frame->previous in init_frame to fix segmentation fault (GH-100182) (#100478) (cherry picked from commit 88d565f32a709140664444c6dea20ecd35a10e94) Co-authored-by: Bill Fisher <william.w.fisher at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst M Include/internal/pycore_frame.h M Lib/test/test_capi/test_misc.py M Modules/_testcapimodule.c M Objects/frameobject.c diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index ecc8346f5ef0..4866ea21b998 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -94,7 +94,10 @@ static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); -/* Consumes reference to func */ +/* Consumes reference to func and locals. + Does not initialize frame->previous, which happens + when frame is linked into the frame stack. + */ static inline void _PyFrame_InitializeSpecials( _PyInterpreterFrame *frame, PyFunctionObject *func, diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 6cda91677054..db7a77416219 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1335,6 +1335,16 @@ def test_frame_get_generator(self): frame = next(gen) self.assertIs(gen, _testcapi.frame_getgenerator(frame)) + def test_frame_fback_api(self): + """Test that accessing `f_back` does not cause a segmentation fault on + a frame created with `PyFrame_New` (GH-99110).""" + def dummy(): + pass + + frame = _testcapi.frame_new(dummy.__code__, globals(), locals()) + # The following line should not cause a segmentation fault. + self.assertIsNone(frame.f_back) + SUFFICIENT_TO_DEOPT_AND_SPECIALIZE = 100 diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst new file mode 100644 index 000000000000..175740dfca07 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst @@ -0,0 +1,2 @@ +Initialize frame->previous in frameobject.c to fix a segmentation fault when +accessing frames created by :c:func:`PyFrame_New`. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 70242d78ab8c..bb7de220a2db 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -22,6 +22,7 @@ #include "Python.h" #include "datetime.h" // PyDateTimeAPI +#include "frameobject.h" // PyFrame_New #include "marshal.h" // PyMarshal_WriteLongToFile #include "structmember.h" // PyMemberDef #include <float.h> // FLT_MAX @@ -6000,6 +6001,22 @@ frame_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(lasti); } +static PyObject * +frame_new(PyObject *self, PyObject *args) +{ + PyObject *code, *globals, *locals; + if (!PyArg_ParseTuple(args, "OOO", &code, &globals, &locals)) { + return NULL; + } + if (!PyCode_Check(code)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); + return NULL; + } + PyThreadState *tstate = PyThreadState_Get(); + + return (PyObject *)PyFrame_New(tstate, (PyCodeObject *)code, globals, locals); +} + static PyObject * eval_get_func_name(PyObject *self, PyObject *func) { @@ -6492,6 +6509,7 @@ static PyMethodDef TestMethods[] = { {"frame_getgenerator", frame_getgenerator, METH_O, NULL}, {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL}, {"frame_getlasti", frame_getlasti, METH_O, NULL}, + {"frame_new", frame_new, METH_VARARGS, NULL}, {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f77f5ed39392..1e6bdcfb592d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1010,6 +1010,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) Py_INCREF(func); PyCodeObject *code = (PyCodeObject *)func->func_code; _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus); + frame->previous = NULL; for (Py_ssize_t i = 0; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = NULL; } From webhook-mailer at python.org Sat Dec 24 00:51:53 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 24 Dec 2022 05:51:53 -0000 Subject: [Python-checkins] GH-91166: Implement zero copy writes for `SelectorSocketTransport` in asyncio (#31871) Message-ID: <mailman.3060.1671861115.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2eea9598e39c577e26461789272c93fd96db7956 commit: 2eea9598e39c577e26461789272c93fd96db7956 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-24T11:21:48+05:30 summary: GH-91166: Implement zero copy writes for `SelectorSocketTransport` in asyncio (#31871) Co-authored-by: Guido van Rossum <gvanrossum at gmail.com> files: From webhook-mailer at python.org Sat Dec 24 03:21:35 2022 From: webhook-mailer at python.org (rhettinger) Date: Sat, 24 Dec 2022 08:21:35 -0000 Subject: [Python-checkins] Misc Itertools recipe tweaks (GH-100493) Message-ID: <mailman.3061.1671870097.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0769f957514300a75be51fc6d1b963c8e359208b commit: 0769f957514300a75be51fc6d1b963c8e359208b branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-24T00:21:30-08:00 summary: Misc Itertools recipe tweaks (GH-100493) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9688aae68e6c..b3634aecd10d 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -788,6 +788,11 @@ which incur interpreter overhead. .. testcode:: + import collections + import math + import operator + import random + def take(n, iterable): "Return first n items of the iterable as a list" return list(islice(iterable, n)) @@ -892,6 +897,21 @@ which incur interpreter overhead. data[2] = 1 return iter_index(data, 1) if n > 2 else iter([]) + def factor(n): + "Prime factors of n." + # factor(97) --> 97 + # factor(98) --> 2 7 7 + # factor(99) --> 3 3 11 + for prime in sieve(n+1): + while True: + quotient, remainder = divmod(n, prime) + if remainder: + break + yield prime + n = quotient + if n == 1: + return + def flatten(list_of_lists): "Flatten one level of nesting" return chain.from_iterable(list_of_lists) @@ -1134,11 +1154,6 @@ which incur interpreter overhead. Now, we test all of the itertool recipes - >>> import operator - >>> import collections - >>> import math - >>> import random - >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] @@ -1251,6 +1266,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True + list(factor(0)) + [] + list(factor(1)) + [] + list(factor(2)) + [2] + list(factor(3)) + [3] + list(factor(4)) + [2, 2] + list(factor(5)) + [5] + list(factor(6)) + [2, 3] + list(factor(7)) + [7] + list(factor(8)) + [2, 2, 2] + list(factor(9)) + [3, 3] + list(factor(10)) + [2, 5] + all(math.prod(factor(n)) == n for n in range(1, 1000)) + True + all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + True + all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] From webhook-mailer at python.org Sat Dec 24 03:31:47 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 08:31:47 -0000 Subject: [Python-checkins] Misc Itertools recipe tweaks (GH-100493) Message-ID: <mailman.3062.1671870709.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ba87dae4536bbc55db559950e0b8fa1201086586 commit: ba87dae4536bbc55db559950e0b8fa1201086586 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T00:31:36-08:00 summary: Misc Itertools recipe tweaks (GH-100493) (cherry picked from commit 0769f957514300a75be51fc6d1b963c8e359208b) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 371f7c1fdcd4..219ad95c76a1 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -749,6 +749,11 @@ which incur interpreter overhead. .. testcode:: + import collections + import math + import operator + import random + def take(n, iterable): "Return first n items of the iterable as a list" return list(islice(iterable, n)) @@ -853,6 +858,21 @@ which incur interpreter overhead. data[2] = 1 return iter_index(data, 1) if n > 2 else iter([]) + def factor(n): + "Prime factors of n." + # factor(97) --> 97 + # factor(98) --> 2 7 7 + # factor(99) --> 3 3 11 + for prime in sieve(n+1): + while True: + quotient, remainder = divmod(n, prime) + if remainder: + break + yield prime + n = quotient + if n == 1: + return + def flatten(list_of_lists): "Flatten one level of nesting" return chain.from_iterable(list_of_lists) @@ -1104,11 +1124,6 @@ which incur interpreter overhead. Now, we test all of the itertool recipes - >>> import operator - >>> import collections - >>> import math - >>> import random - >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] @@ -1221,6 +1236,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True + list(factor(0)) + [] + list(factor(1)) + [] + list(factor(2)) + [2] + list(factor(3)) + [3] + list(factor(4)) + [2, 2] + list(factor(5)) + [5] + list(factor(6)) + [2, 3] + list(factor(7)) + [7] + list(factor(8)) + [2, 2, 2] + list(factor(9)) + [3, 3] + list(factor(10)) + [2, 5] + all(math.prod(factor(n)) == n for n in range(1, 1000)) + True + all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + True + all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] From webhook-mailer at python.org Sat Dec 24 09:46:16 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Sat, 24 Dec 2022 14:46:16 -0000 Subject: [Python-checkins] gh-100357: Convert several functions in `bltinsmodule` to AC (#100358) Message-ID: <mailman.3063.1671893177.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bdfb6943861431a79e63f0da2e6b3fe163c12bc7 commit: bdfb6943861431a79e63f0da2e6b3fe163c12bc7 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-24T06:45:47-08:00 summary: gh-100357: Convert several functions in `bltinsmodule` to AC (#100358) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst M Python/bltinmodule.c M Python/clinic/bltinmodule.c.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst new file mode 100644 index 000000000000..fb25de6c9a3c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst @@ -0,0 +1,2 @@ +Convert ``vars``, ``dir``, ``next``, ``getattr``, and ``iter`` to argument +clinic. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2d4822e6d468..9ebe4c8353d0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -837,31 +837,33 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, return result; } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +dir as builtin_dir + + arg: object = NULL + / + +Show attributes of an object. + +If called without an argument, return the names in the current scope. +Else, return an alphabetized list of names comprising (some of) the attributes +of the given object, and of attributes reachable from it. +If the object supplies a method named __dir__, it will be used; otherwise +the default dir() logic is used and returns: + for a module object: the module's attributes. + for a class object: its attributes, and recursively the attributes + of its bases. + for any other object: its attributes, its class's attributes, and + recursively the attributes of its class's base classes. +[clinic start generated code]*/ + static PyObject * -builtin_dir(PyObject *self, PyObject *args) +builtin_dir_impl(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=24f2c7a52c1e3b08 input=ed6d6ccb13d52251]*/ { - PyObject *arg = NULL; - - if (!PyArg_UnpackTuple(args, "dir", 0, 1, &arg)) - return NULL; return PyObject_Dir(arg); } -PyDoc_STRVAR(dir_doc, -"dir([object]) -> list of strings\n" -"\n" -"If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" -" for a module object: the module's attributes.\n" -" for a class object: its attributes, and recursively the attributes\n" -" of its bases.\n" -" for any other object: its attributes, its class's attributes, and\n" -" recursively the attributes of its class's base classes."); - /*[clinic input] divmod as builtin_divmod @@ -1109,36 +1111,39 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +getattr as builtin_getattr + + object: object + name: object + default: object = NULL + / + +Get a named attribute from an object. + +getattr(x, 'y') is equivalent to x.y +When a default argument is given, it is returned when the attribute doesn't +exist; without it, an exception is raised in that case. +[clinic start generated code]*/ + static PyObject * -builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=74ad0e225e3f701c input=d7562cd4c3556171]*/ { - PyObject *v, *name, *result; - - if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) - return NULL; + PyObject *result; - v = args[0]; - name = args[1]; - if (nargs > 2) { - if (_PyObject_LookupAttr(v, name, &result) == 0) { - PyObject *dflt = args[2]; - return Py_NewRef(dflt); + if (default_value != NULL) { + if (_PyObject_LookupAttr(object, name, &result) == 0) { + return Py_NewRef(default_value); } } else { - result = PyObject_GetAttr(v, name); + result = PyObject_GetAttr(object, name); } return result; } -PyDoc_STRVAR(getattr_doc, -"getattr(object, name[, default]) -> value\n\ -\n\ -Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ -When a default argument is given, it is returned when the attribute doesn't\n\ -exist; without it, an exception is raised in that case."); - /*[clinic input] globals as builtin_globals @@ -1450,34 +1455,43 @@ PyTypeObject PyMap_Type = { }; -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +next as builtin_next + + iterator: object + default: object = NULL + / + +Return the next item from the iterator. + +If default is given and the iterator is exhausted, +it is returned instead of raising StopIteration. +[clinic start generated code]*/ + static PyObject * -builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_next_impl(PyObject *module, PyObject *iterator, + PyObject *default_value) +/*[clinic end generated code: output=a38a94eeb447fef9 input=180f9984f182020f]*/ { - PyObject *it, *res; - - if (!_PyArg_CheckPositional("next", nargs, 1, 2)) - return NULL; + PyObject *res; - it = args[0]; - if (!PyIter_Check(it)) { + if (!PyIter_Check(iterator)) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", - Py_TYPE(it)->tp_name); + Py_TYPE(iterator)->tp_name); return NULL; } - res = (*Py_TYPE(it)->tp_iternext)(it); + res = (*Py_TYPE(iterator)->tp_iternext)(iterator); if (res != NULL) { return res; - } else if (nargs > 1) { - PyObject *def = args[1]; + } else if (default_value != NULL) { if (PyErr_Occurred()) { if(!PyErr_ExceptionMatches(PyExc_StopIteration)) return NULL; PyErr_Clear(); } - return Py_NewRef(def); + return Py_NewRef(default_value); } else if (PyErr_Occurred()) { return NULL; } else { @@ -1486,12 +1500,6 @@ builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } } -PyDoc_STRVAR(next_doc, -"next(iterator[, default])\n\ -\n\ -Return the next item from the iterator. If default is given and the iterator\n\ -is exhausted, it is returned instead of raising StopIteration."); - /*[clinic input] setattr as builtin_setattr @@ -1584,34 +1592,33 @@ builtin_hex(PyObject *module, PyObject *number) } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +iter as builtin_iter + + object: object + sentinel: object = NULL + / + +Get an iterator from an object. + +In the first form, the argument must supply its own iterator, or be a sequence. +In the second form, the callable is called until it returns the sentinel. +[clinic start generated code]*/ + static PyObject * -builtin_iter(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel) +/*[clinic end generated code: output=12cf64203c195a94 input=a5d64d9d81880ba6]*/ { - PyObject *v; - - if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) - return NULL; - v = args[0]; - if (nargs == 1) - return PyObject_GetIter(v); - if (!PyCallable_Check(v)) { + if (sentinel == NULL) + return PyObject_GetIter(object); + if (!PyCallable_Check(object)) { PyErr_SetString(PyExc_TypeError, - "iter(v, w): v must be callable"); + "iter(object, sentinel): object must be callable"); return NULL; } - PyObject *sentinel = args[1]; - return PyCallIter_New(v, sentinel); + return PyCallIter_New(object, sentinel); } -PyDoc_STRVAR(iter_doc, -"iter(iterable) -> iterator\n\ -iter(callable, sentinel) -> iterator\n\ -\n\ -Get an iterator from an object. In the first form, the argument must\n\ -supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the sentinel."); - /*[clinic input] aiter as builtin_aiter @@ -2390,20 +2397,29 @@ builtin_sorted(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +vars as builtin_vars + + object: object = NULL + / + +Show vars. + +Without arguments, equivalent to locals(). +With an argument, equivalent to object.__dict__. +[clinic start generated code]*/ + static PyObject * -builtin_vars(PyObject *self, PyObject *args) +builtin_vars_impl(PyObject *module, PyObject *object) +/*[clinic end generated code: output=840a7f64007a3e0a input=80cbdef9182c4ba3]*/ { - PyObject *v = NULL; PyObject *d; - if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) - return NULL; - if (v == NULL) { + if (object == NULL) { d = Py_XNewRef(PyEval_GetLocals()); } else { - if (_PyObject_LookupAttr(v, &_Py_ID(__dict__), &d) == 0) { + if (_PyObject_LookupAttr(object, &_Py_ID(__dict__), &d) == 0) { PyErr_SetString(PyExc_TypeError, "vars() argument must have __dict__ attribute"); } @@ -2411,12 +2427,6 @@ builtin_vars(PyObject *self, PyObject *args) return d; } -PyDoc_STRVAR(vars_doc, -"vars([object]) -> dictionary\n\ -\n\ -Without arguments, equivalent to locals().\n\ -With an argument, equivalent to object.__dict__."); - /*[clinic input] sum as builtin_sum @@ -2966,12 +2976,12 @@ static PyMethodDef builtin_methods[] = { BUILTIN_CHR_METHODDEF BUILTIN_COMPILE_METHODDEF BUILTIN_DELATTR_METHODDEF - {"dir", builtin_dir, METH_VARARGS, dir_doc}, + BUILTIN_DIR_METHODDEF BUILTIN_DIVMOD_METHODDEF BUILTIN_EVAL_METHODDEF BUILTIN_EXEC_METHODDEF BUILTIN_FORMAT_METHODDEF - {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, getattr_doc}, + BUILTIN_GETATTR_METHODDEF BUILTIN_GLOBALS_METHODDEF BUILTIN_HASATTR_METHODDEF BUILTIN_HASH_METHODDEF @@ -2980,13 +2990,13 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF - {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, iter_doc}, + BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", _PyCFunction_CAST(builtin_max), METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", _PyCFunction_CAST(builtin_min), METH_VARARGS | METH_KEYWORDS, min_doc}, - {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, + BUILTIN_NEXT_METHODDEF BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF @@ -2997,7 +3007,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_SETATTR_METHODDEF BUILTIN_SORTED_METHODDEF BUILTIN_SUM_METHODDEF - {"vars", builtin_vars, METH_VARARGS, vars_doc}, + BUILTIN_VARS_METHODDEF {NULL, NULL}, }; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 89f069dd97f6..baf955558a21 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -386,6 +386,49 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } +PyDoc_STRVAR(builtin_dir__doc__, +"dir($module, arg=<unrepresentable>, /)\n" +"--\n" +"\n" +"Show attributes of an object.\n" +"\n" +"If called without an argument, return the names in the current scope.\n" +"Else, return an alphabetized list of names comprising (some of) the attributes\n" +"of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used; otherwise\n" +"the default dir() logic is used and returns:\n" +" for a module object: the module\'s attributes.\n" +" for a class object: its attributes, and recursively the attributes\n" +" of its bases.\n" +" for any other object: its attributes, its class\'s attributes, and\n" +" recursively the attributes of its class\'s base classes."); + +#define BUILTIN_DIR_METHODDEF \ + {"dir", _PyCFunction_CAST(builtin_dir), METH_FASTCALL, builtin_dir__doc__}, + +static PyObject * +builtin_dir_impl(PyObject *module, PyObject *arg); + +static PyObject * +builtin_dir(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *arg = NULL; + + if (!_PyArg_CheckPositional("dir", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + arg = args[0]; +skip_optional: + return_value = builtin_dir_impl(module, arg); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_divmod__doc__, "divmod($module, x, y, /)\n" "--\n" @@ -546,6 +589,47 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject return return_value; } +PyDoc_STRVAR(builtin_getattr__doc__, +"getattr($module, object, name, default=<unrepresentable>, /)\n" +"--\n" +"\n" +"Get a named attribute from an object.\n" +"\n" +"getattr(x, \'y\') is equivalent to x.y\n" +"When a default argument is given, it is returned when the attribute doesn\'t\n" +"exist; without it, an exception is raised in that case."); + +#define BUILTIN_GETATTR_METHODDEF \ + {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, builtin_getattr__doc__}, + +static PyObject * +builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, + PyObject *default_value); + +static PyObject * +builtin_getattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *object; + PyObject *name; + PyObject *default_value = NULL; + + if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) { + goto exit; + } + object = args[0]; + name = args[1]; + if (nargs < 3) { + goto skip_optional; + } + default_value = args[2]; +skip_optional: + return_value = builtin_getattr_impl(module, object, name, default_value); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_globals__doc__, "globals($module, /)\n" "--\n" @@ -611,6 +695,44 @@ PyDoc_STRVAR(builtin_id__doc__, #define BUILTIN_ID_METHODDEF \ {"id", (PyCFunction)builtin_id, METH_O, builtin_id__doc__}, +PyDoc_STRVAR(builtin_next__doc__, +"next($module, iterator, default=<unrepresentable>, /)\n" +"--\n" +"\n" +"Return the next item from the iterator.\n" +"\n" +"If default is given and the iterator is exhausted,\n" +"it is returned instead of raising StopIteration."); + +#define BUILTIN_NEXT_METHODDEF \ + {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, builtin_next__doc__}, + +static PyObject * +builtin_next_impl(PyObject *module, PyObject *iterator, + PyObject *default_value); + +static PyObject * +builtin_next(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *iterator; + PyObject *default_value = NULL; + + if (!_PyArg_CheckPositional("next", nargs, 1, 2)) { + goto exit; + } + iterator = args[0]; + if (nargs < 2) { + goto skip_optional; + } + default_value = args[1]; +skip_optional: + return_value = builtin_next_impl(module, iterator, default_value); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_setattr__doc__, "setattr($module, obj, name, value, /)\n" "--\n" @@ -702,6 +824,43 @@ PyDoc_STRVAR(builtin_hex__doc__, #define BUILTIN_HEX_METHODDEF \ {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, +PyDoc_STRVAR(builtin_iter__doc__, +"iter($module, object, sentinel=<unrepresentable>, /)\n" +"--\n" +"\n" +"Get an iterator from an object.\n" +"\n" +"In the first form, the argument must supply its own iterator, or be a sequence.\n" +"In the second form, the callable is called until it returns the sentinel."); + +#define BUILTIN_ITER_METHODDEF \ + {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, builtin_iter__doc__}, + +static PyObject * +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel); + +static PyObject * +builtin_iter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *object; + PyObject *sentinel = NULL; + + if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) { + goto exit; + } + object = args[0]; + if (nargs < 2) { + goto skip_optional; + } + sentinel = args[1]; +skip_optional: + return_value = builtin_iter_impl(module, object, sentinel); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_aiter__doc__, "aiter($module, async_iterable, /)\n" "--\n" @@ -1080,6 +1239,41 @@ builtin_round(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } +PyDoc_STRVAR(builtin_vars__doc__, +"vars($module, object=<unrepresentable>, /)\n" +"--\n" +"\n" +"Show vars.\n" +"\n" +"Without arguments, equivalent to locals().\n" +"With an argument, equivalent to object.__dict__."); + +#define BUILTIN_VARS_METHODDEF \ + {"vars", _PyCFunction_CAST(builtin_vars), METH_FASTCALL, builtin_vars__doc__}, + +static PyObject * +builtin_vars_impl(PyObject *module, PyObject *object); + +static PyObject * +builtin_vars(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *object = NULL; + + if (!_PyArg_CheckPositional("vars", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + object = args[0]; +skip_optional: + return_value = builtin_vars_impl(module, object); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_sum__doc__, "sum($module, iterable, /, start=0)\n" "--\n" @@ -1215,4 +1409,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=973da43fa65aa727 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a6a8efe82cf8b81 input=a9049054013a1b77]*/ From webhook-mailer at python.org Sat Dec 24 09:48:48 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Sat, 24 Dec 2022 14:48:48 -0000 Subject: [Python-checkins] Remove wrong comment about `repr` in `test_unicode` (#100495) Message-ID: <mailman.3064.1671893329.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c6dac128612df08fbd7f5e91dcd74a367bce0ed9 commit: c6dac128612df08fbd7f5e91dcd74a367bce0ed9 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-24T06:48:43-08:00 summary: Remove wrong comment about `repr` in `test_unicode` (#100495) files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 41094a7e917d..4ebbb9d32a3d 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -94,7 +94,6 @@ def test_literals(self): self.assertNotEqual(r"\u0020", " ") def test_ascii(self): - # Test basic sanity of repr() self.assertEqual(ascii('abc'), "'abc'") self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") self.assertEqual(ascii('ab\\'), "'ab\\\\'") From webhook-mailer at python.org Sat Dec 24 10:23:30 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 24 Dec 2022 15:23:30 -0000 Subject: [Python-checkins] gh-99908: Tutorial: Modernize the 'data-record class' example (#100499) Message-ID: <mailman.3065.1671895410.3313.python-checkins@python.org> https://github.com/python/cpython/commit/00afa5066bd45348ed82a38d3442763b2ed1a068 commit: 00afa5066bd45348ed82a38d3442763b2ed1a068 branch: main author: JosephSBoyle <48555120+JosephSBoyle at users.noreply.github.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-24T15:23:24Z summary: gh-99908: Tutorial: Modernize the 'data-record class' example (#100499) Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 0e5a9402bc50..a206ba371976 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -738,18 +738,24 @@ Odds and Ends ============= Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. An empty class definition -will do nicely:: +"struct", bundling together a few named data items. The idiomatic approach +is to use :mod:`dataclasses` for this purpose:: - class Employee: - pass + from dataclasses import dataclasses - john = Employee() # Create an empty employee record + @dataclass + class Employee: + name: str + dept: str + salary: int - # Fill the fields of the record - john.name = 'John Doe' - john.dept = 'computer lab' - john.salary = 1000 +:: + + >>> john = Employee('john', 'computer lab', 1000) + >>> john.dept + 'computer lab' + >>> john.salary + 1000 A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For From webhook-mailer at python.org Sat Dec 24 10:32:05 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 15:32:05 -0000 Subject: [Python-checkins] gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) Message-ID: <mailman.3066.1671895926.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a7eee898abb6314669d700e9a22013f2210b1b7e commit: a7eee898abb6314669d700e9a22013f2210b1b7e branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T07:32:00-08:00 summary: gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) (cherry picked from commit 00afa5066bd45348ed82a38d3442763b2ed1a068) Co-authored-by: JosephSBoyle <48555120+JosephSBoyle at users.noreply.github.com> Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index d7a24b4893fb..5abb767cb0a8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -737,18 +737,24 @@ Odds and Ends ============= Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. An empty class definition -will do nicely:: +"struct", bundling together a few named data items. The idiomatic approach +is to use :mod:`dataclasses` for this purpose:: - class Employee: - pass + from dataclasses import dataclasses - john = Employee() # Create an empty employee record + @dataclass + class Employee: + name: str + dept: str + salary: int - # Fill the fields of the record - john.name = 'John Doe' - john.dept = 'computer lab' - john.salary = 1000 +:: + + >>> john = Employee('john', 'computer lab', 1000) + >>> john.dept + 'computer lab' + >>> john.salary + 1000 A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For From webhook-mailer at python.org Sat Dec 24 10:32:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 15:32:34 -0000 Subject: [Python-checkins] gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) Message-ID: <mailman.3067.1671895956.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3ea6f7fee883faf5f4cf8abfff62deb986b89d53 commit: 3ea6f7fee883faf5f4cf8abfff62deb986b89d53 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T07:32:29-08:00 summary: gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) (cherry picked from commit 00afa5066bd45348ed82a38d3442763b2ed1a068) Co-authored-by: JosephSBoyle <48555120+JosephSBoyle at users.noreply.github.com> Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index d7a24b4893fb..5abb767cb0a8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -737,18 +737,24 @@ Odds and Ends ============= Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. An empty class definition -will do nicely:: +"struct", bundling together a few named data items. The idiomatic approach +is to use :mod:`dataclasses` for this purpose:: - class Employee: - pass + from dataclasses import dataclasses - john = Employee() # Create an empty employee record + @dataclass + class Employee: + name: str + dept: str + salary: int - # Fill the fields of the record - john.name = 'John Doe' - john.dept = 'computer lab' - john.salary = 1000 +:: + + >>> john = Employee('john', 'computer lab', 1000) + >>> john.dept + 'computer lab' + >>> john.salary + 1000 A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For From webhook-mailer at python.org Sat Dec 24 13:29:06 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 18:29:06 -0000 Subject: [Python-checkins] gh-100474: Fix handling of dirs named index.html in http.server (GH-100475) Message-ID: <mailman.3068.1671906546.3313.python-checkins@python.org> https://github.com/python/cpython/commit/46e6a28308def2c3a71c679a6fa4ed7d520802b9 commit: 46e6a28308def2c3a71c679a6fa4ed7d520802b9 branch: main author: James Frost <git at frost.cx> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T10:28:59-08:00 summary: gh-100474: Fix handling of dirs named index.html in http.server (GH-100475) If you had a directory called index.html or index.htm within a directory, it would cause http.server to return a 404 Not Found error instead of the directory listing. This came about due to not checking that the index was a regular file. I have also added a test case for this situation. Automerge-Triggered-By: GH:merwok files: A Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 8acabff605e7..221c8be4ae4b 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -711,7 +711,7 @@ def send_head(self): return None for index in self.index_pages: index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index ca078862cca6..cbcf94136ac4 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -489,6 +489,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: diff --git a/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst new file mode 100644 index 000000000000..31abfb8b87fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst @@ -0,0 +1,2 @@ +:mod:`http.server` now checks that an index page is actually a regular file before trying +to serve it. This avoids issues with directories named ``index.html``. From webhook-mailer at python.org Sat Dec 24 14:39:44 2022 From: webhook-mailer at python.org (cjw296) Date: Sat, 24 Dec 2022 19:39:44 -0000 Subject: [Python-checkins] gh-100287: Fix unittest.mock.seal with AsyncMock (#100496) Message-ID: <mailman.3069.1671910785.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e4b43ebb3afbd231a4e5630e7e358aa3093f8677 commit: e4b43ebb3afbd231a4e5630e7e358aa3093f8677 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: cjw296 <chris at withers.org> date: 2022-12-24T19:39:39Z summary: gh-100287: Fix unittest.mock.seal with AsyncMock (#100496) files: A Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst M Lib/test/test_unittest/testmock/testasync.py M Lib/unittest/mock.py diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 52a3b71be1ef..471162dc5050 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -11,7 +11,7 @@ from asyncio import run, iscoroutinefunction from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, - create_autospec, sentinel, _CallList) + create_autospec, sentinel, _CallList, seal) def tearDownModule(): @@ -300,6 +300,14 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_normal_methods_on_class_with_mock_seal(self): + mock = Mock(AsyncClass) + seal(mock) + with self.assertRaises(AttributeError): + mock.normal_method + with self.assertRaises(AttributeError): + mock.async_method + def test_spec_async_attributes_instance(self): async_instance = AsyncClass() async_instance.async_func_attr = async_func @@ -1089,3 +1097,7 @@ async def f(x=None): pass 'Actual: [call(1)]'))) as cm: self.mock.assert_has_awaits([call(), call(1, 2)]) self.assertIsInstance(cm.exception.__cause__, TypeError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 583ab74a8255..994947cad518 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1019,15 +1019,15 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" - _new_name = kw.get("_new_name") - if _new_name in self.__dict__['_spec_asyncs']: - return AsyncMock(**kw) - if self._mock_sealed: attribute = f".{kw['name']}" if "name" in kw else "()" mock_name = self._extract_mock_name() + attribute raise AttributeError(mock_name) + _new_name = kw.get("_new_name") + if _new_name in self.__dict__['_spec_asyncs']: + return AsyncMock(**kw) + _type = type(self) if issubclass(_type, MagicMock) and _new_name in _async_method_magics: # Any asynchronous magic becomes an AsyncMock diff --git a/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst new file mode 100644 index 000000000000..b353f0810c6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst @@ -0,0 +1 @@ +Fix the interaction of :func:`unittest.mock.seal` with :class:`unittest.mock.AsyncMock`. From webhook-mailer at python.org Sat Dec 24 15:07:20 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Sat, 24 Dec 2022 20:07:20 -0000 Subject: [Python-checkins] gh-99535: Add test for inheritance of annotations and update documentation (#99990) Message-ID: <mailman.3070.1671912441.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f5b7b19bf10724d831285fb04e00f763838bd555 commit: f5b7b19bf10724d831285fb04e00f763838bd555 branch: main author: MonadChains <monadchains at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-24T12:07:14-08:00 summary: gh-99535: Add test for inheritance of annotations and update documentation (#99990) files: M Doc/howto/annotations.rst M Doc/library/typing.rst M Lib/test/test_grammar.py diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 2bc2f2d4c839..472069032d65 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -57,6 +57,12 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer newer is to call :func:`getattr` with three arguments, for example ``getattr(o, '__annotations__', None)``. + Before Python 3.10, accessing ``__annotations__`` on a class that + defines no annotations but that has a parent class with + annotations would return the parent's ``__annotations__``. + In Python 3.10 and newer, the child class's annotations + will be an empty dict instead. + Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 356f919a1897..4eed6b4ea887 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2777,6 +2777,10 @@ Introspection helpers .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. + .. versionchanged:: 3.10 + Calling ``get_type_hints()`` on a class no longer returns the annotations + of its base classes. + .. versionchanged:: 3.11 Previously, ``Optional[t]`` was added for function and method annotations if a default value equal to ``None`` was set. diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 58f907eac09d..5b946020994e 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -415,6 +415,28 @@ class Cbad2(C): x: int x.y: list = [] + def test_annotations_inheritance(self): + # Check that annotations are not inherited by derived classes + class A: + attr: int + class B(A): + pass + class C(A): + attr: str + class D: + attr2: int + class E(A, D): + pass + class F(C, A): + pass + self.assertEqual(A.__annotations__, {"attr": int}) + self.assertEqual(B.__annotations__, {}) + self.assertEqual(C.__annotations__, {"attr" : str}) + self.assertEqual(D.__annotations__, {"attr2" : int}) + self.assertEqual(E.__annotations__, {}) + self.assertEqual(F.__annotations__, {}) + + def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod From webhook-mailer at python.org Sat Dec 24 15:09:36 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:09:36 -0000 Subject: [Python-checkins] gh-100428: Make float documentation more accurate (#100437) Message-ID: <mailman.3071.1671912577.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2e1a9ce9890aba748a518a39d01d1ea6d623d0d9 commit: 2e1a9ce9890aba748a518a39d01d1ea6d623d0d9 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:09:31-06:00 summary: gh-100428: Make float documentation more accurate (#100437) Previously, the grammar did not accept `float("10")`. Also implement mdickinson's suggestion of removing the indirection. files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 817c1f858aae..2e988257d5d3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -650,20 +650,23 @@ are always available. They are listed here in alphabetical order. sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the following grammar after leading and trailing - whitespace characters are removed: + input must conform to the ``floatvalue`` production rule in the following + grammar, after leading and trailing whitespace characters are removed: .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - numeric_value: `floatnumber` | `infinity` | `nan` - numeric_string: [`sign`] `numeric_value` - - Here ``floatnumber`` is the form of a Python floating-point literal, - described in :ref:`floating`. Case is not significant, so, for example, - "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for - positive infinity. + digitpart: `digit` (["_"] `digit`)* + number: [`digitpart`] "." `digitpart` | `digitpart` ["."] + exponent: ("e" | "E") ["+" | "-"] `digitpart` + floatnumber: number [`exponent`] + floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + + Here ``digit`` is a Unicode decimal digit (character in the Unicode general + category ``Nd``). Case is not significant, so, for example, "inf", "Inf", + "INFINITY", and "iNfINity" are all acceptable spellings for positive + infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point From webhook-mailer at python.org Sat Dec 24 15:12:10 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 20:12:10 -0000 Subject: [Python-checkins] [3.11] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100486) Message-ID: <mailman.3072.1671912730.3313.python-checkins@python.org> https://github.com/python/cpython/commit/341bdd6589cf11b1d01e8928fde8c180e1a92b5a commit: 341bdd6589cf11b1d01e8928fde8c180e1a92b5a branch: 3.11 author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-24T12:12:04-08:00 summary: [3.11] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100486) [3.11] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (GH-100456). (cherry picked from commit a23cb72ac82372fac05ba36ce08923840ca0de06) Co-authored-by: Illia Volochii <illia.volochii at gmail.com> files: A Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 328714e87714..5647d6b29b13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -268,7 +268,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1q, 3.0.5] + openssl_ver: [1.1.1s, 3.0.7, 3.1.0-beta1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst new file mode 100644 index 000000000000..8b08ca0dcef7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst @@ -0,0 +1 @@ +Start running SSL tests with OpenSSL 3.1.0-beta1. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 91d6f558bc51..b1aacfa550f1 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -403,15 +403,15 @@ class BuildOpenSSL(AbstractBuilder): depend_target = 'depend' def _post_install(self): - if self.version.startswith("3.0"): - self._post_install_300() + if self.version.startswith("3."): + self._post_install_3xx() def _build_src(self, config_args=()): - if self.version.startswith("3.0"): + if self.version.startswith("3."): config_args += ("enable-fips",) super()._build_src(config_args) - def _post_install_300(self): + def _post_install_3xx(self): # create ssl/ subdir with example configs # Install FIPS module self._subprocess_call( From webhook-mailer at python.org Sat Dec 24 15:15:52 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:15:52 -0000 Subject: [Python-checkins] gh-100428: Make float documentation more accurate (GH-100437) Message-ID: <mailman.3073.1671912953.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0dea92409e291cd61135d509204f60c46f6dfc0b commit: 0dea92409e291cd61135d509204f60c46f6dfc0b branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:15:46-08:00 summary: gh-100428: Make float documentation more accurate (GH-100437) Previously, the grammar did not accept `float("10")`. Also implement mdickinson's suggestion of removing the indirection. (cherry picked from commit 2e1a9ce9890aba748a518a39d01d1ea6d623d0d9) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 97641d1b85cb..46e77fdb4155 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -628,20 +628,23 @@ are always available. They are listed here in alphabetical order. sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the following grammar after leading and trailing - whitespace characters are removed: + input must conform to the ``floatvalue`` production rule in the following + grammar, after leading and trailing whitespace characters are removed: .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - numeric_value: `floatnumber` | `infinity` | `nan` - numeric_string: [`sign`] `numeric_value` - - Here ``floatnumber`` is the form of a Python floating-point literal, - described in :ref:`floating`. Case is not significant, so, for example, - "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for - positive infinity. + digitpart: `digit` (["_"] `digit`)* + number: [`digitpart`] "." `digitpart` | `digitpart` ["."] + exponent: ("e" | "E") ["+" | "-"] `digitpart` + floatnumber: number [`exponent`] + floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + + Here ``digit`` is a Unicode decimal digit (character in the Unicode general + category ``Nd``). Case is not significant, so, for example, "inf", "Inf", + "INFINITY", and "iNfINity" are all acceptable spellings for positive + infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point From webhook-mailer at python.org Sat Dec 24 15:17:44 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 20:17:44 -0000 Subject: [Python-checkins] [Minor PR] Quotes in documentation changed into code blocks (#99536) Message-ID: <mailman.3074.1671913065.3313.python-checkins@python.org> https://github.com/python/cpython/commit/efccd04b9efc1752a845b377399d2068b06d04e7 commit: efccd04b9efc1752a845b377399d2068b06d04e7 branch: main author: Bart Broere <mail at bartbroere.eu> committer: gpshead <greg at krypto.org> date: 2022-12-24T12:17:39-08:00 summary: [Minor PR] Quotes in documentation changed into code blocks (#99536) Minor formatting fix in documentation Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 14414ea7f81e..e4e38e933681 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1567,6 +1567,8 @@ If you ever encounter a presumed highly unusual situation where you need to prevent ``vfork()`` from being used by Python, you can set the :attr:`subprocess._USE_VFORK` attribute to a false value. +:: + subprocess._USE_VFORK = False # See CPython issue gh-NNNNNN. Setting this has no impact on use of ``posix_spawn()`` which could use @@ -1574,6 +1576,8 @@ Setting this has no impact on use of ``posix_spawn()`` which could use :attr:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of that. +:: + subprocess._USE_POSIX_SPAWN = False # See CPython issue gh-NNNNNN. It is safe to set these to false on any Python version. They will have no From webhook-mailer at python.org Sat Dec 24 15:18:46 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:18:46 -0000 Subject: [Python-checkins] gh-100428: Make float documentation more accurate (GH-100437) Message-ID: <mailman.3075.1671913127.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bf0f306bcd4cc6ebe7dcb32c79dd1aba0b910804 commit: bf0f306bcd4cc6ebe7dcb32c79dd1aba0b910804 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:18:40-08:00 summary: gh-100428: Make float documentation more accurate (GH-100437) Previously, the grammar did not accept `float("10")`. Also implement mdickinson's suggestion of removing the indirection. (cherry picked from commit 2e1a9ce9890aba748a518a39d01d1ea6d623d0d9) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 110e7e5d7fb9..304716194d22 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -649,20 +649,23 @@ are always available. They are listed here in alphabetical order. sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the following grammar after leading and trailing - whitespace characters are removed: + input must conform to the ``floatvalue`` production rule in the following + grammar, after leading and trailing whitespace characters are removed: .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - numeric_value: `floatnumber` | `infinity` | `nan` - numeric_string: [`sign`] `numeric_value` - - Here ``floatnumber`` is the form of a Python floating-point literal, - described in :ref:`floating`. Case is not significant, so, for example, - "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for - positive infinity. + digitpart: `digit` (["_"] `digit`)* + number: [`digitpart`] "." `digitpart` | `digitpart` ["."] + exponent: ("e" | "E") ["+" | "-"] `digitpart` + floatnumber: number [`exponent`] + floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + + Here ``digit`` is a Unicode decimal digit (character in the Unicode general + category ``Nd``). Case is not significant, so, for example, "inf", "Inf", + "INFINITY", and "iNfINity" are all acceptable spellings for positive + infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point From webhook-mailer at python.org Sat Dec 24 15:22:54 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:22:54 -0000 Subject: [Python-checkins] gh-100472: Fix docs claim that compileall parameters could be bytes (#100473) Message-ID: <mailman.3076.1671913375.3313.python-checkins@python.org> https://github.com/python/cpython/commit/046cbc2080360b0b0bbe6ea7554045a6bbbd94bd commit: 046cbc2080360b0b0bbe6ea7554045a6bbbd94bd branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:22:49-06:00 summary: gh-100472: Fix docs claim that compileall parameters could be bytes (#100473) files: A Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst M Doc/library/compileall.rst diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 7af46cf32008..180f5b81c2b6 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -199,7 +199,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. @@ -269,7 +269,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst new file mode 100644 index 000000000000..4f4162150750 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst @@ -0,0 +1 @@ +Remove claim in documentation that the ``stripdir``, ``prependdir`` and ``limit_sl_dest`` parameters of :func:`compileall.compile_dir` and :func:`compileall.compile_file` could be :class:`bytes`. From webhook-mailer at python.org Sat Dec 24 15:25:39 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:25:39 -0000 Subject: [Python-checkins] GH-93179: Document the thread safety of functools.lru_cache (GH-95970) Message-ID: <mailman.3077.1671913541.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3b704874592a3a1895c9a69a39c23ea22d1d06ca commit: 3b704874592a3a1895c9a69a39c23ea22d1d06ca branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:25:34-08:00 summary: GH-93179: Document the thread safety of functools.lru_cache (GH-95970) (cherry picked from commit ba4bb7e4649be99d5d6b4151a1bd2eac89ef97f2) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/functools.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 9beebd2a85e5..2f0a9bd8be88 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -49,6 +49,9 @@ The :mod:`functools` module defines the following functions: >>> factorial(12) # makes two new recursive calls, the other 10 are cached 479001600 + The cache is threadsafe so the wrapped function can be used in multiple + threads. + .. versionadded:: 3.9 @@ -140,6 +143,9 @@ The :mod:`functools` module defines the following functions: *maxsize* most recent calls. It can save time when an expensive or I/O bound function is periodically called with the same arguments. + The cache is threadsafe so the wrapped function can be used in multiple + threads. + Since a dictionary is used to cache results, the positional and keyword arguments to the function must be hashable. From webhook-mailer at python.org Sat Dec 24 15:26:16 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:26:16 -0000 Subject: [Python-checkins] [Minor PR] Quotes in documentation changed into code blocks (GH-99536) Message-ID: <mailman.3078.1671913576.3313.python-checkins@python.org> https://github.com/python/cpython/commit/80013d78e0efa0bbb610408227ead50a4f16588d commit: 80013d78e0efa0bbb610408227ead50a4f16588d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:26:11-08:00 summary: [Minor PR] Quotes in documentation changed into code blocks (GH-99536) Minor formatting fix in documentation (cherry picked from commit efccd04b9efc1752a845b377399d2068b06d04e7) Co-authored-by: Bart Broere <mail at bartbroere.eu> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 14414ea7f81e..e4e38e933681 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1567,6 +1567,8 @@ If you ever encounter a presumed highly unusual situation where you need to prevent ``vfork()`` from being used by Python, you can set the :attr:`subprocess._USE_VFORK` attribute to a false value. +:: + subprocess._USE_VFORK = False # See CPython issue gh-NNNNNN. Setting this has no impact on use of ``posix_spawn()`` which could use @@ -1574,6 +1576,8 @@ Setting this has no impact on use of ``posix_spawn()`` which could use :attr:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of that. +:: + subprocess._USE_POSIX_SPAWN = False # See CPython issue gh-NNNNNN. It is safe to set these to false on any Python version. They will have no From webhook-mailer at python.org Sat Dec 24 15:28:46 2022 From: webhook-mailer at python.org (merwok) Date: Sat, 24 Dec 2022 20:28:46 -0000 Subject: [Python-checkins] [3.11] gh-100474: Fix handling of dirs named index.html in http.server (GH-100505) Message-ID: <mailman.3079.1671913727.3313.python-checkins@python.org> https://github.com/python/cpython/commit/714a93f6383042c1c12d9bdf2b5c2cdd7a72c20d commit: 714a93f6383042c1c12d9bdf2b5c2cdd7a72c20d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: merwok <merwok at netwok.org> date: 2022-12-24T15:28:41-05:00 summary: [3.11] gh-100474: Fix handling of dirs named index.html in http.server (GH-100505) Co-authored-by: James Frost <git at frost.cx> files: A Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 058ee47ba10e..eef7cbb3ec75 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -708,7 +708,7 @@ def send_head(self): return None for index in "index.html", "index.htm": index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index ca078862cca6..cbcf94136ac4 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -489,6 +489,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: diff --git a/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst new file mode 100644 index 000000000000..31abfb8b87fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst @@ -0,0 +1,2 @@ +:mod:`http.server` now checks that an index page is actually a regular file before trying +to serve it. This avoids issues with directories named ``index.html``. From webhook-mailer at python.org Sat Dec 24 15:29:27 2022 From: webhook-mailer at python.org (merwok) Date: Sat, 24 Dec 2022 20:29:27 -0000 Subject: [Python-checkins] [3.10] gh-100474: Fix handling of dirs named index.html in http.server (GH-100504) Message-ID: <mailman.3080.1671913768.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ecbf136702267857b261a81b64f234965b9de913 commit: ecbf136702267857b261a81b64f234965b9de913 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: merwok <merwok at netwok.org> date: 2022-12-24T15:29:21-05:00 summary: [3.10] gh-100474: Fix handling of dirs named index.html in http.server (GH-100504) Co-authored-by: James Frost <git at frost.cx> files: A Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 03dbaa51b798..9c218d06acf5 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -709,7 +709,7 @@ def send_head(self): return None for index in "index.html", "index.htm": index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index ac8da494e9bb..a5f787ff48c9 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -488,6 +488,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: diff --git a/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst new file mode 100644 index 000000000000..31abfb8b87fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst @@ -0,0 +1,2 @@ +:mod:`http.server` now checks that an index page is actually a regular file before trying +to serve it. This avoids issues with directories named ``index.html``. From webhook-mailer at python.org Sat Dec 24 15:29:46 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:29:46 -0000 Subject: [Python-checkins] gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) Message-ID: <mailman.3081.1671913787.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ad8d2ef54ffde39c6d59c4fc6c0e9b8c529b306d commit: ad8d2ef54ffde39c6d59c4fc6c0e9b8c529b306d branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:29:41-08:00 summary: gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) (cherry picked from commit 046cbc2080360b0b0bbe6ea7554045a6bbbd94bd) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst M Doc/library/compileall.rst diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index de34664acb84..b020b71dea5f 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -198,7 +198,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. @@ -268,7 +268,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst new file mode 100644 index 000000000000..4f4162150750 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst @@ -0,0 +1 @@ +Remove claim in documentation that the ``stripdir``, ``prependdir`` and ``limit_sl_dest`` parameters of :func:`compileall.compile_dir` and :func:`compileall.compile_file` could be :class:`bytes`. From webhook-mailer at python.org Sat Dec 24 15:29:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:29:55 -0000 Subject: [Python-checkins] gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) Message-ID: <mailman.3082.1671913796.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f801fa8ee5e6f67481b78cce495a9228d82c81c3 commit: f801fa8ee5e6f67481b78cce495a9228d82c81c3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:29:50-08:00 summary: gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) (cherry picked from commit 046cbc2080360b0b0bbe6ea7554045a6bbbd94bd) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst M Doc/library/compileall.rst diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 7af46cf32008..180f5b81c2b6 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -199,7 +199,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. @@ -269,7 +269,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst new file mode 100644 index 000000000000..4f4162150750 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst @@ -0,0 +1 @@ +Remove claim in documentation that the ``stripdir``, ``prependdir`` and ``limit_sl_dest`` parameters of :func:`compileall.compile_dir` and :func:`compileall.compile_file` could be :class:`bytes`. From webhook-mailer at python.org Sat Dec 24 15:31:15 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:31:15 -0000 Subject: [Python-checkins] gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) Message-ID: <mailman.3083.1671913876.3313.python-checkins@python.org> https://github.com/python/cpython/commit/44b664e057772d1491748809a6a356694b3d3979 commit: 44b664e057772d1491748809a6a356694b3d3979 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:31:10-08:00 summary: gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) (cherry picked from commit f5b7b19bf10724d831285fb04e00f763838bd555) Co-authored-by: MonadChains <monadchains at gmail.com> files: M Doc/howto/annotations.rst M Doc/library/typing.rst M Lib/test/test_grammar.py diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 2bc2f2d4c839..472069032d65 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -57,6 +57,12 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer newer is to call :func:`getattr` with three arguments, for example ``getattr(o, '__annotations__', None)``. + Before Python 3.10, accessing ``__annotations__`` on a class that + defines no annotations but that has a parent class with + annotations would return the parent's ``__annotations__``. + In Python 3.10 and newer, the child class's annotations + will be an empty dict instead. + Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 7fc0aa3a8e62..a3a6181eba29 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2767,6 +2767,10 @@ Introspection helpers .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. + .. versionchanged:: 3.10 + Calling ``get_type_hints()`` on a class no longer returns the annotations + of its base classes. + .. versionchanged:: 3.11 Previously, ``Optional[t]`` was added for function and method annotations if a default value equal to ``None`` was set. diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index da8851986271..bcd8d584b0f5 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -415,6 +415,28 @@ class Cbad2(C): x: int x.y: list = [] + def test_annotations_inheritance(self): + # Check that annotations are not inherited by derived classes + class A: + attr: int + class B(A): + pass + class C(A): + attr: str + class D: + attr2: int + class E(A, D): + pass + class F(C, A): + pass + self.assertEqual(A.__annotations__, {"attr": int}) + self.assertEqual(B.__annotations__, {}) + self.assertEqual(C.__annotations__, {"attr" : str}) + self.assertEqual(D.__annotations__, {"attr2" : int}) + self.assertEqual(E.__annotations__, {}) + self.assertEqual(F.__annotations__, {}) + + def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod From webhook-mailer at python.org Sat Dec 24 15:39:25 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:39:25 -0000 Subject: [Python-checkins] [3.11] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100506) Message-ID: <mailman.3084.1671914366.3313.python-checkins@python.org> https://github.com/python/cpython/commit/75b75dfdacc25a0ffc125c2e372036c3f799f5ea commit: 75b75dfdacc25a0ffc125c2e372036c3f799f5ea branch: 3.11 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:39:19-06:00 summary: [3.11] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100506) (cherry picked from commit e4b43ebb3afbd231a4e5630e7e358aa3093f8677) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst M Lib/unittest/mock.py M Lib/unittest/test/testmock/testasync.py diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 6720e5bc22dc..6a1c932081e0 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1014,15 +1014,15 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" - _new_name = kw.get("_new_name") - if _new_name in self.__dict__['_spec_asyncs']: - return AsyncMock(**kw) - if self._mock_sealed: attribute = f".{kw['name']}" if "name" in kw else "()" mock_name = self._extract_mock_name() + attribute raise AttributeError(mock_name) + _new_name = kw.get("_new_name") + if _new_name in self.__dict__['_spec_asyncs']: + return AsyncMock(**kw) + _type = type(self) if issubclass(_type, MagicMock) and _new_name in _async_method_magics: # Any asynchronous magic becomes an AsyncMock diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py index e05a22861d47..df260abde950 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/unittest/test/testmock/testasync.py @@ -11,7 +11,7 @@ from asyncio import run, iscoroutinefunction from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, - create_autospec, sentinel, _CallList) + create_autospec, sentinel, _CallList, seal) def tearDownModule(): @@ -300,6 +300,14 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_normal_methods_on_class_with_mock_seal(self): + mock = Mock(AsyncClass) + seal(mock) + with self.assertRaises(AttributeError): + mock.normal_method + with self.assertRaises(AttributeError): + mock.async_method + def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) @@ -1076,3 +1084,7 @@ async def f(x=None): pass 'Actual: [call(1)]'))) as cm: self.mock.assert_has_awaits([call(), call(1, 2)]) self.assertIsInstance(cm.exception.__cause__, TypeError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst new file mode 100644 index 000000000000..b353f0810c6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst @@ -0,0 +1 @@ +Fix the interaction of :func:`unittest.mock.seal` with :class:`unittest.mock.AsyncMock`. From webhook-mailer at python.org Sat Dec 24 15:39:32 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:39:32 -0000 Subject: [Python-checkins] [3.10] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100508) Message-ID: <mailman.3085.1671914374.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9975d4e7bac052c320880e026f757cddaa91b4bf commit: 9975d4e7bac052c320880e026f757cddaa91b4bf branch: 3.10 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:39:27-06:00 summary: [3.10] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100508) (cherry picked from commit e4b43ebb3afbd231a4e5630e7e358aa3093f8677) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst M Lib/unittest/mock.py M Lib/unittest/test/testmock/testasync.py diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index a647e5dbc857..7832092d4b98 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1010,15 +1010,15 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" - _new_name = kw.get("_new_name") - if _new_name in self.__dict__['_spec_asyncs']: - return AsyncMock(**kw) - if self._mock_sealed: attribute = f".{kw['name']}" if "name" in kw else "()" mock_name = self._extract_mock_name() + attribute raise AttributeError(mock_name) + _new_name = kw.get("_new_name") + if _new_name in self.__dict__['_spec_asyncs']: + return AsyncMock(**kw) + _type = type(self) if issubclass(_type, MagicMock) and _new_name in _async_method_magics: # Any asynchronous magic becomes an AsyncMock diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py index 22228b47dee5..61269d637f30 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/unittest/test/testmock/testasync.py @@ -8,7 +8,7 @@ from asyncio import run, iscoroutinefunction from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, - create_autospec, sentinel, _CallList) + create_autospec, sentinel, _CallList, seal) def tearDownModule(): @@ -298,6 +298,14 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_normal_methods_on_class_with_mock_seal(self): + mock = Mock(AsyncClass) + seal(mock) + with self.assertRaises(AttributeError): + mock.normal_method + with self.assertRaises(AttributeError): + mock.async_method + def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) @@ -1074,3 +1082,7 @@ async def f(x=None): pass 'Actual: [call(1)]'))) as cm: self.mock.assert_has_awaits([call(), call(1, 2)]) self.assertIsInstance(cm.exception.__cause__, TypeError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst new file mode 100644 index 000000000000..b353f0810c6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst @@ -0,0 +1 @@ +Fix the interaction of :func:`unittest.mock.seal` with :class:`unittest.mock.AsyncMock`. From webhook-mailer at python.org Sat Dec 24 19:15:04 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sun, 25 Dec 2022 00:15:04 -0000 Subject: [Python-checkins] gh-100519: simplification to `eff_request_host` in cookiejar.py (#99588) Message-ID: <mailman.3086.1671927305.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b9aa14a484f653cb6a3a242776df9ac5fe161bfc commit: b9aa14a484f653cb6a3a242776df9ac5fe161bfc branch: main author: Glyph <code at glyph.im> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T18:14:51-06:00 summary: gh-100519: simplification to `eff_request_host` in cookiejar.py (#99588) `IPV4_RE` includes a `.`, and the `.find(".") == -1` included here is already testing to make sure there's no dot, so this part of the expression is tautological. Instead use more modern `in` syntax to make it clear what the check is doing here. The simplified implementation more clearly matches the wording in RFC 2965. Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: A Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst M Lib/http/cookiejar.py diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index e3df007033b3..93b10d26c845 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -640,7 +640,7 @@ def eff_request_host(request): """ erhn = req_host = request_host(request) - if req_host.find(".") == -1 and not IPV4_RE.search(req_host): + if "." not in req_host: erhn = req_host + ".local" return req_host, erhn diff --git a/Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst b/Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst new file mode 100644 index 000000000000..6b889b61c274 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst @@ -0,0 +1,2 @@ +Small simplification of :func:`http.cookiejar.eff_request_host` that +improves readability and better matches the RFC wording. From webhook-mailer at python.org Sun Dec 25 01:55:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 25 Dec 2022 06:55:32 -0000 Subject: [Python-checkins] gh-99308: Clarify re docs for byte pattern group names (#99311) Message-ID: <mailman.3087.1671951333.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dbc1e696ebf273bc62545d999eb185d6c9470e71 commit: dbc1e696ebf273bc62545d999eb185d6c9470e71 branch: main author: Ilya Kulakov <kulakov.ilya at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-25T12:25:27+05:30 summary: gh-99308: Clarify re docs for byte pattern group names (#99311) files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index cbee70b01d9f..d0a16b951844 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -395,9 +395,9 @@ The special characters are: ``(?P<name>...)`` Similar to regular parentheses, but the substring matched by the group is accessible via the symbolic group name *name*. Group names must be valid - Python identifiers, and in bytes patterns they must contain only characters - in the ASCII range. Each group name must be defined only once within a - regular expression. A symbolic group is also a numbered group, just as if + Python identifiers, and in :class:`bytes` patterns they can only contain + bytes in the ASCII range. Each group name must be defined only once within + a regular expression. A symbolic group is also a numbered group, just as if the group were not named. Named groups can be referenced in three contexts. If the pattern is @@ -419,8 +419,8 @@ The special characters are: +---------------------------------------+----------------------------------+ .. versionchanged:: 3.12 - In bytes patterns group names must contain only characters in - the ASCII range. + In :class:`bytes` patterns, group *name* can only contain bytes + in the ASCII range (``b'\x00'``-``b'\x7f'``). .. index:: single: (?P=; in regular expressions @@ -496,6 +496,8 @@ The special characters are: .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. + In :class:`bytes` patterns, group *name* can only contain bytes + in the ASCII range (``b'\x00'``-``b'\x7f'``). The special sequences consist of ``'\'`` and a character from the list below. @@ -1018,8 +1020,8 @@ Functions .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. - In bytes replacement strings group names must contain only characters - in the ASCII range. + In :class:`bytes` replacement strings, group *name* can only contain bytes + in the ASCII range (``b'\x00'``-``b'\x7f'``). .. function:: subn(pattern, repl, string, count=0, flags=0) From webhook-mailer at python.org Mon Dec 26 01:23:18 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Mon, 26 Dec 2022 06:23:18 -0000 Subject: [Python-checkins] gh-92446: Improve argparse choices docs; revert bad change to lzma docs (#94627) Message-ID: <mailman.3088.1672035799.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ad3c99e521151680afc65d3f8a7d2167ec1969ad commit: ad3c99e521151680afc65d3f8a7d2167ec1969ad branch: main author: Guy Yagev <yourlefthandman8 at gmail.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-26T00:22:53-06:00 summary: gh-92446: Improve argparse choices docs; revert bad change to lzma docs (#94627) Based on the definition of the collections.abc classes, it is more accurate to use "sequence" instead of "container" when describing argparse choices. A previous attempt at fixing this in #92450 was mistaken; this PR reverts that change. Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/argparse.rst M Doc/library/lzma.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index e6c964864925..475cac70291e 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -765,7 +765,7 @@ The add_argument() method * type_ - The type to which the command-line argument should be converted. - * choices_ - A container of the allowable values for the argument. + * choices_ - A sequence of the allowable values for the argument. * required_ - Whether or not the command-line option may be omitted (optionals only). @@ -1209,7 +1209,7 @@ choices ^^^^^^^ Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword +These can be handled by passing a sequence object as the *choices* keyword argument to :meth:`~ArgumentParser.add_argument`. When the command line is parsed, argument values will be checked, and an error message will be displayed if the argument was not one of the acceptable values:: @@ -1223,9 +1223,9 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* container is checked after any type_ +Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -container should match the type_ specified:: +sequence should match the type_ specified:: >>> parser = argparse.ArgumentParser(prog='doors.py') >>> parser.add_argument('door', type=int, choices=range(1, 4)) @@ -1235,8 +1235,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any container can be passed as the *choices* value, so :class:`list` objects, -:class:`set` objects, and custom containers are all supported. +Any sequence can be passed as the *choices* value, so :class:`list` objects, +:class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index a9311f2a0356..868d4dcfb6c9 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -147,7 +147,7 @@ Compressing and decompressing data in memory This format is more limited than ``.xz`` -- it does not support integrity checks or multiple filters. - * :const:`FORMAT_RAW`: A raw data stream, not using sequences format. + * :const:`FORMAT_RAW`: A raw data stream, not using any container format. This format specifier does not support integrity checks, and requires that you always specify a custom filter chain (for both compression and decompression). Additionally, data compressed in this manner cannot be From webhook-mailer at python.org Mon Dec 26 02:09:11 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Mon, 26 Dec 2022 07:09:11 -0000 Subject: [Python-checkins] [3.11] gh-92446: Improve argparse choices docs (GH-94627) (#100528) Message-ID: <mailman.3089.1672038553.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2cb4b13af6ac975b1a98ddc7d117c3b25398ccf0 commit: 2cb4b13af6ac975b1a98ddc7d117c3b25398ccf0 branch: 3.11 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-26T01:09:06-06:00 summary: [3.11] gh-92446: Improve argparse choices docs (GH-94627) (#100528) Based on the definition of the collections.abc classes, it is more accurate to use "sequence" instead of "container" when describing argparse choices. (cherry picked from commit ad3c99e521) Co-authored-by: Guy Yagev <yourlefthandman8 at gmail.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/argparse.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 67ca6e77bc00..eeb65160bbc8 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -755,7 +755,7 @@ The add_argument() method * type_ - The type to which the command-line argument should be converted. - * choices_ - A container of the allowable values for the argument. + * choices_ - A sequence of the allowable values for the argument. * required_ - Whether or not the command-line option may be omitted (optionals only). @@ -1199,7 +1199,7 @@ choices ^^^^^^^ Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword +These can be handled by passing a sequence object as the *choices* keyword argument to :meth:`~ArgumentParser.add_argument`. When the command line is parsed, argument values will be checked, and an error message will be displayed if the argument was not one of the acceptable values:: @@ -1213,9 +1213,9 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* container is checked after any type_ +Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -container should match the type_ specified:: +sequence should match the type_ specified:: >>> parser = argparse.ArgumentParser(prog='doors.py') >>> parser.add_argument('door', type=int, choices=range(1, 4)) @@ -1225,8 +1225,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any container can be passed as the *choices* value, so :class:`list` objects, -:class:`set` objects, and custom containers are all supported. +Any sequence can be passed as the *choices* value, so :class:`list` objects, +:class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. From webhook-mailer at python.org Mon Dec 26 02:10:54 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Mon, 26 Dec 2022 07:10:54 -0000 Subject: [Python-checkins] [3.10] gh-92446: Improve argparse choices docs (GH-94627) (#100529) Message-ID: <mailman.3090.1672038654.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a167365d8585c373ff3bdff0d48bffb7e74581e5 commit: a167365d8585c373ff3bdff0d48bffb7e74581e5 branch: 3.10 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-26T01:10:48-06:00 summary: [3.10] gh-92446: Improve argparse choices docs (GH-94627) (#100529) Based on the definition of the collections.abc classes, it is more accurate to use "sequence" instead of "container" when describing argparse choices. (cherry picked from commit ad3c99e521) Co-authored-by: Guy Yagev <yourlefthandman8 at gmail.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/argparse.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index bdc2ccebb7bb..3971c000b2a0 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -702,7 +702,7 @@ The add_argument() method * type_ - The type to which the command-line argument should be converted. - * choices_ - A container of the allowable values for the argument. + * choices_ - A sequence of the allowable values for the argument. * required_ - Whether or not the command-line option may be omitted (optionals only). @@ -1124,7 +1124,7 @@ choices ^^^^^^^ Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword +These can be handled by passing a sequence object as the *choices* keyword argument to :meth:`~ArgumentParser.add_argument`. When the command line is parsed, argument values will be checked, and an error message will be displayed if the argument was not one of the acceptable values:: @@ -1138,9 +1138,9 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* container is checked after any type_ +Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -container should match the type_ specified:: +sequence should match the type_ specified:: >>> parser = argparse.ArgumentParser(prog='doors.py') >>> parser.add_argument('door', type=int, choices=range(1, 4)) @@ -1150,8 +1150,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any container can be passed as the *choices* value, so :class:`list` objects, -:class:`set` objects, and custom containers are all supported. +Any sequence can be passed as the *choices* value, so :class:`list` objects, +:class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. From webhook-mailer at python.org Mon Dec 26 06:34:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 11:34:24 -0000 Subject: [Python-checkins] Fix name of removed `inspect.Signature.from_builtin` method in 3.11.0a2 changelog (#100525) Message-ID: <mailman.3091.1672054465.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3ccc98fc24b278e0f8195686f3651c7c9fabeb59 commit: 3ccc98fc24b278e0f8195686f3651c7c9fabeb59 branch: main author: Jakub Kuczys <me at jacken.men> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T17:04:18+05:30 summary: Fix name of removed `inspect.Signature.from_builtin` method in 3.11.0a2 changelog (#100525) files: M Misc/NEWS.d/3.11.0a2.rst diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index 8ae8847d846b..225bd61e90d4 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -618,7 +618,7 @@ Removed from the :mod:`inspect` module: use the :func:`inspect.signature` function and :class:`Signature` object directly. -* the undocumented ``Signature.from_callable`` and ``Signature.from_function`` +* the undocumented ``Signature.from_builtin`` and ``Signature.from_function`` functions, deprecated since Python 3.5; use the :meth:`Signature.from_callable() <inspect.Signature.from_callable>` method instead. From webhook-mailer at python.org Mon Dec 26 06:38:54 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 11:38:54 -0000 Subject: [Python-checkins] gh-100520: Fix `rst` markup in `configparser` docstrings (#100524) Message-ID: <mailman.3092.1672054735.3313.python-checkins@python.org> https://github.com/python/cpython/commit/199507b81a302ea19f93593965b1e5088195a6c5 commit: 199507b81a302ea19f93593965b1e5088195a6c5 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T17:08:49+05:30 summary: gh-100520: Fix `rst` markup in `configparser` docstrings (#100524) files: M Lib/configparser.py diff --git a/Lib/configparser.py b/Lib/configparser.py index f98e6fb01f97..dee5a0db7e7d 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -19,36 +19,37 @@ inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=<unset>, converters=<unset>): - Create the parser. When `defaults' is given, it is initialized into the + + Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. - When `dict_type' is given, it will be used to create the dictionary + When `dict_type` is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. - When `delimiters' is given, it will be used as the set of substrings + When `delimiters` is given, it will be used as the set of substrings that divide keys from values. - When `comment_prefixes' is given, it will be used as the set of + When `comment_prefixes` is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. - When `inline_comment_prefixes' is given, it will be used as the set of + When `inline_comment_prefixes` is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. - When `empty_lines_in_values' is False (default: True), each empty line + When `empty_lines_in_values` is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. - When `allow_no_value' is True (default: False), options without + When `allow_no_value` is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is + When `default_section` is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` @@ -87,7 +88,7 @@ read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `<???>' is used). + messages (if f has no `name` attribute, the string `<???>` is used). read_string(string) Read configuration from a given string. @@ -103,9 +104,9 @@ Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. + provided using the `vars` argument, which must be a dictionary whose + contents override any pre-existing defaults. If `option` is a key in + `vars`, the value from `vars` is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. @@ -134,7 +135,7 @@ write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters + `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. """ @@ -323,7 +324,7 @@ def __init__(self, filename, lineno, line): # Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as +# option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. _UNSET = object() @@ -357,7 +358,7 @@ class BasicInterpolation(Interpolation): would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" + is considered a user error and raises `InterpolationSyntaxError`.""" _KEYCRE = re.compile(r"%$([^)]+)$s") @@ -418,7 +419,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" + `zc.buildout`. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") @@ -691,10 +692,10 @@ def read(self, filenames, encoding=None): def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. - The `f' argument must be iterable, returning one line at a time. - Optional second argument is the `source' specifying the name of the - file being read. If not given, it is taken from f.name. If `f' has no - `name' attribute, `<???>' is used. + The `f` argument must be iterable, returning one line at a time. + Optional second argument is the `source` specifying the name of the + file being read. If not given, it is taken from f.name. If `f` has no + `name` attribute, `<???>` is used. """ if source is None: try: @@ -718,7 +719,7 @@ def read_dict(self, dictionary, source='<dict>'): All types held in the dictionary are converted to strings during reading, including section names, option names and keys. - Optional second argument is the `source' specifying the name of the + Optional second argument is the `source` specifying the name of the dictionary being read. """ elements_added = set() @@ -742,15 +743,15 @@ def read_dict(self, dictionary, source='<dict>'): def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. - If `vars' is provided, it must be a dictionary. The option is looked up - in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. - If the key is not found and `fallback' is provided, it is used as - a fallback value. `None' can be provided as a `fallback' value. + If `vars` is provided, it must be a dictionary. The option is looked up + in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order. + If the key is not found and `fallback` is provided, it is used as + a fallback value. `None` can be provided as a `fallback` value. - If interpolation is enabled and the optional argument `raw' is False, + If interpolation is enabled and the optional argument `raw` is False, all interpolations are expanded in the return values. - Arguments `raw', `vars', and `fallback' are keyword only. + Arguments `raw`, `vars`, and `fallback` are keyword only. The section DEFAULT is special. """ @@ -810,8 +811,8 @@ def items(self, section=_UNSET, raw=False, vars=None): All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument - `raw' is true. Additional substitutions may be provided using the - `vars' argument, which must be a dictionary whose contents overrides + `raw` is true. Additional substitutions may be provided using the + `vars` argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. @@ -853,8 +854,8 @@ def optionxform(self, optionstr): def has_option(self, section, option): """Check for the existence of a given option in a given section. - If the specified `section' is None or an empty string, DEFAULT is - assumed. If the specified `section' does not exist, returns False.""" + If the specified `section` is None or an empty string, DEFAULT is + assumed. If the specified `section` does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults @@ -882,7 +883,7 @@ def set(self, section, option, value=None): def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. - If `space_around_delimiters' is True (the default), delimiters + If `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. Please note that comments in the original configuration file are not @@ -900,7 +901,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified `fp`.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, @@ -974,8 +975,8 @@ def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by - a name in square brackets (`[]'), plus key/value options, indicated by - `name' and `value' delimited with a specific substring (`=' or `:' by + a name in square brackets (`[]`), plus key/value options, indicated by + `name` and `value` delimited with a specific substring (`=` or `:` by default). Values can span multiple lines, as long as they are indented deeper @@ -983,7 +984,7 @@ def _read(self, fp, fpname): lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific - characters (`#' and `;' by default). Comments may appear on their own + characters (`#` and `;` by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ From webhook-mailer at python.org Mon Dec 26 10:43:01 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 15:43:01 -0000 Subject: [Python-checkins] [3.10] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100534) Message-ID: <mailman.3093.1672069382.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bb159b448170b24756db9ddfde0e0a735489fce6 commit: bb159b448170b24756db9ddfde0e0a735489fce6 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T21:12:56+05:30 summary: [3.10] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100534) gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (cherry picked from commit 199507b81a302ea19f93593965b1e5088195a6c5) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/configparser.py diff --git a/Lib/configparser.py b/Lib/configparser.py index 3470624e63f6..785527082d07 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -19,36 +19,37 @@ inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=<unset>, converters=<unset>): - Create the parser. When `defaults' is given, it is initialized into the + + Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. - When `dict_type' is given, it will be used to create the dictionary + When `dict_type` is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. - When `delimiters' is given, it will be used as the set of substrings + When `delimiters` is given, it will be used as the set of substrings that divide keys from values. - When `comment_prefixes' is given, it will be used as the set of + When `comment_prefixes` is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. - When `inline_comment_prefixes' is given, it will be used as the set of + When `inline_comment_prefixes` is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. - When `empty_lines_in_values' is False (default: True), each empty line + When `empty_lines_in_values` is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. - When `allow_no_value' is True (default: False), options without + When `allow_no_value` is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is + When `default_section` is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` @@ -87,7 +88,7 @@ read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `<???>' is used). + messages (if f has no `name` attribute, the string `<???>` is used). read_string(string) Read configuration from a given string. @@ -103,9 +104,9 @@ Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. + provided using the `vars` argument, which must be a dictionary whose + contents override any pre-existing defaults. If `option` is a key in + `vars`, the value from `vars` is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. @@ -134,7 +135,7 @@ write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters + `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. """ @@ -352,7 +353,7 @@ def __init__(self, filename, lineno, line): # Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as +# option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. _UNSET = object() @@ -386,7 +387,7 @@ class BasicInterpolation(Interpolation): would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" + is considered a user error and raises `InterpolationSyntaxError`.""" _KEYCRE = re.compile(r"%$([^)]+)$s") @@ -447,7 +448,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" + `zc.buildout`. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") @@ -706,10 +707,10 @@ def read(self, filenames, encoding=None): def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. - The `f' argument must be iterable, returning one line at a time. - Optional second argument is the `source' specifying the name of the - file being read. If not given, it is taken from f.name. If `f' has no - `name' attribute, `<???>' is used. + The `f` argument must be iterable, returning one line at a time. + Optional second argument is the `source` specifying the name of the + file being read. If not given, it is taken from f.name. If `f` has no + `name` attribute, `<???>` is used. """ if source is None: try: @@ -733,7 +734,7 @@ def read_dict(self, dictionary, source='<dict>'): All types held in the dictionary are converted to strings during reading, including section names, option names and keys. - Optional second argument is the `source' specifying the name of the + Optional second argument is the `source` specifying the name of the dictionary being read. """ elements_added = set() @@ -766,15 +767,15 @@ def readfp(self, fp, filename=None): def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. - If `vars' is provided, it must be a dictionary. The option is looked up - in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. - If the key is not found and `fallback' is provided, it is used as - a fallback value. `None' can be provided as a `fallback' value. + If `vars` is provided, it must be a dictionary. The option is looked up + in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order. + If the key is not found and `fallback` is provided, it is used as + a fallback value. `None` can be provided as a `fallback` value. - If interpolation is enabled and the optional argument `raw' is False, + If interpolation is enabled and the optional argument `raw` is False, all interpolations are expanded in the return values. - Arguments `raw', `vars', and `fallback' are keyword only. + Arguments `raw`, `vars`, and `fallback` are keyword only. The section DEFAULT is special. """ @@ -834,8 +835,8 @@ def items(self, section=_UNSET, raw=False, vars=None): All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument - `raw' is true. Additional substitutions may be provided using the - `vars' argument, which must be a dictionary whose contents overrides + `raw` is true. Additional substitutions may be provided using the + `vars` argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. @@ -877,8 +878,8 @@ def optionxform(self, optionstr): def has_option(self, section, option): """Check for the existence of a given option in a given section. - If the specified `section' is None or an empty string, DEFAULT is - assumed. If the specified `section' does not exist, returns False.""" + If the specified `section` is None or an empty string, DEFAULT is + assumed. If the specified `section` does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults @@ -906,7 +907,7 @@ def set(self, section, option, value=None): def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. - If `space_around_delimiters' is True (the default), delimiters + If `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. Please note that comments in the original configuration file are not @@ -924,7 +925,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified `fp`.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, @@ -998,8 +999,8 @@ def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by - a name in square brackets (`[]'), plus key/value options, indicated by - `name' and `value' delimited with a specific substring (`=' or `:' by + a name in square brackets (`[]`), plus key/value options, indicated by + `name` and `value` delimited with a specific substring (`=` or `:` by default). Values can span multiple lines, as long as they are indented deeper @@ -1007,7 +1008,7 @@ def _read(self, fp, fpname): lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific - characters (`#' and `;' by default). Comments may appear on their own + characters (`#` and `;` by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ From webhook-mailer at python.org Mon Dec 26 10:43:17 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 15:43:17 -0000 Subject: [Python-checkins] [3.11] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100533) Message-ID: <mailman.3094.1672069397.3313.python-checkins@python.org> https://github.com/python/cpython/commit/18b43cf95f2455b63a6dbefb813f48fa85a07c7b commit: 18b43cf95f2455b63a6dbefb813f48fa85a07c7b branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T21:13:11+05:30 summary: [3.11] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100533) gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (cherry picked from commit 199507b81a302ea19f93593965b1e5088195a6c5) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/configparser.py diff --git a/Lib/configparser.py b/Lib/configparser.py index de9ee537ac18..df2d7e335d9d 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -19,36 +19,37 @@ inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=<unset>, converters=<unset>): - Create the parser. When `defaults' is given, it is initialized into the + + Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. - When `dict_type' is given, it will be used to create the dictionary + When `dict_type` is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. - When `delimiters' is given, it will be used as the set of substrings + When `delimiters` is given, it will be used as the set of substrings that divide keys from values. - When `comment_prefixes' is given, it will be used as the set of + When `comment_prefixes` is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. - When `inline_comment_prefixes' is given, it will be used as the set of + When `inline_comment_prefixes` is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. - When `empty_lines_in_values' is False (default: True), each empty line + When `empty_lines_in_values` is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. - When `allow_no_value' is True (default: False), options without + When `allow_no_value` is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is + When `default_section` is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` @@ -87,7 +88,7 @@ read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `<???>' is used). + messages (if f has no `name` attribute, the string `<???>` is used). read_string(string) Read configuration from a given string. @@ -103,9 +104,9 @@ Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. + provided using the `vars` argument, which must be a dictionary whose + contents override any pre-existing defaults. If `option` is a key in + `vars`, the value from `vars` is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. @@ -134,7 +135,7 @@ write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters + `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. """ @@ -352,7 +353,7 @@ def __init__(self, filename, lineno, line): # Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as +# option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. _UNSET = object() @@ -386,7 +387,7 @@ class BasicInterpolation(Interpolation): would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" + is considered a user error and raises `InterpolationSyntaxError`.""" _KEYCRE = re.compile(r"%$([^)]+)$s") @@ -447,7 +448,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" + `zc.buildout`. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") @@ -720,10 +721,10 @@ def read(self, filenames, encoding=None): def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. - The `f' argument must be iterable, returning one line at a time. - Optional second argument is the `source' specifying the name of the - file being read. If not given, it is taken from f.name. If `f' has no - `name' attribute, `<???>' is used. + The `f` argument must be iterable, returning one line at a time. + Optional second argument is the `source` specifying the name of the + file being read. If not given, it is taken from f.name. If `f` has no + `name` attribute, `<???>` is used. """ if source is None: try: @@ -747,7 +748,7 @@ def read_dict(self, dictionary, source='<dict>'): All types held in the dictionary are converted to strings during reading, including section names, option names and keys. - Optional second argument is the `source' specifying the name of the + Optional second argument is the `source` specifying the name of the dictionary being read. """ elements_added = set() @@ -780,15 +781,15 @@ def readfp(self, fp, filename=None): def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. - If `vars' is provided, it must be a dictionary. The option is looked up - in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. - If the key is not found and `fallback' is provided, it is used as - a fallback value. `None' can be provided as a `fallback' value. + If `vars` is provided, it must be a dictionary. The option is looked up + in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order. + If the key is not found and `fallback` is provided, it is used as + a fallback value. `None` can be provided as a `fallback` value. - If interpolation is enabled and the optional argument `raw' is False, + If interpolation is enabled and the optional argument `raw` is False, all interpolations are expanded in the return values. - Arguments `raw', `vars', and `fallback' are keyword only. + Arguments `raw`, `vars`, and `fallback` are keyword only. The section DEFAULT is special. """ @@ -848,8 +849,8 @@ def items(self, section=_UNSET, raw=False, vars=None): All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument - `raw' is true. Additional substitutions may be provided using the - `vars' argument, which must be a dictionary whose contents overrides + `raw` is true. Additional substitutions may be provided using the + `vars` argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. @@ -891,8 +892,8 @@ def optionxform(self, optionstr): def has_option(self, section, option): """Check for the existence of a given option in a given section. - If the specified `section' is None or an empty string, DEFAULT is - assumed. If the specified `section' does not exist, returns False.""" + If the specified `section` is None or an empty string, DEFAULT is + assumed. If the specified `section` does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults @@ -920,7 +921,7 @@ def set(self, section, option, value=None): def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. - If `space_around_delimiters' is True (the default), delimiters + If `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. Please note that comments in the original configuration file are not @@ -938,7 +939,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified `fp`.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, @@ -1012,8 +1013,8 @@ def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by - a name in square brackets (`[]'), plus key/value options, indicated by - `name' and `value' delimited with a specific substring (`=' or `:' by + a name in square brackets (`[]`), plus key/value options, indicated by + `name` and `value` delimited with a specific substring (`=` or `:` by default). Values can span multiple lines, as long as they are indented deeper @@ -1021,7 +1022,7 @@ def _read(self, fp, fpname): lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific - characters (`#' and `;' by default). Comments may appear on their own + characters (`#` and `;` by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ From webhook-mailer at python.org Mon Dec 26 23:44:59 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Tue, 27 Dec 2022 04:44:59 -0000 Subject: [Python-checkins] [3.10] gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) (#100509) Message-ID: <mailman.3095.1672116301.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1ffc67265f1a622751069997e7ca8193f983e1e2 commit: 1ffc67265f1a622751069997e7ca8193f983e1e2 branch: 3.10 author: Jelle Zijlstra <jelle.zijlstra at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-26T20:44:53-08:00 summary: [3.10] gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) (#100509) (cherry picked from commit f5b7b19bf10724d831285fb04e00f763838bd555) Co-authored-by: MonadChains <monadchains at gmail.com> files: M Doc/howto/annotations.rst M Doc/library/typing.rst M Lib/test/test_grammar.py diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 2bc2f2d4c839..472069032d65 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -57,6 +57,12 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer newer is to call :func:`getattr` with three arguments, for example ``getattr(o, '__annotations__', None)``. + Before Python 3.10, accessing ``__annotations__`` on a class that + defines no annotations but that has a parent class with + annotations would return the parent's ``__annotations__``. + In Python 3.10 and newer, the child class's annotations + will be an empty dict instead. + Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index d415f149027f..d3275209e938 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2233,6 +2233,10 @@ Introspection helpers .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. + .. versionchanged:: 3.10 + Calling ``get_type_hints()`` on a class no longer returns the annotations + of its base classes. + .. function:: get_args(tp) .. function:: get_origin(tp) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 2b9f7df0bd25..27b14ab836b6 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -406,6 +406,28 @@ class Cbad2(C): x: int x.y: list = [] + def test_annotations_inheritance(self): + # Check that annotations are not inherited by derived classes + class A: + attr: int + class B(A): + pass + class C(A): + attr: str + class D: + attr2: int + class E(A, D): + pass + class F(C, A): + pass + self.assertEqual(A.__annotations__, {"attr": int}) + self.assertEqual(B.__annotations__, {}) + self.assertEqual(C.__annotations__, {"attr" : str}) + self.assertEqual(D.__annotations__, {"attr2" : int}) + self.assertEqual(E.__annotations__, {}) + self.assertEqual(F.__annotations__, {}) + + def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod From webhook-mailer at python.org Mon Dec 26 23:51:00 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Tue, 27 Dec 2022 04:51:00 -0000 Subject: [Python-checkins] gh-99509: Add `__class_getitem__` to `multiprocessing.queues.Queue` (#99511) Message-ID: <mailman.3096.1672116661.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ce39aaffeef9aa8af54a8554fe7a5609a6bba471 commit: ce39aaffeef9aa8af54a8554fe7a5609a6bba471 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-26T20:50:55-08:00 summary: gh-99509: Add `__class_getitem__` to `multiprocessing.queues.Queue` (#99511) files: A Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst M Lib/multiprocessing/queues.py M Lib/test/test_genericalias.py diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index f37f114a9688..daf9ee94a194 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -280,6 +280,8 @@ def _on_queue_feeder_error(e, obj): import traceback traceback.print_exc() + __class_getitem__ = classmethod(types.GenericAlias) + _sentinel = object() diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 6d0a556b1f7f..9b59d1e3e0aa 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -31,11 +31,15 @@ from multiprocessing.managers import ValueProxy from multiprocessing.pool import ApplyResult from multiprocessing.queues import SimpleQueue as MPSimpleQueue + from multiprocessing.queues import Queue as MPQueue + from multiprocessing.queues import JoinableQueue as MPJoinableQueue except ImportError: # _multiprocessing module is optional ValueProxy = None ApplyResult = None MPSimpleQueue = None + MPQueue = None + MPJoinableQueue = None try: from multiprocessing.shared_memory import ShareableList except ImportError: @@ -130,7 +134,8 @@ class BaseTest(unittest.TestCase): if ctypes is not None: generic_types.extend((ctypes.Array, ctypes.LibraryLoader)) if ValueProxy is not None: - generic_types.extend((ValueProxy, ApplyResult, MPSimpleQueue)) + generic_types.extend((ValueProxy, ApplyResult, + MPSimpleQueue, MPQueue, MPJoinableQueue)) def test_subscriptable(self): for t in self.generic_types: diff --git a/Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst b/Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst new file mode 100644 index 000000000000..634281061cec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst @@ -0,0 +1 @@ +Add :pep:`585` support for :class:`multiprocessing.queues.Queue`. From webhook-mailer at python.org Tue Dec 27 05:56:00 2022 From: webhook-mailer at python.org (corona10) Date: Tue, 27 Dec 2022 10:56:00 -0000 Subject: [Python-checkins] gh-94603: micro optimize list.pop (gh-94604) Message-ID: <mailman.3097.1672138562.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b3da6989524001d707ebb386bfab998452aa57bb commit: b3da6989524001d707ebb386bfab998452aa57bb branch: main author: Pieter Eendebak <pieter.eendebak at gmail.com> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-27T19:55:54+09:00 summary: gh-94603: micro optimize list.pop (gh-94604) files: A Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst M Objects/listobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst new file mode 100644 index 000000000000..de4fe4d6df8c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst @@ -0,0 +1 @@ +Improve performance of ``list.pop`` for small lists. diff --git a/Objects/listobject.c b/Objects/listobject.c index 1d32915b17a1..b093f88a35fc 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1022,21 +1022,29 @@ list_pop_impl(PyListObject *self, Py_ssize_t index) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = self->ob_item[index]; - if (index == Py_SIZE(self) - 1) { - status = list_resize(self, Py_SIZE(self) - 1); - if (status >= 0) - return v; /* and v now owns the reference the list had */ - else - return NULL; + + PyObject **items = self->ob_item; + v = items[index]; + const Py_ssize_t size_after_pop = Py_SIZE(self) - 1; + if (size_after_pop == 0) { + Py_INCREF(v); + status = _list_clear(self); } - Py_INCREF(v); - status = list_ass_slice(self, index, index+1, (PyObject *)NULL); - if (status < 0) { - Py_DECREF(v); + else { + if ((size_after_pop - index) > 0) { + memmove(&items[index], &items[index+1], (size_after_pop - index) * sizeof(PyObject *)); + } + status = list_resize(self, size_after_pop); + } + if (status >= 0) { + return v; // and v now owns the reference the list had + } + else { + // list resize failed, need to restore + memmove(&items[index+1], &items[index], (size_after_pop - index)* sizeof(PyObject *)); + items[index] = v; return NULL; } - return v; } /* Reverse a slice of a list in place, from lo up to (exclusive) hi. */ From webhook-mailer at python.org Tue Dec 27 11:11:48 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 27 Dec 2022 16:11:48 -0000 Subject: [Python-checkins] Remove `NoneType` redefinition from `clinic.py` (#100551) Message-ID: <mailman.3098.1672157509.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b0ea28913e3bf684ef847a71afcdfa8224bab63d commit: b0ea28913e3bf684ef847a71afcdfa8224bab63d branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-27T21:41:39+05:30 summary: Remove `NoneType` redefinition from `clinic.py` (#100551) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index fdf8041e14bb..2fb1902a5b54 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -27,7 +27,6 @@ import types from types import * -NoneType = type(None) # TODO: # @@ -42,7 +41,6 @@ version = '1' -NoneType = type(None) NO_VARARG = "PY_SSIZE_T_MAX" CLINIC_PREFIX = "__clinic_" CLINIC_PREFIXED_ARGS = {"args"} From webhook-mailer at python.org Tue Dec 27 19:58:10 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 28 Dec 2022 00:58:10 -0000 Subject: [Python-checkins] gh-100553: Improve accuracy of sqlite3.Row iter test (#100555) Message-ID: <mailman.3099.1672189092.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3dc48dabd48864039951715816e07986a4828d80 commit: 3dc48dabd48864039951715816e07986a4828d80 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-28T01:58:05+01:00 summary: gh-100553: Improve accuracy of sqlite3.Row iter test (#100555) files: M Lib/test/test_sqlite3/test_factory.py diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 7fdc45ab6924..7c36135ecadc 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -179,8 +179,14 @@ def test_sqlite_row_iter(self): """Checks if the row object is iterable""" self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): """Checks if the row object can be converted to a tuple""" From webhook-mailer at python.org Tue Dec 27 20:10:12 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 28 Dec 2022 01:10:12 -0000 Subject: [Python-checkins] [3.11] gh-64490: Fix bugs in argument clinic varargs processing (GH-32092) (#100368) Message-ID: <mailman.3100.1672189814.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a3dbd4c70e93260432fc024265fdf9222926fdad commit: a3dbd4c70e93260432fc024265fdf9222926fdad branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-28T02:10:06+01:00 summary: [3.11] gh-64490: Fix bugs in argument clinic varargs processing (GH-32092) (#100368) (cherry picked from commit 0da728387c99fe6c127b070f2d250dc5bdd62ee5) files: A Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Python/getargs.c M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 991e82d4f966..217aa4c7e97a 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3368,7 +3368,6 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject static const char * const _keywords[] = {"a", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "test_vararg", 0}; PyObject *argsbuf[2]; - Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; @@ -3387,7 +3386,7 @@ exit: static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=a2baf8c1fade41d2 input=81d33815ad1bae6e]*/ +/*[clinic end generated code: output=ce9334333757f6ea input=81d33815ad1bae6e]*/ /*[clinic input] test_vararg_with_default @@ -3418,7 +3417,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar static const char * const _keywords[] = {"a", "b", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "test_vararg_with_default", 0}; PyObject *argsbuf[3]; - Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; int b = 0; @@ -3447,7 +3446,7 @@ exit: static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=3821d282c29f8616 input=6e110b54acd9b22d]*/ +/*[clinic end generated code: output=32fb19dd6bcf9185 input=6e110b54acd9b22d]*/ /*[clinic input] test_vararg_with_only_defaults diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 3f6cc6069669..a84d24a79844 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -730,6 +730,15 @@ def test_parameters_not_permitted_after_slash_for_now(self): x: int """) + def test_parameters_no_more_than_one_vararg(self): + s = self.parse_function_should_fail(""" +module foo +foo.bar + *vararg1: object + *vararg2: object +""") + self.assertEqual(s, "Error on line 0:\nToo many var args\n") + def test_function_not_at_column_0(self): function = self.parse_function(""" module foo @@ -1222,6 +1231,13 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + def test_posonly_vararg(self): + with self.assertRaises(TypeError): + ac_tester.posonly_vararg() + self.assertEqual(ac_tester.posonly_vararg(1, 2), (1, 2, ())) + self.assertEqual(ac_tester.posonly_vararg(1, b=2), (1, 2, ())) + self.assertEqual(ac_tester.posonly_vararg(1, 2, 3, 4), (1, 2, (3, 4))) + def test_vararg_and_posonly(self): with self.assertRaises(TypeError): ac_tester.vararg_and_posonly() @@ -1229,6 +1245,33 @@ def test_vararg_and_posonly(self): ac_tester.vararg_and_posonly(1, b=2) self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) + def test_vararg(self): + with self.assertRaises(TypeError): + ac_tester.vararg() + with self.assertRaises(TypeError): + ac_tester.vararg(1, b=2) + self.assertEqual(ac_tester.vararg(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_vararg_with_default(self): + with self.assertRaises(TypeError): + ac_tester.vararg_with_default() + self.assertEqual(ac_tester.vararg_with_default(1, b=False), (1, (), False)) + self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4, b=True), (1, (2, 3, 4), True)) + + def test_vararg_with_only_defaults(self): + self.assertEqual(ac_tester.vararg_with_only_defaults(), ((), None)) + self.assertEqual(ac_tester.vararg_with_only_defaults(b=2), ((), 2)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, b=2), ((1, ), 2)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4), ((1, 2, 3, 4), None)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + + def test_gh_32092_oob(self): + ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) + + def test_gh_32092_kw_pass(self): + ac_tester.gh_32092_kw_pass(1, 2, 3) + def test_gh_99233_refcount(self): arg = '*A unique string is not referenced by anywhere else.*' arg_refcount_origin = sys.getrefcount(arg) @@ -1241,5 +1284,6 @@ def test_gh_99240_double_free(self): with self.assertRaisesRegex(TypeError, expected_error): ac_tester.gh_99240_double_free('a', '\0b') + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst new file mode 100644 index 000000000000..4a308a930605 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst @@ -0,0 +1,7 @@ +Argument Clinic varargs bugfixes + +* Fix out-of-bounds error in :c:func:`!_PyArg_UnpackKeywordsWithVararg`. +* Fix incorrect check which allowed more than one varargs in clinic.py. +* Fix miscalculation of ``noptargs`` in generated code. +* Do not generate ``noptargs`` when there is a vararg argument and no optional argument. + diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 56eddfd6fdbf..91fdee24d328 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -950,6 +950,25 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) } +/*[clinic input] +posonly_vararg + + a: object + / + b: object + *args: object + +[clinic start generated code]*/ + +static PyObject * +posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=ee6713acda6b954e input=783427fe7ec2b67a]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + /*[clinic input] vararg_and_posonly @@ -967,6 +986,100 @@ vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) } +/*[clinic input] +vararg + + a: object + *args: object + +[clinic start generated code]*/ + +static PyObject * +vararg_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=91ab7a0efc52dd5e input=02c0f772d05f591e]*/ +{ + return pack_arguments_newref(2, a, args); +} + + +/*[clinic input] +vararg_with_default + + a: object + *args: object + b: bool = False + +[clinic start generated code]*/ + +static PyObject * +vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b) +/*[clinic end generated code: output=182c01035958ce92 input=68cafa6a79f89e36]*/ +{ + PyObject *obj_b = b ? Py_True : Py_False; + return pack_arguments_newref(3, a, args, obj_b); +} + + +/*[clinic input] +vararg_with_only_defaults + + *args: object + b: object = None + +[clinic start generated code]*/ + +static PyObject * +vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b) +/*[clinic end generated code: output=c06b1826d91f2f7b input=678c069bc67550e1]*/ +{ + return pack_arguments_newref(2, args, b); +} + + + +/*[clinic input] +gh_32092_oob + + pos1: object + pos2: object + *varargs: object + kw1: object = None + kw2: object = None + +Proof-of-concept of GH-32092 OOB bug. + +[clinic start generated code]*/ + +static PyObject * +gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, + PyObject *varargs, PyObject *kw1, PyObject *kw2) +/*[clinic end generated code: output=ee259c130054653f input=46d15c881608f8ff]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +gh_32092_kw_pass + + pos: object + *args: object + kw: object = None + +Proof-of-concept of GH-32092 keyword args passing bug. + +[clinic start generated code]*/ + +static PyObject * +gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, + PyObject *kw) +/*[clinic end generated code: output=4a2bbe4f7c8604e9 input=5c0bd5b9079a0cce]*/ +{ + Py_RETURN_NONE; +} + + /*[clinic input] gh_99233_refcount @@ -1046,7 +1159,13 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF + POSONLY_VARARG_METHODDEF VARARG_AND_POSONLY_METHODDEF + VARARG_METHODDEF + VARARG_WITH_DEFAULT_METHODDEF + VARARG_WITH_ONLY_DEFAULTS_METHODDEF + GH_32092_OOB_METHODDEF + GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF {NULL, NULL} diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 222ea783486e..41555d801059 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1952,6 +1952,43 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(posonly_vararg__doc__, +"posonly_vararg($module, a, /, b, *args)\n" +"--\n" +"\n"); + +#define POSONLY_VARARG_METHODDEF \ + {"posonly_vararg", _PyCFunction_CAST(posonly_vararg), METH_FASTCALL|METH_KEYWORDS, posonly_vararg__doc__}, + +static PyObject * +posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_vararg", 0}; + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + __clinic_args = args[2]; + return_value = posonly_vararg_impl(module, a, b, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(vararg_and_posonly__doc__, "vararg_and_posonly($module, a, /, *args)\n" "--\n" @@ -1985,6 +2022,219 @@ vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(vararg__doc__, +"vararg($module, /, a, *args)\n" +"--\n" +"\n"); + +#define VARARG_METHODDEF \ + {"vararg", _PyCFunction_CAST(vararg), METH_FASTCALL|METH_KEYWORDS, vararg__doc__}, + +static PyObject * +vararg_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "vararg", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + return_value = vararg_impl(module, a, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(vararg_with_default__doc__, +"vararg_with_default($module, /, a, *args, b=False)\n" +"--\n" +"\n"); + +#define VARARG_WITH_DEFAULT_METHODDEF \ + {"vararg_with_default", _PyCFunction_CAST(vararg_with_default), METH_FASTCALL|METH_KEYWORDS, vararg_with_default__doc__}, + +static PyObject * +vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b); + +static PyObject * +vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "vararg_with_default", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *__clinic_args = NULL; + int b = 0; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = PyObject_IsTrue(args[2]); + if (b < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = vararg_with_default_impl(module, a, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(vararg_with_only_defaults__doc__, +"vararg_with_only_defaults($module, /, *args, b=None)\n" +"--\n" +"\n"); + +#define VARARG_WITH_ONLY_DEFAULTS_METHODDEF \ + {"vararg_with_only_defaults", _PyCFunction_CAST(vararg_with_only_defaults), METH_FASTCALL|METH_KEYWORDS, vararg_with_only_defaults__doc__}, + +static PyObject * +vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b); + +static PyObject * +vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "vararg_with_only_defaults", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *__clinic_args = NULL; + PyObject *b = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = args[1]; +skip_optional_kwonly: + return_value = vararg_with_only_defaults_impl(module, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(gh_32092_oob__doc__, +"gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" +"--\n" +"\n" +"Proof-of-concept of GH-32092 OOB bug."); + +#define GH_32092_OOB_METHODDEF \ + {"gh_32092_oob", _PyCFunction_CAST(gh_32092_oob), METH_FASTCALL|METH_KEYWORDS, gh_32092_oob__doc__}, + +static PyObject * +gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, + PyObject *varargs, PyObject *kw1, PyObject *kw2); + +static PyObject * +gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"pos1", "pos2", "kw1", "kw2", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "gh_32092_oob", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = Py_MIN(nargs, 2) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *pos1; + PyObject *pos2; + PyObject *varargs = NULL; + PyObject *kw1 = Py_None; + PyObject *kw2 = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); + if (!args) { + goto exit; + } + pos1 = args[0]; + pos2 = args[1]; + varargs = args[2]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + kw1 = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + kw2 = args[4]; +skip_optional_kwonly: + return_value = gh_32092_oob_impl(module, pos1, pos2, varargs, kw1, kw2); + +exit: + Py_XDECREF(varargs); + return return_value; +} + +PyDoc_STRVAR(gh_32092_kw_pass__doc__, +"gh_32092_kw_pass($module, /, pos, *args, kw=None)\n" +"--\n" +"\n" +"Proof-of-concept of GH-32092 keyword args passing bug."); + +#define GH_32092_KW_PASS_METHODDEF \ + {"gh_32092_kw_pass", _PyCFunction_CAST(gh_32092_kw_pass), METH_FASTCALL|METH_KEYWORDS, gh_32092_kw_pass__doc__}, + +static PyObject * +gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, + PyObject *kw); + +static PyObject * +gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"pos", "kw", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "gh_32092_kw_pass", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *pos; + PyObject *__clinic_args = NULL; + PyObject *kw = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + pos = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + kw = args[2]; +skip_optional_kwonly: + return_value = gh_32092_kw_pass_impl(module, pos, __clinic_args, kw); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(gh_99233_refcount__doc__, "gh_99233_refcount($module, /, *args)\n" "--\n" @@ -2049,4 +2299,4 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=4147b953eebc3c82 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fe398ac790310bc4 input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index 9d6483f4fe0f..3105bd556c1b 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2570,7 +2570,25 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, current_arg = NULL; } - buf[i + vararg + 1] = current_arg; + /* If an arguments is passed in as a keyword argument, + * it should be placed before `buf[vararg]`. + * + * For example: + * def f(a, /, b, *args): + * pass + * f(1, b=2) + * + * This `buf` array should be: [1, 2, NULL]. + * In this case, nargs < vararg. + * + * Otherwise, we leave a place at `buf[vararg]` for vararg tuple + * so the index is `i + 1`. */ + if (nargs < vararg) { + buf[i] = current_arg; + } + else { + buf[i + 1] = current_arg; + } if (current_arg) { --nkwargs; diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c49805ce1a7e..a6d8b86a83ce 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -660,7 +660,7 @@ def output_templates(self, f): vararg = NO_VARARG pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 for i, p in enumerate(parameters, 1): - if p.is_keyword_only() or vararg != NO_VARARG: + if p.is_keyword_only(): assert not p.is_positional_only() if not p.is_optional(): min_kw_only = i - max_pos @@ -957,13 +957,14 @@ def parser_body(prototype, *fields, declarations=''): parser_definition = parser_body(parser_prototype, *parser_code) else: - has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters)) + has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) if vararg == NO_VARARG: args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( min_pos, max_pos, min_kw_only ) + nargs = "nargs" else: args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( min_pos, @@ -971,6 +972,7 @@ def parser_body(prototype, *fields, declarations=''): min_kw_only, vararg ) + nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" parser_prototype = parser_prototype_fastcall_keywords @@ -981,8 +983,7 @@ def parser_body(prototype, *fields, declarations=''): PyObject *argsbuf[%s]; """ % len(converters)) if has_optional_kw: - pre_buffer = "0" if vararg != NO_VARARG else "nargs" - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (pre_buffer, min_pos + min_kw_only) + declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) parser_code = [normalize_snippet(""" args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); if (!args) {{ @@ -1002,7 +1003,7 @@ def parser_body(prototype, *fields, declarations=''): Py_ssize_t nargs = PyTuple_GET_SIZE(args); """ % len(converters)) if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (min_pos + min_kw_only) + declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) parser_code = [normalize_snippet(""" fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); if (!fastargs) {{ From webhook-mailer at python.org Tue Dec 27 20:11:09 2022 From: webhook-mailer at python.org (gvanrossum) Date: Wed, 28 Dec 2022 01:11:09 -0000 Subject: [Python-checkins] GH-98831: Modernize a ton of simpler instructions (#100545) Message-ID: <mailman.3101.1672189869.3313.python-checkins@python.org> https://github.com/python/cpython/commit/08e5594cf3d42391a48e0311f6b9393ec2e00e1e commit: 08e5594cf3d42391a48e0311f6b9393ec2e00e1e branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-27T17:11:03-08:00 summary: GH-98831: Modernize a ton of simpler instructions (#100545) * load_const and load_fast aren't families for now * Don't decref unmoved names * Modernize GET_ANEXT * Modernize GET_AWAITABLE * Modernize ASYNC_GEN_WRAP * Modernize YIELD_VALUE * Modernize POP_EXCEPT (in more than one way) * Modernize PREP_RERAISE_STAR * Modernize LOAD_ASSERTION_ERROR * Modernize LOAD_BUILD_CLASS * Modernize STORE_NAME * Modernize LOAD_NAME * Modernize LOAD_CLASSDEREF * Modernize LOAD_DEREF * Modernize STORE_DEREF * Modernize COPY_FREE_VARS (mark it as done) * Modernize LIST_TO_TUPLE * Modernize LIST_EXTEND * Modernize SET_UPDATE * Modernize SETUP_ANNOTATIONS * Modernize DICT_UPDATE * Modernize DICT_MERGE * Modernize MAP_ADD * Modernize IS_OP * Modernize CONTAINS_OP * Modernize CHECK_EXC_MATCH * Modernize IMPORT_NAME * Modernize IMPORT_STAR * Modernize IMPORT_FROM * Modernize JUMP_FORWARD (mark it as done) * Modernize JUMP_BACKWARD (mark it as done) files: M Python/bytecodes.c M Python/generated_cases.c.h M Tools/cases_generator/generate_cases.py diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c0b625bd662c..e1c73ab6b32f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -83,9 +83,11 @@ static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub static PyObject *container, *start, *stop, *v, *lhs, *rhs; static PyObject *list, *tuple, *dict, *owner; static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter; +static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc; +static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from; static size_t jump; // Dummy variables for cache effects -static _Py_CODEUNIT when_to_jump_mask, invert, counter, index, hint; +static uint16_t when_to_jump_mask, invert, counter, index, hint; static uint32_t type_version; // Dummy opcode names for 'op' opcodes #define _COMPARE_OP_FLOAT 1003 @@ -638,12 +640,9 @@ dummy_func( } } - // stack effect: ( -- __0) - inst(GET_ANEXT) { + inst(GET_ANEXT, (aiter -- aiter, awaitable)) { unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyObject *awaitable = NULL; - PyObject *aiter = TOP(); PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { @@ -685,20 +684,17 @@ dummy_func( } } - PUSH(awaitable); PREDICT(LOAD_CONST); } - // stack effect: ( -- ) - inst(GET_AWAITABLE) { - PyObject *iterable = TOP(); - PyObject *iter = _PyCoro_GetAwaitableIter(iterable); + inst(GET_AWAITABLE, (iterable -- iter)) { + iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); + DECREF_INPUTS(); if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -714,11 +710,7 @@ dummy_func( } } - SET_TOP(iter); /* Even if it's NULL */ - - if (iter == NULL) { - goto error; - } + ERROR_IF(iter == NULL, error); PREDICT(LOAD_CONST); } @@ -773,29 +765,22 @@ dummy_func( } } - // stack effect: ( -- ) - inst(ASYNC_GEN_WRAP) { - PyObject *v = TOP(); + inst(ASYNC_GEN_WRAP, (v -- w)) { assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR); - PyObject *w = _PyAsyncGenValueWrapperNew(v); - if (w == NULL) { - goto error; - } - SET_TOP(w); - Py_DECREF(v); + w = _PyAsyncGenValueWrapperNew(v); + DECREF_INPUTS(); + ERROR_IF(w == NULL, error); } - // stack effect: ( -- ) - inst(YIELD_VALUE) { + inst(YIELD_VALUE, (retval --)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. assert(oparg == STACK_LEVEL()); assert(frame != &entry_frame); - PyObject *retval = POP(); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_SetStackPointer(frame, stack_pointer - 1); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; @@ -809,12 +794,9 @@ dummy_func( goto resume_frame; } - // stack effect: (__0 -- ) - inst(POP_EXCEPT) { + inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - PyObject *value = exc_info->exc_value; - exc_info->exc_value = POP(); - Py_XDECREF(value); + Py_XSETREF(exc_info->exc_value, exc_value); } // stack effect: (__0 -- ) @@ -839,21 +821,13 @@ dummy_func( goto exception_unwind; } - // stack effect: (__0 -- ) - inst(PREP_RERAISE_STAR) { - PyObject *excs = POP(); + inst(PREP_RERAISE_STAR, (orig, excs -- val)) { assert(PyList_Check(excs)); - PyObject *orig = POP(); - - PyObject *val = _PyExc_PrepReraiseStar(orig, excs); - Py_DECREF(excs); - Py_DECREF(orig); - if (val == NULL) { - goto error; - } + val = _PyExc_PrepReraiseStar(orig, excs); + DECREF_INPUTS(); - PUSH(val); + ERROR_IF(val == NULL, error); } // stack effect: (__0, __1 -- ) @@ -934,16 +908,11 @@ dummy_func( } } - - // stack effect: ( -- __0) - inst(LOAD_ASSERTION_ERROR) { - PyObject *value = PyExc_AssertionError; - PUSH(Py_NewRef(value)); + inst(LOAD_ASSERTION_ERROR, ( -- value)) { + value = Py_NewRef(PyExc_AssertionError); } - // stack effect: ( -- __0) - inst(LOAD_BUILD_CLASS) { - PyObject *bc; + inst(LOAD_BUILD_CLASS, ( -- bc)) { if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -952,7 +921,7 @@ dummy_func( _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); } - goto error; + ERROR_IF(true, error); } Py_INCREF(bc); } @@ -962,31 +931,27 @@ dummy_func( if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - goto error; + ERROR_IF(true, error); } } - PUSH(bc); } - // stack effect: (__0 -- ) - inst(STORE_NAME) { + inst(STORE_NAME, (v -- )) { PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err != 0) - goto error; + DECREF_INPUTS(); + ERROR_IF(err, error); } inst(DELETE_NAME, (--)) { @@ -1139,11 +1104,9 @@ dummy_func( } } - // stack effect: ( -- __0) - inst(LOAD_NAME) { + inst(LOAD_NAME, ( -- v)) { PyObject *name = GETITEM(names, oparg); PyObject *locals = LOCALS(); - PyObject *v; if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when loading %R", name); @@ -1200,7 +1163,6 @@ dummy_func( } } } - PUSH(v); } // error: LOAD_GLOBAL has irregular stack effect @@ -1339,9 +1301,8 @@ dummy_func( Py_DECREF(oldobj); } - // stack effect: ( -- __0) - inst(LOAD_CLASSDEREF) { - PyObject *name, *value, *locals = LOCALS(); + inst(LOAD_CLASSDEREF, ( -- value)) { + PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg); @@ -1372,31 +1333,26 @@ dummy_func( } Py_INCREF(value); } - PUSH(value); } - // stack effect: ( -- __0) - inst(LOAD_DEREF) { + inst(LOAD_DEREF, ( -- value)) { PyObject *cell = GETLOCAL(oparg); - PyObject *value = PyCell_GET(cell); + value = PyCell_GET(cell); if (value == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); - goto error; + ERROR_IF(true, error); } - PUSH(Py_NewRef(value)); + Py_INCREF(value); } - // stack effect: (__0 -- ) - inst(STORE_DEREF) { - PyObject *v = POP(); + inst(STORE_DEREF, (v --)) { PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); } - // stack effect: ( -- ) - inst(COPY_FREE_VARS) { + inst(COPY_FREE_VARS, (--)) { /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1444,21 +1400,14 @@ dummy_func( PUSH(list); } - // stack effect: ( -- ) - inst(LIST_TO_TUPLE) { - PyObject *list = POP(); - PyObject *tuple = PyList_AsTuple(list); - Py_DECREF(list); - if (tuple == NULL) { - goto error; - } - PUSH(tuple); + inst(LIST_TO_TUPLE, (list -- tuple)) { + tuple = PyList_AsTuple(list); + DECREF_INPUTS(); + ERROR_IF(tuple == NULL, error); } - // stack effect: (__0 -- ) - inst(LIST_EXTEND) { - PyObject *iterable = POP(); - PyObject *list = PEEK(oparg); + inst(LIST_EXTEND, (iterable -- )) { + PyObject *list = PEEK(oparg + 1); // iterable is still on the stack PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1469,22 +1418,18 @@ dummy_func( "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } Py_DECREF(none_val); - Py_DECREF(iterable); + DECREF_INPUTS(); } - // stack effect: (__0 -- ) - inst(SET_UPDATE) { - PyObject *iterable = POP(); - PyObject *set = PEEK(oparg); + inst(SET_UPDATE, (iterable --)) { + PyObject *set = PEEK(oparg + 1); // iterable is still on the stack int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) { - goto error; - } + DECREF_INPUTS(); + ERROR_IF(err < 0, error); } // stack effect: (__array[oparg] -- __0) @@ -1524,54 +1469,41 @@ dummy_func( PUSH(map); } - // stack effect: ( -- ) - inst(SETUP_ANNOTATIONS) { + inst(SETUP_ANNOTATIONS, (--)) { int err; PyObject *ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - goto error; + ERROR_IF(true, error); } /* check if __annotations__ in locals()... */ if (PyDict_CheckExact(LOCALS())) { ann_dict = _PyDict_GetItemWithError(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) { - goto error; - } + ERROR_IF(_PyErr_Occurred(tstate), error); /* ...if not, create a new one */ ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + ERROR_IF(ann_dict == NULL, error); err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + ERROR_IF(err, error); } } else { /* do the same if locals() is not a dict */ ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } + ERROR_IF(!_PyErr_ExceptionMatches(tstate, PyExc_KeyError), error); _PyErr_Clear(tstate); ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + ERROR_IF(ann_dict == NULL, error); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + ERROR_IF(err, error); } else { Py_DECREF(ann_dict); @@ -1603,48 +1535,38 @@ dummy_func( PUSH(map); } - // stack effect: (__0 -- ) - inst(DICT_UPDATE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + inst(DICT_UPDATE, (update --)) { + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - Py_DECREF(update); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } - Py_DECREF(update); + DECREF_INPUTS(); } - // stack effect: (__0 -- ) - inst(DICT_MERGE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + inst(DICT_MERGE, (update --)) { + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(2 + oparg), update); - Py_DECREF(update); - goto error; + format_kwargs_error(tstate, PEEK(3 + oparg), update); + DECREF_INPUTS(); + ERROR_IF(true, error); } - Py_DECREF(update); + DECREF_INPUTS(); PREDICT(CALL_FUNCTION_EX); } - // stack effect: (__0, __1 -- ) - inst(MAP_ADD) { - PyObject *value = TOP(); - PyObject *key = SECOND(); - PyObject *map; - STACK_SHRINK(2); - map = PEEK(oparg); /* dict */ - assert(PyDict_CheckExact(map)); - /* map[key] = value */ - if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) { - goto error; - } + inst(MAP_ADD, (key, value --)) { + PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); PREDICT(JUMP_BACKWARD); } @@ -2073,29 +1995,17 @@ dummy_func( } super(COMPARE_OP_STR_JUMP) = _COMPARE_OP_STR + _JUMP_IF; - // stack effect: (__0 -- ) - inst(IS_OP) { - PyObject *right = POP(); - PyObject *left = TOP(); + inst(IS_OP, (left, right -- b)) { int res = Py_Is(left, right) ^ oparg; - PyObject *b = res ? Py_True : Py_False; - SET_TOP(Py_NewRef(b)); - Py_DECREF(left); - Py_DECREF(right); + DECREF_INPUTS(); + b = Py_NewRef(res ? Py_True : Py_False); } - // stack effect: (__0 -- ) - inst(CONTAINS_OP) { - PyObject *right = POP(); - PyObject *left = POP(); + inst(CONTAINS_OP, (left, right -- b)) { int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) { - goto error; - } - PyObject *b = (res^oparg) ? Py_True : Py_False; - PUSH(Py_NewRef(b)); + DECREF_INPUTS(); + ERROR_IF(res < 0, error); + b = Py_NewRef((res^oparg) ? Py_True : Py_False); } // stack effect: ( -- ) @@ -2139,76 +2049,57 @@ dummy_func( } } - // stack effect: ( -- ) - inst(CHECK_EXC_MATCH) { - PyObject *right = POP(); - PyObject *left = TOP(); + inst(CHECK_EXC_MATCH, (left, right -- left, b)) { assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - Py_DECREF(right); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - PUSH(Py_NewRef(res ? Py_True : Py_False)); + DECREF_INPUTS(); + b = Py_NewRef(res ? Py_True : Py_False); } - // stack effect: (__0 -- ) - inst(IMPORT_NAME) { + inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(names, oparg); - PyObject *fromlist = POP(); - PyObject *level = TOP(); - PyObject *res; res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - SET_TOP(res); - if (res == NULL) - goto error; + DECREF_INPUTS(); + ERROR_IF(res == NULL, error); } - // stack effect: (__0 -- ) - inst(IMPORT_STAR) { - PyObject *from = POP(), *locals; + inst(IMPORT_STAR, (from --)) { + PyObject *locals; int err; if (_PyFrame_FastToLocalsWithError(frame) < 0) { - Py_DECREF(from); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); - Py_DECREF(from); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } err = import_all_from(tstate, locals, from); _PyFrame_LocalsToFast(frame, 0); - Py_DECREF(from); - if (err != 0) - goto error; + DECREF_INPUTS(); + ERROR_IF(err, error); } - // stack effect: ( -- __0) - inst(IMPORT_FROM) { + inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(names, oparg); - PyObject *from = TOP(); - PyObject *res; res = import_from(tstate, from, name); - PUSH(res); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); } - // stack effect: ( -- ) - inst(JUMP_FORWARD) { + inst(JUMP_FORWARD, (--)) { JUMPBY(oparg); } - // stack effect: ( -- ) - inst(JUMP_BACKWARD) { + inst(JUMP_BACKWARD, (--)) { assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); @@ -3631,8 +3522,6 @@ family(load_attr) = { LOAD_ATTR_PROPERTY, LOAD_ATTR_SLOT, LOAD_ATTR_WITH_HINT, LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_METHOD_NO_DICT, LOAD_ATTR_METHOD_WITH_DICT, LOAD_ATTR_METHOD_WITH_VALUES }; -family(load_const) = { LOAD_CONST, LOAD_CONST__LOAD_FAST }; -family(load_fast) = { LOAD_FAST, LOAD_FAST__LOAD_CONST, LOAD_FAST__LOAD_FAST }; family(load_global) = { LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN, LOAD_GLOBAL_MODULE }; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 42b7ca086705..1179bdfc696c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -795,10 +795,10 @@ } TARGET(GET_ANEXT) { + PyObject *aiter = PEEK(1); + PyObject *awaitable; unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyObject *awaitable = NULL; - PyObject *aiter = TOP(); PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { @@ -840,15 +840,17 @@ } } - PUSH(awaitable); + STACK_GROW(1); + POKE(1, awaitable); PREDICT(LOAD_CONST); DISPATCH(); } TARGET(GET_AWAITABLE) { PREDICTED(GET_AWAITABLE); - PyObject *iterable = TOP(); - PyObject *iter = _PyCoro_GetAwaitableIter(iterable); + PyObject *iterable = PEEK(1); + PyObject *iter; + iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); @@ -870,12 +872,9 @@ } } - SET_TOP(iter); /* Even if it's NULL */ - - if (iter == NULL) { - goto error; - } + if (iter == NULL) goto pop_1_error; + POKE(1, iter); PREDICT(LOAD_CONST); DISPATCH(); } @@ -931,27 +930,26 @@ } TARGET(ASYNC_GEN_WRAP) { - PyObject *v = TOP(); + PyObject *v = PEEK(1); + PyObject *w; assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR); - PyObject *w = _PyAsyncGenValueWrapperNew(v); - if (w == NULL) { - goto error; - } - SET_TOP(w); + w = _PyAsyncGenValueWrapperNew(v); Py_DECREF(v); + if (w == NULL) goto pop_1_error; + POKE(1, w); DISPATCH(); } TARGET(YIELD_VALUE) { + PyObject *retval = PEEK(1); // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. assert(oparg == STACK_LEVEL()); assert(frame != &entry_frame); - PyObject *retval = POP(); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_SetStackPointer(frame, stack_pointer - 1); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; @@ -966,10 +964,10 @@ } TARGET(POP_EXCEPT) { + PyObject *exc_value = PEEK(1); _PyErr_StackItem *exc_info = tstate->exc_info; - PyObject *value = exc_info->exc_value; - exc_info->exc_value = POP(); - Py_XDECREF(value); + Py_XSETREF(exc_info->exc_value, exc_value); + STACK_SHRINK(1); DISPATCH(); } @@ -995,19 +993,18 @@ } TARGET(PREP_RERAISE_STAR) { - PyObject *excs = POP(); + PyObject *excs = PEEK(1); + PyObject *orig = PEEK(2); + PyObject *val; assert(PyList_Check(excs)); - PyObject *orig = POP(); - PyObject *val = _PyExc_PrepReraiseStar(orig, excs); - Py_DECREF(excs); + val = _PyExc_PrepReraiseStar(orig, excs); Py_DECREF(orig); + Py_DECREF(excs); - if (val == NULL) { - goto error; - } - - PUSH(val); + if (val == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, val); DISPATCH(); } @@ -1091,8 +1088,10 @@ } TARGET(LOAD_ASSERTION_ERROR) { - PyObject *value = PyExc_AssertionError; - PUSH(Py_NewRef(value)); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } @@ -1106,7 +1105,7 @@ _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); } - goto error; + if (true) goto error; } Py_INCREF(bc); } @@ -1116,31 +1115,32 @@ if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - goto error; + if (true) goto error; } } - PUSH(bc); + STACK_GROW(1); + POKE(1, bc); DISPATCH(); } TARGET(STORE_NAME) { + PyObject *v = PEEK(1); PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); Py_DECREF(v); - goto error; + if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); Py_DECREF(v); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } @@ -1304,9 +1304,9 @@ } TARGET(LOAD_NAME) { + PyObject *v; PyObject *name = GETITEM(names, oparg); PyObject *locals = LOCALS(); - PyObject *v; if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when loading %R", name); @@ -1363,7 +1363,8 @@ } } } - PUSH(v); + STACK_GROW(1); + POKE(1, v); DISPATCH(); } @@ -1508,7 +1509,8 @@ } TARGET(LOAD_CLASSDEREF) { - PyObject *name, *value, *locals = LOCALS(); + PyObject *value; + PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg); @@ -1539,27 +1541,32 @@ } Py_INCREF(value); } - PUSH(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(LOAD_DEREF) { + PyObject *value; PyObject *cell = GETLOCAL(oparg); - PyObject *value = PyCell_GET(cell); + value = PyCell_GET(cell); if (value == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); - goto error; + if (true) goto error; } - PUSH(Py_NewRef(value)); + Py_INCREF(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(STORE_DEREF) { - PyObject *v = POP(); + PyObject *v = PEEK(1); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); + STACK_SHRINK(1); DISPATCH(); } @@ -1613,19 +1620,18 @@ } TARGET(LIST_TO_TUPLE) { - PyObject *list = POP(); - PyObject *tuple = PyList_AsTuple(list); + PyObject *list = PEEK(1); + PyObject *tuple; + tuple = PyList_AsTuple(list); Py_DECREF(list); - if (tuple == NULL) { - goto error; - } - PUSH(tuple); + if (tuple == NULL) goto pop_1_error; + POKE(1, tuple); DISPATCH(); } TARGET(LIST_EXTEND) { - PyObject *iterable = POP(); - PyObject *list = PEEK(oparg); + PyObject *iterable = PEEK(1); + PyObject *list = PEEK(oparg + 1); // iterable is still on the stack PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1637,21 +1643,21 @@ Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); - goto error; + if (true) goto pop_1_error; } Py_DECREF(none_val); Py_DECREF(iterable); + STACK_SHRINK(1); DISPATCH(); } TARGET(SET_UPDATE) { - PyObject *iterable = POP(); - PyObject *set = PEEK(oparg); + PyObject *iterable = PEEK(1); + PyObject *set = PEEK(oparg + 1); // iterable is still on the stack int err = _PySet_Update(set, iterable); Py_DECREF(iterable); - if (err < 0) { - goto error; - } + if (err < 0) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } @@ -1698,47 +1704,35 @@ if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - goto error; + if (true) goto error; } /* check if __annotations__ in locals()... */ if (PyDict_CheckExact(LOCALS())) { ann_dict = _PyDict_GetItemWithError(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) { - goto error; - } + if (_PyErr_Occurred(tstate)) goto error; /* ...if not, create a new one */ ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + if (ann_dict == NULL) goto error; err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + if (err) goto error; } } else { /* do the same if locals() is not a dict */ ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) goto error; _PyErr_Clear(tstate); ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + if (ann_dict == NULL) goto error; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + if (err) goto error; } else { Py_DECREF(ann_dict); @@ -1772,8 +1766,8 @@ } TARGET(DICT_UPDATE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + PyObject *update = PEEK(1); + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -1781,37 +1775,37 @@ Py_TYPE(update)->tp_name); } Py_DECREF(update); - goto error; + if (true) goto pop_1_error; } Py_DECREF(update); + STACK_SHRINK(1); DISPATCH(); } TARGET(DICT_MERGE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + PyObject *update = PEEK(1); + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(2 + oparg), update); + format_kwargs_error(tstate, PEEK(3 + oparg), update); Py_DECREF(update); - goto error; + if (true) goto pop_1_error; } Py_DECREF(update); + STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); DISPATCH(); } TARGET(MAP_ADD) { - PyObject *value = TOP(); - PyObject *key = SECOND(); - PyObject *map; + PyObject *value = PEEK(1); + PyObject *key = PEEK(2); + PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; STACK_SHRINK(2); - map = PEEK(oparg); /* dict */ - assert(PyDict_CheckExact(map)); - /* map[key] = value */ - if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) { - goto error; - } PREDICT(JUMP_BACKWARD); DISPATCH(); } @@ -2312,27 +2306,29 @@ } TARGET(IS_OP) { - PyObject *right = POP(); - PyObject *left = TOP(); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *b; int res = Py_Is(left, right) ^ oparg; - PyObject *b = res ? Py_True : Py_False; - SET_TOP(Py_NewRef(b)); Py_DECREF(left); Py_DECREF(right); + b = Py_NewRef(res ? Py_True : Py_False); + STACK_SHRINK(1); + POKE(1, b); DISPATCH(); } TARGET(CONTAINS_OP) { - PyObject *right = POP(); - PyObject *left = POP(); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *b; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) { - goto error; - } - PyObject *b = (res^oparg) ? Py_True : Py_False; - PUSH(Py_NewRef(b)); + if (res < 0) goto pop_2_error; + b = Py_NewRef((res^oparg) ? Py_True : Py_False); + STACK_SHRINK(1); + POKE(1, b); DISPATCH(); } @@ -2378,40 +2374,43 @@ } TARGET(CHECK_EXC_MATCH) { - PyObject *right = POP(); - PyObject *left = TOP(); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *b; assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { Py_DECREF(right); - goto error; + if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); - PUSH(Py_NewRef(res ? Py_True : Py_False)); + b = Py_NewRef(res ? Py_True : Py_False); + POKE(1, b); DISPATCH(); } TARGET(IMPORT_NAME) { - PyObject *name = GETITEM(names, oparg); - PyObject *fromlist = POP(); - PyObject *level = TOP(); + PyObject *fromlist = PEEK(1); + PyObject *level = PEEK(2); PyObject *res; + PyObject *name = GETITEM(names, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); Py_DECREF(fromlist); - SET_TOP(res); - if (res == NULL) - goto error; + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, res); DISPATCH(); } TARGET(IMPORT_STAR) { - PyObject *from = POP(), *locals; + PyObject *from = PEEK(1); + PyObject *locals; int err; if (_PyFrame_FastToLocalsWithError(frame) < 0) { Py_DECREF(from); - goto error; + if (true) goto pop_1_error; } locals = LOCALS(); @@ -2419,24 +2418,24 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); Py_DECREF(from); - goto error; + if (true) goto pop_1_error; } err = import_all_from(tstate, locals, from); _PyFrame_LocalsToFast(frame, 0); Py_DECREF(from); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } TARGET(IMPORT_FROM) { - PyObject *name = GETITEM(names, oparg); - PyObject *from = TOP(); + PyObject *from = PEEK(1); PyObject *res; + PyObject *name = GETITEM(names, oparg); res = import_from(tstate, from, name); - PUSH(res); - if (res == NULL) - goto error; + if (res == NULL) goto error; + STACK_GROW(1); + POKE(1, res); DISPATCH(); } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 85a7c6098e0b..5eed74c5e147 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -125,6 +125,7 @@ class Instruction: # Set later family: parser.Family | None = None predicted: bool = False + unmoved_names: frozenset[str] = frozenset() def __init__(self, inst: parser.InstDef): self.inst = inst @@ -141,6 +142,13 @@ def __init__(self, inst: parser.InstDef): effect for effect in inst.inputs if isinstance(effect, StackEffect) ] self.output_effects = inst.outputs # For consistency/completeness + unmoved_names: set[str] = set() + for ieffect, oeffect in zip(self.input_effects, self.output_effects): + if ieffect.name == oeffect.name: + unmoved_names.add(ieffect.name) + else: + break + self.unmoved_names = frozenset(unmoved_names) def write(self, out: Formatter) -> None: """Write one instruction, sans prologue and epilogue.""" @@ -175,12 +183,8 @@ def write(self, out: Formatter) -> None: out.stack_adjust(diff) # Write output stack effect assignments - unmoved_names: set[str] = set() - for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect.name == oeffect.name: - unmoved_names.add(ieffect.name) for i, oeffect in enumerate(reversed(self.output_effects), 1): - if oeffect.name not in unmoved_names: + if oeffect.name not in self.unmoved_names: dst = StackEffect(f"PEEK({i})", "") out.assign(dst, oeffect) @@ -235,7 +239,8 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None elif m := re.match(r"(\s*)DECREF_INPUTS;\s*$", line): space = m.group(1) for ieff in self.input_effects: - out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") + if ieff.name not in self.unmoved_names: + out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") else: out.write_raw(extra + line) @@ -533,7 +538,7 @@ def stack_analysis( ) -> tuple[list[StackEffect], int]: """Analyze a super-instruction or macro. - Print an error if there's a cache effect (which we don't support yet). + Ignore cache effects. Return the list of variable names and the initial stack pointer. """ From webhook-mailer at python.org Tue Dec 27 20:16:58 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 01:16:58 -0000 Subject: [Python-checkins] gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) Message-ID: <mailman.3102.1672190219.3313.python-checkins@python.org> https://github.com/python/cpython/commit/04285502ba7d90e41138b0282434372be18ac042 commit: 04285502ba7d90e41138b0282434372be18ac042 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T17:16:53-08:00 summary: gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) (cherry picked from commit 3dc48dabd48864039951715816e07986a4828d80) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/sqlite3/test/factory.py diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py index 876428497542..40a290f0c981 100644 --- a/Lib/sqlite3/test/factory.py +++ b/Lib/sqlite3/test/factory.py @@ -155,8 +155,14 @@ def test_sqlite_row_iter(self): """Checks if the row object is iterable""" self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): """Checks if the row object can be converted to a tuple""" From webhook-mailer at python.org Tue Dec 27 20:25:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 01:25:34 -0000 Subject: [Python-checkins] gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) Message-ID: <mailman.3103.1672190734.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fba8c7cf740433cc71280ea6dce6b44ee477b2d3 commit: fba8c7cf740433cc71280ea6dce6b44ee477b2d3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T17:25:28-08:00 summary: gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) (cherry picked from commit 3dc48dabd48864039951715816e07986a4828d80) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_sqlite3/test_factory.py diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index cdaf12588a7f..2b70093bd9b6 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -179,8 +179,14 @@ def test_sqlite_row_iter(self): """Checks if the row object is iterable""" self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): """Checks if the row object can be converted to a tuple""" From webhook-mailer at python.org Tue Dec 27 23:12:22 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:12:22 -0000 Subject: [Python-checkins] GH-100192: add more `asyncio` subprocess tests (#100194) Message-ID: <mailman.3104.1672200743.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e97afefda54b818b140b3cc905642b69d9d65f0c commit: e97afefda54b818b140b3cc905642b69d9d65f0c branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T09:42:16+05:30 summary: GH-100192: add more `asyncio` subprocess tests (#100194) files: M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3830dea7d9ba..1ae290a003f2 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -686,6 +686,88 @@ async def execute(): self.assertIsNone(self.loop.run_until_complete(execute())) + async def check_stdout_output(self, coro, output): + proc = await coro + stdout, _ = await proc.communicate() + self.assertEqual(stdout, output) + self.assertEqual(proc.returncode, 0) + task = asyncio.create_task(proc.wait()) + await asyncio.sleep(0) + self.assertEqual(task.result(), proc.returncode) + + def test_create_subprocess_env_shell(self) -> None: + async def main() -> None: + cmd = f'''{sys.executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' + env = {"FOO": 'bar'} + proc = await asyncio.create_subprocess_shell( + cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'bar')) + + def test_create_subprocess_env_exec(self) -> None: + async def main() -> None: + cmd = [sys.executable, "-c", + "import os, sys; sys.stdout.write(os.getenv('FOO'))"] + env = {"FOO": 'baz'} + proc = await asyncio.create_subprocess_exec( + *cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'baz')) + + + def test_subprocess_concurrent_wait(self) -> None: + async def main() -> None: + proc = await asyncio.create_subprocess_exec( + *PROGRAM_CAT, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, _ = await proc.communicate(b'some data') + self.assertEqual(stdout, b"some data") + self.assertEqual(proc.returncode, 0) + self.assertEqual(await asyncio.gather(*[proc.wait() for _ in range(10)]), + [proc.returncode] * 10) + + self.loop.run_until_complete(main()) + + def test_subprocess_consistent_callbacks(self): + events = [] + class MyProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future: asyncio.Future) -> None: + self.exit_future = exit_future + + def pipe_data_received(self, fd, data) -> None: + events.append(('pipe_data_received', fd, data)) + + def pipe_connection_lost(self, fd, exc) -> None: + events.append('pipe_connection_lost') + + def process_exited(self) -> None: + events.append('process_exited') + self.exit_future.set_result(True) + + async def main() -> None: + loop = asyncio.get_running_loop() + exit_future = asyncio.Future() + code = 'import sys; sys.stdout.write("stdout"); sys.stderr.write("stderr")' + transport, _ = await loop.subprocess_exec(lambda: MyProtocol(exit_future), + sys.executable, '-c', code, stdin=None) + await exit_future + transport.close() + self.assertEqual(events, [ + ('pipe_data_received', 1, b'stdout'), + ('pipe_data_received', 2, b'stderr'), + 'pipe_connection_lost', + 'pipe_connection_lost', + 'process_exited', + ]) + + self.loop.run_until_complete(main()) + def test_subprocess_communicate_stdout(self): # See https://github.com/python/cpython/issues/100133 async def get_command_stdout(cmd, *args): From webhook-mailer at python.org Tue Dec 27 23:16:34 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:16:34 -0000 Subject: [Python-checkins] GH-100342: check for allocation failure in AC `*args` parsing (#100343) Message-ID: <mailman.3105.1672200995.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7cf164ad5e3c8c6af5ae8813ad6a784448605418 commit: 7cf164ad5e3c8c6af5ae8813ad6a784448605418 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T09:46:28+05:30 summary: GH-100342: check for allocation failure in AC `*args` parsing (#100343) files: A Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst M Lib/test/clinic.test M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 0d844234d9d1..53e5df5ba872 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3781,6 +3781,9 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -3793,7 +3796,7 @@ exit: static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=081a953b8cbe7617 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=79b75dc07decc8d6 input=08dc2bf7afbf1613]*/ /*[clinic input] test_vararg diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst new file mode 100644 index 000000000000..28f736337526 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst @@ -0,0 +1 @@ +Add missing ``NULL`` check for possible allocation failure in ``*args`` parsing in Argument Clinic. diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 21bde5294702..831f58ca650a 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2409,6 +2409,9 @@ vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -2769,6 +2772,9 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 0; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); } @@ -2811,4 +2817,4 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=9a5ca5909c087102 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e8211606b03d733a input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2fb1902a5b54..552ed2c889a7 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -960,12 +960,16 @@ def parser_body(prototype, *fields, declarations=''): if not new_or_init: parser_code.append(normalize_snippet(""" %s = PyTuple_New(%s); + if (!%s) {{ + goto exit; + }} for (Py_ssize_t i = 0; i < %s; ++i) {{ PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); }} """ % ( p.converter.parser_name, left_args, + p.converter.parser_name, left_args, p.converter.parser_name, max_pos From webhook-mailer at python.org Tue Dec 27 23:28:03 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:28:03 -0000 Subject: [Python-checkins] Fix copy-paste error in `Tools/clinic.py` (#100560) Message-ID: <mailman.3106.1672201684.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3da71ff0f26516c433538b2bfc4db196e20e0d44 commit: 3da71ff0f26516c433538b2bfc4db196e20e0d44 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T09:57:57+05:30 summary: Fix copy-paste error in `Tools/clinic.py` (#100560) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 552ed2c889a7..abff4d2583e1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2832,7 +2832,7 @@ def modify(self): """ The C statements required to modify this variable after parsing. Returns a string containing this code indented at column 0. - If no initialization is necessary, returns an empty string. + If no modification is necessary, returns an empty string. """ return "" From webhook-mailer at python.org Tue Dec 27 23:38:01 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:38:01 -0000 Subject: [Python-checkins] add tests for `asyncio` transport sockets (#100263) Message-ID: <mailman.3107.1672202281.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5369bba8c594de7a643408550e19e1ff6df5178a commit: 5369bba8c594de7a643408550e19e1ff6df5178a branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T10:07:55+05:30 summary: add tests for `asyncio` transport sockets (#100263) files: M Lib/test/test_asyncio/test_events.py diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 153b2de81722..e7771edd2e4a 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -823,6 +823,29 @@ def test_create_server(self): # close server server.close() + def test_create_server_trsock(self): + proto = MyProto(self.loop) + f = self.loop.create_server(lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertIsInstance(sock, asyncio.trsock.TransportSocket) + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + dup = sock.dup() + self.addCleanup(dup.close) + self.assertIsInstance(dup, socket.socket) + self.assertFalse(sock.get_inheritable()) + with self.assertRaises(ValueError): + sock.settimeout(1) + sock.settimeout(0) + self.assertEqual(sock.gettimeout(), 0) + with self.assertRaises(ValueError): + sock.setblocking(True) + sock.setblocking(False) + server.close() + + @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT') def test_create_server_reuse_port(self): proto = MyProto(self.loop) From webhook-mailer at python.org Tue Dec 27 23:41:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:41:32 -0000 Subject: [Python-checkins] [3.11] GH-100342: check for allocation failure in AC `*args` parsing (GH-100343). (#100568) Message-ID: <mailman.3108.1672202493.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ebe428824d9c749577256cbcfb35945d8b19c479 commit: ebe428824d9c749577256cbcfb35945d8b19c479 branch: 3.11 author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T10:11:27+05:30 summary: [3.11] GH-100342: check for allocation failure in AC `*args` parsing (GH-100343). (#100568) (cherry picked from commit 7cf164ad5e3c8c6af5ae8813ad6a784448605418) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst M Lib/test/clinic.test M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 217aa4c7e97a..0abadbe7f347 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3327,6 +3327,9 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -3339,7 +3342,7 @@ exit: static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=081a953b8cbe7617 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=79b75dc07decc8d6 input=08dc2bf7afbf1613]*/ /*[clinic input] test_vararg diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst new file mode 100644 index 000000000000..28f736337526 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst @@ -0,0 +1 @@ +Add missing ``NULL`` check for possible allocation failure in ``*args`` parsing in Argument Clinic. diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 41555d801059..c97ac1c66296 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2012,6 +2012,9 @@ vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -2257,6 +2260,9 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 0; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); } @@ -2299,4 +2305,4 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=fe398ac790310bc4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b719efc1b8bd2c8 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a6d8b86a83ce..82e4919242c7 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -903,12 +903,16 @@ def parser_body(prototype, *fields, declarations=''): if not new_or_init: parser_code.append(normalize_snippet(""" %s = PyTuple_New(%s); + if (!%s) {{ + goto exit; + }} for (Py_ssize_t i = 0; i < %s; ++i) {{ PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); }} """ % ( p.converter.parser_name, left_args, + p.converter.parser_name, left_args, p.converter.parser_name, max_pos From webhook-mailer at python.org Wed Dec 28 00:30:47 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Wed, 28 Dec 2022 05:30:47 -0000 Subject: [Python-checkins] gh-55688: Add note about ending backslashes for raw strings (#94768) Message-ID: <mailman.3109.1672205448.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8 commit: b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-28T00:30:42-05:00 summary: gh-55688: Add note about ending backslashes for raw strings (#94768) Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: M Doc/faq/programming.rst M Doc/tutorial/introduction.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index c396e2b081fc..ba42289f3466 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1026,6 +1026,46 @@ What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? See the :ref:`unicode-howto`. +.. _faq-programming-raw-string-backslash: + +Can I end a raw string with an odd number of backslashes? +--------------------------------------------------------- + +A raw string ending with an odd number of backslashes will escape the string's quote:: + + >>> r'C:\this\will\not\work\' + File "<stdin>", line 1 + r'C:\this\will\not\work\' + ^ + SyntaxError: unterminated string literal (detected at line 1) + +There are several workarounds for this. One is to use regular strings and double +the backslashes:: + + >>> 'C:\\this\\will\\work\\' + 'C:\\this\\will\\work\\' + +Another is to concatenate a regular string containing an escaped backslash to the +raw string:: + + >>> r'C:\this\will\work' '\\' + 'C:\\this\\will\\work\\' + +It is also possible to use :func:`os.path.join` to append a backslash on Windows:: + + >>> os.path.join(r'C:\this\will\work', '') + 'C:\\this\\will\\work\\' + +Note that while a backslash will "escape" a quote for the purposes of +determining where the raw string ends, no escaping occurs when interpreting the +value of the raw string. That is, the backslash remains present in the value of +the raw string:: + + >>> r'backslash\'preserved' + "backslash\\'preserved" + +Also see the specification in the :ref:`language reference <strings>`. + Performance =========== diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 6532a4e2cc83..ebc2e9187534 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -189,6 +189,11 @@ the first quote:: >>> print(r'C:\some\name') # note the r before the quote C:\some\name +There is one subtle aspect to raw strings: a raw string may not end in +an odd number of ``\`` characters; see +:ref:`the FAQ entry <faq-programming-raw-string-backslash>` for more information +and workarounds. + String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at From webhook-mailer at python.org Wed Dec 28 00:39:58 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 05:39:58 -0000 Subject: [Python-checkins] gh-55688: Add note about ending backslashes for raw strings (GH-94768) Message-ID: <mailman.3110.1672205999.3313.python-checkins@python.org> https://github.com/python/cpython/commit/97a3e18c8873228a7ab7f720954798c2c5c7486d commit: 97a3e18c8873228a7ab7f720954798c2c5c7486d branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T21:39:53-08:00 summary: gh-55688: Add note about ending backslashes for raw strings (GH-94768) (cherry picked from commit b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: M Doc/faq/programming.rst M Doc/tutorial/introduction.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index bd75801cc242..1bcca4c62d27 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1026,6 +1026,46 @@ What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? See the :ref:`unicode-howto`. +.. _faq-programming-raw-string-backslash: + +Can I end a raw string with an odd number of backslashes? +--------------------------------------------------------- + +A raw string ending with an odd number of backslashes will escape the string's quote:: + + >>> r'C:\this\will\not\work\' + File "<stdin>", line 1 + r'C:\this\will\not\work\' + ^ + SyntaxError: unterminated string literal (detected at line 1) + +There are several workarounds for this. One is to use regular strings and double +the backslashes:: + + >>> 'C:\\this\\will\\work\\' + 'C:\\this\\will\\work\\' + +Another is to concatenate a regular string containing an escaped backslash to the +raw string:: + + >>> r'C:\this\will\work' '\\' + 'C:\\this\\will\\work\\' + +It is also possible to use :func:`os.path.join` to append a backslash on Windows:: + + >>> os.path.join(r'C:\this\will\work', '') + 'C:\\this\\will\\work\\' + +Note that while a backslash will "escape" a quote for the purposes of +determining where the raw string ends, no escaping occurs when interpreting the +value of the raw string. That is, the backslash remains present in the value of +the raw string:: + + >>> r'backslash\'preserved' + "backslash\\'preserved" + +Also see the specification in the :ref:`language reference <strings>`. + Performance =========== diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 558b1c3eec60..b7a89905e4f3 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -189,6 +189,11 @@ the first quote:: >>> print(r'C:\some\name') # note the r before the quote C:\some\name +There is one subtle aspect to raw strings: a raw string may not end in +an odd number of ``\`` characters; see +:ref:`the FAQ entry <faq-programming-raw-string-backslash>` for more information +and workarounds. + String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at From webhook-mailer at python.org Wed Dec 28 00:45:51 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 05:45:51 -0000 Subject: [Python-checkins] gh-55688: Add note about ending backslashes for raw strings (GH-94768) Message-ID: <mailman.3111.1672206352.3313.python-checkins@python.org> https://github.com/python/cpython/commit/de621281ce704546b87334fd7da7a7ec534719b7 commit: de621281ce704546b87334fd7da7a7ec534719b7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T21:45:45-08:00 summary: gh-55688: Add note about ending backslashes for raw strings (GH-94768) (cherry picked from commit b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: M Doc/faq/programming.rst M Doc/tutorial/introduction.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index c396e2b081fc..ba42289f3466 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1026,6 +1026,46 @@ What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? See the :ref:`unicode-howto`. +.. _faq-programming-raw-string-backslash: + +Can I end a raw string with an odd number of backslashes? +--------------------------------------------------------- + +A raw string ending with an odd number of backslashes will escape the string's quote:: + + >>> r'C:\this\will\not\work\' + File "<stdin>", line 1 + r'C:\this\will\not\work\' + ^ + SyntaxError: unterminated string literal (detected at line 1) + +There are several workarounds for this. One is to use regular strings and double +the backslashes:: + + >>> 'C:\\this\\will\\work\\' + 'C:\\this\\will\\work\\' + +Another is to concatenate a regular string containing an escaped backslash to the +raw string:: + + >>> r'C:\this\will\work' '\\' + 'C:\\this\\will\\work\\' + +It is also possible to use :func:`os.path.join` to append a backslash on Windows:: + + >>> os.path.join(r'C:\this\will\work', '') + 'C:\\this\\will\\work\\' + +Note that while a backslash will "escape" a quote for the purposes of +determining where the raw string ends, no escaping occurs when interpreting the +value of the raw string. That is, the backslash remains present in the value of +the raw string:: + + >>> r'backslash\'preserved' + "backslash\\'preserved" + +Also see the specification in the :ref:`language reference <strings>`. + Performance =========== diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 558b1c3eec60..b7a89905e4f3 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -189,6 +189,11 @@ the first quote:: >>> print(r'C:\some\name') # note the r before the quote C:\some\name +There is one subtle aspect to raw strings: a raw string may not end in +an odd number of ``\`` characters; see +:ref:`the FAQ entry <faq-programming-raw-string-backslash>` for more information +and workarounds. + String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at From webhook-mailer at python.org Wed Dec 28 00:46:09 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 05:46:09 -0000 Subject: [Python-checkins] GH-100192: fix `asyncio` subprocess tests to pass env vars to subprocess (#100569) Message-ID: <mailman.3112.1672206370.3313.python-checkins@python.org> https://github.com/python/cpython/commit/6835184a44389747dd151023b31a20ff75d6a402 commit: 6835184a44389747dd151023b31a20ff75d6a402 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T11:16:04+05:30 summary: GH-100192: fix `asyncio` subprocess tests to pass env vars to subprocess (#100569) files: M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 1ae290a003f2..f1ad10a9903f 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -698,7 +698,8 @@ async def check_stdout_output(self, coro, output): def test_create_subprocess_env_shell(self) -> None: async def main() -> None: cmd = f'''{sys.executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' - env = {"FOO": 'bar'} + env = os.environ.copy() + env["FOO"] = "bar" proc = await asyncio.create_subprocess_shell( cmd, env=env, stdout=subprocess.PIPE ) @@ -710,7 +711,8 @@ def test_create_subprocess_env_exec(self) -> None: async def main() -> None: cmd = [sys.executable, "-c", "import os, sys; sys.stdout.write(os.getenv('FOO'))"] - env = {"FOO": 'baz'} + env = os.environ.copy() + env["FOO"] = "baz" proc = await asyncio.create_subprocess_exec( *cmd, env=env, stdout=subprocess.PIPE ) From webhook-mailer at python.org Wed Dec 28 05:56:18 2022 From: webhook-mailer at python.org (cjw296) Date: Wed, 28 Dec 2022 10:56:18 -0000 Subject: [Python-checkins] Remove skipped test in test for async mocks. (#100559) Message-ID: <mailman.3113.1672224979.3313.python-checkins@python.org> https://github.com/python/cpython/commit/984894a9a25c0f8298565b0c0c2e1f41917e4f88 commit: 984894a9a25c0f8298565b0c0c2e1f41917e4f88 branch: main author: Chris Withers <chris at withers.org> committer: cjw296 <chris at withers.org> date: 2022-12-28T10:55:50Z summary: Remove skipped test in test for async mocks. (#100559) Remove skipped test. See discussion on https://github.com/python/cpython/pull/25326. Fix is apparently here, but no-one is confident to review and land: https://github.com/python/cpython/pull/25347. files: M Lib/test/test_unittest/testmock/testasync.py diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 471162dc5050..5f12f9f95667 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -218,10 +218,6 @@ def test_create_autospec_instance(self): with self.assertRaises(RuntimeError): create_autospec(async_func, instance=True) - @unittest.skip('Broken test from https://bugs.python.org/issue37251') - def test_create_autospec_awaitable_class(self): - self.assertIsInstance(create_autospec(AwaitableClass), AsyncMock) - def test_create_autospec(self): spec = create_autospec(async_func_args) awaitable = spec(1, 2, c=3) From webhook-mailer at python.org Wed Dec 28 06:14:05 2022 From: webhook-mailer at python.org (rhettinger) Date: Wed, 28 Dec 2022 11:14:05 -0000 Subject: [Python-checkins] Improve factor() recipe and fix its tests (GH-100576) Message-ID: <mailman.3114.1672226046.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2d524068351a33feafa905becc148f3447697e92 commit: 2d524068351a33feafa905becc148f3447697e92 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-28T03:13:58-08:00 summary: Improve factor() recipe and fix its tests (GH-100576) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index b3634aecd10d..f1277dfdbdc6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -899,18 +899,16 @@ which incur interpreter overhead. def factor(n): "Prime factors of n." - # factor(97) --> 97 - # factor(98) --> 2 7 7 # factor(99) --> 3 3 11 - for prime in sieve(n+1): - while True: + for prime in sieve(math.isqrt(n) + 1): + while n >= prime: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient - if n == 1: - return + if n >= 2: + yield n def flatten(list_of_lists): "Flatten one level of nesting" @@ -1266,33 +1264,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True - list(factor(0)) + >>> list(factor(0)) [] - list(factor(1)) + >>> list(factor(1)) [] - list(factor(2)) + >>> list(factor(2)) [2] - list(factor(3)) + >>> list(factor(3)) [3] - list(factor(4)) + >>> list(factor(4)) [2, 2] - list(factor(5)) + >>> list(factor(5)) [5] - list(factor(6)) + >>> list(factor(6)) [2, 3] - list(factor(7)) + >>> list(factor(7)) [7] - list(factor(8)) + >>> list(factor(8)) [2, 2, 2] - list(factor(9)) + >>> list(factor(9)) [3, 3] - list(factor(10)) + >>> list(factor(10)) [2, 5] - all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(999953*999983)) + [999953, 999983] + >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) True - all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) True - all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 06:22:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 11:22:17 -0000 Subject: [Python-checkins] Improve factor() recipe and fix its tests (GH-100576) Message-ID: <mailman.3115.1672226537.3313.python-checkins@python.org> https://github.com/python/cpython/commit/95fa61cbce17a13d45aa0d0d85f0ad8e48f3edbd commit: 95fa61cbce17a13d45aa0d0d85f0ad8e48f3edbd branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T03:22:11-08:00 summary: Improve factor() recipe and fix its tests (GH-100576) (cherry picked from commit 2d524068351a33feafa905becc148f3447697e92) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 219ad95c76a1..221863cbbcc5 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -860,18 +860,16 @@ which incur interpreter overhead. def factor(n): "Prime factors of n." - # factor(97) --> 97 - # factor(98) --> 2 7 7 # factor(99) --> 3 3 11 - for prime in sieve(n+1): - while True: + for prime in sieve(math.isqrt(n) + 1): + while n >= prime: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient - if n == 1: - return + if n >= 2: + yield n def flatten(list_of_lists): "Flatten one level of nesting" @@ -1236,33 +1234,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True - list(factor(0)) + >>> list(factor(0)) [] - list(factor(1)) + >>> list(factor(1)) [] - list(factor(2)) + >>> list(factor(2)) [2] - list(factor(3)) + >>> list(factor(3)) [3] - list(factor(4)) + >>> list(factor(4)) [2, 2] - list(factor(5)) + >>> list(factor(5)) [5] - list(factor(6)) + >>> list(factor(6)) [2, 3] - list(factor(7)) + >>> list(factor(7)) [7] - list(factor(8)) + >>> list(factor(8)) [2, 2, 2] - list(factor(9)) + >>> list(factor(9)) [3, 3] - list(factor(10)) + >>> list(factor(10)) [2, 5] - all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(999953*999983)) + [999953, 999983] + >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) True - all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) True - all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 07:36:40 2022 From: webhook-mailer at python.org (cjw296) Date: Wed, 28 Dec 2022 12:36:40 -0000 Subject: [Python-checkins] Fix mock code coverage. (#100580) Message-ID: <mailman.3116.1672231001.3313.python-checkins@python.org> https://github.com/python/cpython/commit/457c1f4a19a096a52d6553687c7c4cee415818dc commit: 457c1f4a19a096a52d6553687c7c4cee415818dc branch: main author: Chris Withers <chris at withers.org> committer: cjw296 <chris at withers.org> date: 2022-12-28T12:36:26Z summary: Fix mock code coverage. (#100580) files: M Lib/test/test_unittest/testmock/testsealable.py diff --git a/Lib/test/test_unittest/testmock/testsealable.py b/Lib/test/test_unittest/testmock/testsealable.py index e0c38293cffd..8bf98cfa562b 100644 --- a/Lib/test/test_unittest/testmock/testsealable.py +++ b/Lib/test/test_unittest/testmock/testsealable.py @@ -175,15 +175,12 @@ def test_seal_with_autospec(self): # https://bugs.python.org/issue45156 class Foo: foo = 0 - def bar1(self): - return 1 - def bar2(self): - return 2 + def bar1(self): pass + def bar2(self): pass class Baz: baz = 3 - def ban(self): - return 4 + def ban(self): pass for spec_set in (True, False): with self.subTest(spec_set=spec_set): From webhook-mailer at python.org Wed Dec 28 10:32:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 15:32:23 -0000 Subject: [Python-checkins] gh-100577: Replace `assert(0)` with `Py_UNREACHABLE` in `symtable.c` (#100579) Message-ID: <mailman.3117.1672241544.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5e1adb4f8861f2a5969952d24c8ad0ce8ec0a8ec commit: 5e1adb4f8861f2a5969952d24c8ad0ce8ec0a8ec branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T21:01:53+05:30 summary: gh-100577: Replace `assert(0)` with `Py_UNREACHABLE` in `symtable.c` (#100579) files: M Python/symtable.c diff --git a/Python/symtable.c b/Python/symtable.c index fb2bb7d83835..3c130186d0ad 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1539,7 +1539,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* We should always find either a FunctionBlock, ModuleBlock or ClassBlock and should never fall to this case */ - assert(0); + Py_UNREACHABLE(); return 0; } From webhook-mailer at python.org Wed Dec 28 13:07:12 2022 From: webhook-mailer at python.org (zware) Date: Wed, 28 Dec 2022 18:07:12 -0000 Subject: [Python-checkins] gh-100540: Remove unnecessary '-DMACOSX' for ctypes on macOS (GH-100542) Message-ID: <mailman.3118.1672250833.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fd4b55ff17b1525d36f1e81762fca6f1a7bc59c5 commit: fd4b55ff17b1525d36f1e81762fca6f1a7bc59c5 branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-28T12:06:58-06:00 summary: gh-100540: Remove unnecessary '-DMACOSX' for ctypes on macOS (GH-100542) The define was only used to protect a `#pragma clang diagnostic` setting, which is already better guarded by `__clang__` anwyay. files: M Modules/_ctypes/callbacks.c M configure M configure.ac diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 459632b90907..f6880889dc0f 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -426,7 +426,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); goto error; #else -#if defined(__clang__) || defined(MACOSX) +#if defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif @@ -436,7 +436,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, #endif result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); -#if defined(__clang__) || defined(MACOSX) +#if defined(__clang__) #pragma clang diagnostic pop #endif #if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5))) diff --git a/configure b/configure index 3f8daf9dad5f..72299939892d 100755 --- a/configure +++ b/configure @@ -12846,7 +12846,7 @@ if test "x$have_libffi" = xyes; then : case $ac_sys_system in #( Darwin) : - as_fn_append LIBFFI_CFLAGS " -I\$(srcdir)/Modules/_ctypes/darwin -DMACOSX" + as_fn_append LIBFFI_CFLAGS " -I\$(srcdir)/Modules/_ctypes/darwin" ctypes_malloc_closure=yes ;; #( sunos5) : diff --git a/configure.ac b/configure.ac index 734a4db83899..d6499be4c007 100644 --- a/configure.ac +++ b/configure.ac @@ -3769,7 +3769,7 @@ AS_VAR_IF([have_libffi], [yes], [ AS_CASE([$ac_sys_system], [Darwin], [ dnl when do we need USING_APPLE_OS_LIBFFI? - AS_VAR_APPEND([LIBFFI_CFLAGS], [" -I\$(srcdir)/Modules/_ctypes/darwin -DMACOSX"]) + AS_VAR_APPEND([LIBFFI_CFLAGS], [" -I\$(srcdir)/Modules/_ctypes/darwin"]) ctypes_malloc_closure=yes ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] From webhook-mailer at python.org Wed Dec 28 14:36:15 2022 From: webhook-mailer at python.org (zware) Date: Wed, 28 Dec 2022 19:36:15 -0000 Subject: [Python-checkins] gh-100540: Remove obsolete '--with-system-ffi' configure option (GH-100544) Message-ID: <mailman.3119.1672256176.3313.python-checkins@python.org> https://github.com/python/cpython/commit/25590eb5dee5176f3ac60916b19450f8198e7ffc commit: 25590eb5dee5176f3ac60916b19450f8198e7ffc branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-28T13:36:03-06:00 summary: gh-100540: Remove obsolete '--with-system-ffi' configure option (GH-100544) It has had no effect on non-macOS platforms for a long time, and has had the non-obvious effect of invoking `pkg_config` and not setting `-DUSING_APPLE_OS_LIBFFI` on macOS since GH-22855. files: A Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst M Doc/using/configure.rst M configure M configure.ac diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index db4bf7412292..8fa8d250d533 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -471,11 +471,6 @@ Libraries options Build the :mod:`pyexpat` module using an installed ``expat`` library (default is no). -.. cmdoption:: --with-system-ffi - - Build the :mod:`_ctypes` extension module using an installed ``ffi`` - library, see the :mod:`ctypes` module (default is system-dependent). - .. cmdoption:: --with-system-libmpdec Build the ``_decimal`` extension module using an installed ``mpdec`` diff --git a/Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst b/Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst new file mode 100644 index 000000000000..c924ab520f3d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst @@ -0,0 +1,8 @@ +Removed the ``--with-system-ffi`` ``configure`` option; ``libffi`` must +now always be supplied by the system on all non-Windows platforms. The +option has had no effect on non-Darwin platforms for several releases, and +in 3.11 only had the non-obvious effect of invoking ``pkg-config`` to +find ``libffi`` and never setting ``-DUSING_APPLE_OS_LIBFFI``. Now on +Darwin platforms ``configure`` will first check for the OS ``libffi`` and +then fall back to the same processing as other platforms if it is not +found. diff --git a/configure b/configure index 72299939892d..6afd1e9c367c 100755 --- a/configure +++ b/configure @@ -1064,7 +1064,6 @@ with_hash_algorithm with_tzpath with_libs with_system_expat -with_system_ffi with_system_libmpdec with_decimal_contextvar enable_loadable_sqlite_extensions @@ -1847,9 +1846,6 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-ffi build _ctypes module using an installed ffi library, - see Doc/library/ctypes.rst (default is - system-dependent) --with-system-libmpdec build _decimal module using an installed libmpdec library, see Doc/library/decimal.rst (default is no) --with-decimal-contextvar @@ -12516,43 +12512,77 @@ fi -# Check for use of the system libffi library -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-ffi" >&5 -$as_echo_n "checking for --with-system-ffi... " >&6; } +have_libffi=missing +if test "x$ac_sys_system" = xDarwin; then : -# Check whether --with-system_ffi was given. -if test "${with_system_ffi+set}" = set; then : - withval=$with_system_ffi; -fi + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS -if test "$ac_sys_system" = "Darwin" -then - case "$with_system_ffi" in - "") - with_system_ffi="no" - ;; - yes|no) - ;; - *) - as_fn_error $? "--with-system-ffi accepts no arguments" "$LINENO" 5 - ;; - esac - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_system_ffi" >&5 -$as_echo "$with_system_ffi" >&6; } + CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" + ac_fn_c_check_header_mongrel "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" +if test "x$ac_cv_header_ffi_h" = xyes; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_call in -lffi" >&5 +$as_echo_n "checking for ffi_call in -lffi... " >&6; } +if ${ac_cv_lib_ffi_ffi_call+:} false; then : + $as_echo_n "(cached) " >&6 else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - if test "$with_system_ffi" != "" - then - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with(out)-system-ffi is ignored on this platform" >&5 -$as_echo "$as_me: WARNING: --with(out)-system-ffi is ignored on this platform" >&2;} - fi - with_system_ffi="yes" + ac_check_lib_save_LIBS=$LIBS +LIBS="-lffi $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (); +int +main () +{ +return ffi_call (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_ffi_ffi_call=yes +else + ac_cv_lib_ffi_ffi_call=no fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 +$as_echo "$ac_cv_lib_ffi_ffi_call" >&6; } +if test "x$ac_cv_lib_ffi_ffi_call" = xyes; then : -have_libffi=missing -if test "x$with_system_ffi" = xyes; then : + have_libffi=yes + LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" + LIBFFI_LIBS="-lffi" + +fi + + +fi + + + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + +fi +if test "x$have_libffi" = xmissing; then : pkg_failed=no @@ -12762,80 +12792,6 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } have_libffi=yes -fi - -else - - if test "x$ac_sys_system" = xDarwin; then : - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" - ac_fn_c_check_header_mongrel "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" -if test "x$ac_cv_header_ffi_h" = xyes; then : - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_call in -lffi" >&5 -$as_echo_n "checking for ffi_call in -lffi... " >&6; } -if ${ac_cv_lib_ffi_ffi_call+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lffi $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char ffi_call (); -int -main () -{ -return ffi_call (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_ffi_ffi_call=yes -else - ac_cv_lib_ffi_ffi_call=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 -$as_echo "$ac_cv_lib_ffi_ffi_call" >&6; } -if test "x$ac_cv_lib_ffi_ffi_call" = xyes; then : - - have_libffi=yes - LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" - LIBFFI_LIBS="-lffi" - -else - have_libffi=no -fi - - -fi - - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - fi fi diff --git a/configure.ac b/configure.ac index d6499be4c007..48736649a98e 100644 --- a/configure.ac +++ b/configure.ac @@ -3705,36 +3705,22 @@ AS_VAR_IF([with_system_expat], [yes], [ AC_SUBST([LIBEXPAT_CFLAGS]) AC_SUBST([LIBEXPAT_INTERNAL]) -# Check for use of the system libffi library -AC_MSG_CHECKING(for --with-system-ffi) -AC_ARG_WITH(system_ffi, - AS_HELP_STRING([--with-system-ffi], [build _ctypes module using an installed ffi library, see Doc/library/ctypes.rst (default is system-dependent)]),,,) - -if test "$ac_sys_system" = "Darwin" -then - case "$with_system_ffi" in - "") - with_system_ffi="no" - ;; - yes|no) - ;; - *) - AC_MSG_ERROR([--with-system-ffi accepts no arguments]) - ;; - esac - AC_MSG_RESULT($with_system_ffi) -else - AC_MSG_RESULT(yes) - if test "$with_system_ffi" != "" - then - AC_MSG_WARN([--with(out)-system-ffi is ignored on this platform]) - fi - with_system_ffi="yes" -fi - dnl detect libffi have_libffi=missing -AS_VAR_IF([with_system_ffi], [yes], [ +AS_VAR_IF([ac_sys_system], [Darwin], [ + WITH_SAVE_ENV([ + CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" + AC_CHECK_HEADER([ffi.h], [ + AC_CHECK_LIB([ffi], [ffi_call], [ + dnl use ffi from SDK root + have_libffi=yes + LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" + LIBFFI_LIBS="-lffi" + ]) + ]) + ]) +]) +AS_VAR_IF([have_libffi], [missing], [ PKG_CHECK_MODULES([LIBFFI], [libffi], [have_libffi=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" @@ -3748,20 +3734,6 @@ AS_VAR_IF([with_system_ffi], [yes], [ ]) ]) ]) -], [ - AS_VAR_IF([ac_sys_system], [Darwin], [ - WITH_SAVE_ENV([ - CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" - AC_CHECK_HEADER([ffi.h], [ - AC_CHECK_LIB([ffi], [ffi_call], [ - dnl use ffi from SDK root - have_libffi=yes - LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" - LIBFFI_LIBS="-lffi" - ], [have_libffi=no]) - ]) - ]) - ]) ]) AS_VAR_IF([have_libffi], [yes], [ From webhook-mailer at python.org Wed Dec 28 14:38:58 2022 From: webhook-mailer at python.org (zware) Date: Wed, 28 Dec 2022 19:38:58 -0000 Subject: [Python-checkins] gh-100540: Remove unused Modules/_ctypes/libffi_osx/ (GH-100543) Message-ID: <mailman.3120.1672256339.3313.python-checkins@python.org> https://github.com/python/cpython/commit/025b5c37800ce7a0e487a522171b479bf8235267 commit: 025b5c37800ce7a0e487a522171b479bf8235267 branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-28T13:38:52-06:00 summary: gh-100540: Remove unused Modules/_ctypes/libffi_osx/ (GH-100543) It was an ancient, modified copy of libffi that has not been in use since GH-22855. files: D Modules/_ctypes/libffi_osx/LICENSE D Modules/_ctypes/libffi_osx/README D Modules/_ctypes/libffi_osx/README.pyobjc D Modules/_ctypes/libffi_osx/ffi.c D Modules/_ctypes/libffi_osx/include/ffi.h D Modules/_ctypes/libffi_osx/include/ffi_common.h D Modules/_ctypes/libffi_osx/include/fficonfig.h D Modules/_ctypes/libffi_osx/include/ffitarget.h D Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h D Modules/_ctypes/libffi_osx/include/x86-ffitarget.h D Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S D Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h D Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S D Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c D Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S D Modules/_ctypes/libffi_osx/types.c D Modules/_ctypes/libffi_osx/x86/darwin64.S D Modules/_ctypes/libffi_osx/x86/x86-darwin.S D Modules/_ctypes/libffi_osx/x86/x86-ffi64.c D Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c M Tools/c-analyzer/cpython/_parser.py M Tools/patchcheck/patchcheck.py diff --git a/Modules/_ctypes/libffi_osx/LICENSE b/Modules/_ctypes/libffi_osx/LICENSE deleted file mode 100644 index f591795152d6..000000000000 --- a/Modules/_ctypes/libffi_osx/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -libffi - Copyright (c) 1996-2003 Red Hat, Inc. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -``Software''), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. diff --git a/Modules/_ctypes/libffi_osx/README b/Modules/_ctypes/libffi_osx/README deleted file mode 100644 index 54f00e3ec1ff..000000000000 --- a/Modules/_ctypes/libffi_osx/README +++ /dev/null @@ -1,500 +0,0 @@ -This directory contains the libffi package, which is not part of GCC but -shipped with GCC as convenience. - -Status -====== - -libffi-2.00 has not been released yet! This is a development snapshot! - -libffi-1.20 was released on October 5, 1998. Check the libffi web -page for updates: <URL:http://sources.redhat.com/libffi/>. - - -What is libffi? -=============== - -Compilers for high level languages generate code that follow certain -conventions. These conventions are necessary, in part, for separate -compilation to work. One such convention is the "calling -convention". The "calling convention" is essentially a set of -assumptions made by the compiler about where function arguments will -be found on entry to a function. A "calling convention" also specifies -where the return value for a function is found. - -Some programs may not know at the time of compilation what arguments -are to be passed to a function. For instance, an interpreter may be -told at run-time about the number and types of arguments used to call -a given function. Libffi can be used in such programs to provide a -bridge from the interpreter program to compiled code. - -The libffi library provides a portable, high level programming -interface to various calling conventions. This allows a programmer to -call any function specified by a call interface description at run -time. - -Ffi stands for Foreign Function Interface. A foreign function -interface is the popular name for the interface that allows code -written in one language to call code written in another language. The -libffi library really only provides the lowest, machine dependent -layer of a fully featured foreign function interface. A layer must -exist above libffi that handles type conversions for values passed -between the two languages. - - -Supported Platforms and Prerequisites -===================================== - -Libffi has been ported to: - - SunOS 4.1.3 & Solaris 2.x (SPARC-V8, SPARC-V9) - - Irix 5.3 & 6.2 (System V/o32 & n32) - - Intel x86 - Linux (System V ABI) - - Alpha - Linux and OSF/1 - - m68k - Linux (System V ABI) - - PowerPC - Linux (System V ABI, Darwin, AIX) - - ARM - Linux (System V ABI) - -Libffi has been tested with the egcs 1.0.2 gcc compiler. Chances are -that other versions will work. Libffi has also been built and tested -with the SGI compiler tools. - -On PowerPC, the tests failed (see the note below). - -You must use GNU make to build libffi. SGI's make will not work. -Sun's probably won't either. - -If you port libffi to another platform, please let me know! I assume -that some will be easy (x86 NetBSD), and others will be more difficult -(HP). - - -Installing libffi -================= - -[Note: before actually performing any of these installation steps, - you may wish to read the "Platform Specific Notes" below.] - -First you must configure the distribution for your particular -system. Go to the directory you wish to build libffi in and run the -"configure" program found in the root directory of the libffi source -distribution. - -You may want to tell configure where to install the libffi library and -header files. To do that, use the --prefix configure switch. Libffi -will install under /usr/local by default. - -If you want to enable extra run-time debugging checks use the the ---enable-debug configure switch. This is useful when your program dies -mysteriously while using libffi. - -Another useful configure switch is --enable-purify-safety. Using this -will add some extra code which will suppress certain warnings when you -are using Purify with libffi. Only use this switch when using -Purify, as it will slow down the library. - -Configure has many other options. Use "configure --help" to see them all. - -Once configure has finished, type "make". Note that you must be using -GNU make. SGI's make will not work. Sun's probably won't either. -You can ftp GNU make from prep.ai.mit.edu:/pub/gnu. - -To ensure that libffi is working as advertised, type "make test". - -To install the library and header files, type "make install". - - -Using libffi -============ - - The Basics - ---------- - -Libffi assumes that you have a pointer to the function you wish to -call and that you know the number and types of arguments to pass it, -as well as the return type of the function. - -The first thing you must do is create an ffi_cif object that matches -the signature of the function you wish to call. The cif in ffi_cif -stands for Call InterFace. To prepare a call interface object, use the -following function: - -ffi_status ffi_prep_cif(ffi_cif *cif, ffi_abi abi, - unsigned int nargs, - ffi_type *rtype, ffi_type **atypes); - - CIF is a pointer to the call interface object you wish - to initialize. - - ABI is an enum that specifies the calling convention - to use for the call. FFI_DEFAULT_ABI defaults - to the system's native calling convention. Other - ABI's may be used with care. They are system - specific. - - NARGS is the number of arguments this function accepts. - libffi does not yet support vararg functions. - - RTYPE is a pointer to an ffi_type structure that represents - the return type of the function. Ffi_type objects - describe the types of values. libffi provides - ffi_type objects for many of the native C types: - signed int, unsigned int, signed char, unsigned char, - etc. There is also a pointer ffi_type object and - a void ffi_type. Use &ffi_type_void for functions that - don't return values. - - ATYPES is a vector of ffi_type pointers. ARGS must be NARGS long. - If NARGS is 0, this is ignored. - - -ffi_prep_cif will return a status code that you are responsible -for checking. It will be one of the following: - - FFI_OK - All is good. - - FFI_BAD_TYPEDEF - One of the ffi_type objects that ffi_prep_cif - came across is bad. - - -Before making the call, the VALUES vector should be initialized -with pointers to the appropriate argument values. - -To call the function using the initialized ffi_cif, use the -ffi_call function: - -void ffi_call(ffi_cif *cif, void *fn, void *rvalue, void **avalues); - - CIF is a pointer to the ffi_cif initialized specifically - for this function. - - FN is a pointer to the function you want to call. - - RVALUE is a pointer to a chunk of memory that is to hold the - result of the function call. Currently, it must be - at least one word in size (except for the n32 version - under Irix 6.x, which must be a pointer to an 8 byte - aligned value (a long long). It must also be at least - word aligned (depending on the return type, and the - system's alignment requirements). If RTYPE is - &ffi_type_void, this is ignored. If RVALUE is NULL, - the return value is discarded. - - AVALUES is a vector of void* that point to the memory locations - holding the argument values for a call. - If NARGS is 0, this is ignored. - - -If you are expecting a return value from FN it will have been stored -at RVALUE. - - - - An Example - ---------- - -Here is a trivial example that calls puts() a few times. - - #include <stdio.h> - #include <ffi.h> - - int main() - { - ffi_cif cif; - ffi_type *args[1]; - void *values[1]; - char *s; - int rc; - - /* Initialize the argument info vectors */ - args[0] = &ffi_type_uint; - values[0] = &s; - - /* Initialize the cif */ - if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, 1, - &ffi_type_uint, args) == FFI_OK) - { - s = "Hello World!"; - ffi_call(&cif, puts, &rc, values); - /* rc now holds the result of the call to puts */ - - /* values holds a pointer to the function's arg, so to - call puts() again all we need to do is change the - value of s */ - s = "This is cool!"; - ffi_call(&cif, puts, &rc, values); - } - - return 0; - } - - - - Aggregate Types - --------------- - -Although libffi has no special support for unions or bit-fields, it is -perfectly happy passing structures back and forth. You must first -describe the structure to libffi by creating a new ffi_type object -for it. Here is the definition of ffi_type: - - typedef struct _ffi_type - { - unsigned size; - short alignment; - short type; - struct _ffi_type **elements; - } ffi_type; - -All structures must have type set to FFI_TYPE_STRUCT. You may set -size and alignment to 0. These will be calculated and reset to the -appropriate values by ffi_prep_cif(). - -elements is a NULL terminated array of pointers to ffi_type objects -that describe the type of the structure elements. These may, in turn, -be structure elements. - -The following example initializes a ffi_type object representing the -tm struct from Linux's time.h: - - struct tm { - int tm_sec; - int tm_min; - int tm_hour; - int tm_mday; - int tm_mon; - int tm_year; - int tm_wday; - int tm_yday; - int tm_isdst; - /* Those are for future use. */ - long int __tm_gmtoff__; - __const char *__tm_zone__; - }; - - { - ffi_type tm_type; - ffi_type *tm_type_elements[12]; - int i; - - tm_type.size = tm_type.alignment = 0; - tm_type.elements = &tm_type_elements; - - for (i = 0; i < 9; i++) - tm_type_elements[i] = &ffi_type_sint; - - tm_type_elements[9] = &ffi_type_slong; - tm_type_elements[10] = &ffi_type_pointer; - tm_type_elements[11] = NULL; - - /* tm_type can now be used to represent tm argument types and - return types for ffi_prep_cif() */ - } - - - -Platform Specific Notes -======================= - - Intel x86 - --------- - -There are no known problems with the x86 port. - - Sun SPARC - SunOS 4.1.3 & Solaris 2.x - ------------------------------------- - -You must use GNU Make to build libffi on Sun platforms. - - MIPS - Irix 5.3 & 6.x - --------------------- - -Irix 6.2 and better supports three different calling conventions: o32, -n32 and n64. Currently, libffi only supports both o32 and n32 under -Irix 6.x, but only o32 under Irix 5.3. Libffi will automatically be -configured for whichever calling convention it was built for. - -By default, the configure script will try to build libffi with the GNU -development tools. To build libffi with the SGI development tools, set -the environment variable CC to either "cc -32" or "cc -n32" before -running configure under Irix 6.x (depending on whether you want an o32 -or n32 library), or just "cc" for Irix 5.3. - -With the n32 calling convention, when returning structures smaller -than 16 bytes, be sure to provide an RVALUE that is 8 byte aligned. -Here's one way of forcing this: - - double struct_storage[2]; - my_small_struct *s = (my_small_struct *) struct_storage; - /* Use s for RVALUE */ - -If you don't do this you are liable to get spurious bus errors. - -"long long" values are not supported yet. - -You must use GNU Make to build libffi on SGI platforms. - - ARM - System V ABI - ------------------ - -The ARM port was performed on a NetWinder running ARM Linux ELF -(2.0.31) and gcc 2.8.1. - - - - PowerPC System V ABI - -------------------- - -There are two `System V ABI's which libffi implements for PowerPC. -They differ only in how small structures are returned from functions. - -In the FFI_SYSV version, structures that are 8 bytes or smaller are -returned in registers. This is what GCC does when it is configured -for solaris, and is what the System V ABI I have (dated September -1995) says. - -In the FFI_GCC_SYSV version, all structures are returned the same way: -by passing a pointer as the first argument to the function. This is -what GCC does when it is configured for linux or a generic sysv -target. - -EGCS 1.0.1 (and probably other versions of EGCS/GCC) also has a -inconsistency with the SysV ABI: When a procedure is called with many -floating-point arguments, some of them get put on the stack. They are -all supposed to be stored in double-precision format, even if they are -only single-precision, but EGCS stores single-precision arguments as -single-precision anyway. This causes one test to fail (the `many -arguments' test). - - -What's With The Cryptic Comments? -================================= - -You might notice a number of cryptic comments in the code, delimited -by /*@ and @*/. These are annotations read by the program LCLint, a -tool for statically checking C programs. You can read all about it at -<http://larch-www.lcs.mit.edu:8001/larch/lclint/index.html>. - - -History -======= - -1.20 Oct-5-98 - Raffaele Sena produces ARM port. - -1.19 Oct-5-98 - Fixed x86 long double and long long return support. - m68k bug fixes from Andreas Schwab. - Patch for DU assembler compatibility for the Alpha from Richard - Henderson. - -1.18 Apr-17-98 - Bug fixes and MIPS configuration changes. - -1.17 Feb-24-98 - Bug fixes and m68k port from Andreas Schwab. PowerPC port from - Geoffrey Keating. Various bug x86, Sparc and MIPS bug fixes. - -1.16 Feb-11-98 - Richard Henderson produces Alpha port. - -1.15 Dec-4-97 - Fixed an n32 ABI bug. New libtool, auto* support. - -1.14 May-13-97 - libtool is now used to generate shared and static libraries. - Fixed a minor portability problem reported by Russ McManus - <mcmanr at eq.gs.com>. - -1.13 Dec-2-96 - Added --enable-purify-safety to keep Purify from complaining - about certain low level code. - Sparc fix for calling functions with < 6 args. - Linux x86 a.out fix. - -1.12 Nov-22-96 - Added missing ffi_type_void, needed for supporting void return - types. Fixed test case for non MIPS machines. Cygnus Support - is now Cygnus Solutions. - -1.11 Oct-30-96 - Added notes about GNU make. - -1.10 Oct-29-96 - Added configuration fix for non GNU compilers. - -1.09 Oct-29-96 - Added --enable-debug configure switch. Clean-ups based on LCLint - feedback. ffi_mips.h is always installed. Many configuration - fixes. Fixed ffitest.c for sparc builds. - -1.08 Oct-15-96 - Fixed n32 problem. Many clean-ups. - -1.07 Oct-14-96 - Gordon Irlam rewrites v8.S again. Bug fixes. - -1.06 Oct-14-96 - Gordon Irlam improved the sparc port. - -1.05 Oct-14-96 - Interface changes based on feedback. - -1.04 Oct-11-96 - Sparc port complete (modulo struct passing bug). - -1.03 Oct-10-96 - Passing struct args, and returning struct values works for - all architectures/calling conventions. Expanded tests. - -1.02 Oct-9-96 - Added SGI n32 support. Fixed bugs in both o32 and Linux support. - Added "make test". - -1.01 Oct-8-96 - Fixed float passing bug in mips version. Restructured some - of the code. Builds cleanly with SGI tools. - -1.00 Oct-7-96 - First release. No public announcement. - - -Authors & Credits -================= - -libffi was written by Anthony Green <green at cygnus.com>. - -Portions of libffi were derived from Gianni Mariani's free gencall -library for Silicon Graphics machines. - -The closure mechanism was designed and implemented by Kresten Krab -Thorup. - -The Sparc port was derived from code contributed by the fine folks at -Visible Decisions Inc <http://www.vdi.com>. Further enhancements were -made by Gordon Irlam at Cygnus Solutions <http://www.cygnus.com>. - -The Alpha port was written by Richard Henderson at Cygnus Solutions. - -Andreas Schwab ported libffi to m68k Linux and provided a number of -bug fixes. - -Geoffrey Keating ported libffi to the PowerPC. - -Raffaele Sena ported libffi to the ARM. - -Jesper Skov and Andrew Haley both did more than their fair share of -stepping through the code and tracking down bugs. - -Thanks also to Tom Tromey for bug fixes and configuration help. - -Thanks to Jim Blandy, who provided some useful feedback on the libffi -interface. - -If you have a problem, or have found a bug, please send a note to -green at cygnus.com. diff --git a/Modules/_ctypes/libffi_osx/README.pyobjc b/Modules/_ctypes/libffi_osx/README.pyobjc deleted file mode 100644 index 405d85fed2de..000000000000 --- a/Modules/_ctypes/libffi_osx/README.pyobjc +++ /dev/null @@ -1,5 +0,0 @@ -This directory contains a slightly modified version of libffi, extracted from -the GCC source-tree. - -The only modifications are those that are necessary to compile libffi using -the Apple provided compiler and outside of the GCC source tree. diff --git a/Modules/_ctypes/libffi_osx/ffi.c b/Modules/_ctypes/libffi_osx/ffi.c deleted file mode 100644 index 1776b795e2f8..000000000000 --- a/Modules/_ctypes/libffi_osx/ffi.c +++ /dev/null @@ -1,227 +0,0 @@ -/* ----------------------------------------------------------------------- - prep_cif.c - Copyright (c) 1996, 1998 Red Hat, Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdbool.h> -#include <stdlib.h> - -/* Round up to FFI_SIZEOF_ARG. */ -#define STACK_ARG_SIZE(x) ALIGN(x, FFI_SIZEOF_ARG) - -/* Perform machine independent initialization of aggregate type - specifications. */ - -static ffi_status -initialize_aggregate( -/*@out@*/ ffi_type* arg) -{ -/*@-usedef@*/ - ffi_type** ptr; - - if (arg == NULL || arg->elements == NULL || - arg->size != 0 || arg->alignment != 0) - return FFI_BAD_TYPEDEF; - - ptr = &(arg->elements[0]); - - while ((*ptr) != NULL) - { - if (((*ptr)->size == 0) && (initialize_aggregate(*ptr) != FFI_OK)) - return FFI_BAD_TYPEDEF; - - /* Perform a sanity check on the argument type */ - FFI_ASSERT_VALID_TYPE(*ptr); - -#ifdef POWERPC_DARWIN - int curalign = (*ptr)->alignment; - - if (ptr != &(arg->elements[0])) - { - if (curalign > 4 && curalign != 16) - curalign = 4; - } - - arg->size = ALIGN(arg->size, curalign); - arg->size += (*ptr)->size; - arg->alignment = (arg->alignment > curalign) ? - arg->alignment : curalign; -#else - arg->size = ALIGN(arg->size, (*ptr)->alignment); - arg->size += (*ptr)->size; - arg->alignment = (arg->alignment > (*ptr)->alignment) ? - arg->alignment : (*ptr)->alignment; -#endif - - ptr++; - } - - /* Structure size includes tail padding. This is important for - structures that fit in one register on ABIs like the PowerPC64 - Linux ABI that right justify small structs in a register. - It's also needed for nested structure layout, for example - struct A { long a; char b; }; struct B { struct A x; char y; }; - should find y at an offset of 2*sizeof(long) and result in a - total size of 3*sizeof(long). */ - arg->size = ALIGN(arg->size, arg->alignment); - - if (arg->size == 0) - return FFI_BAD_TYPEDEF; - - return FFI_OK; - -/*@=usedef@*/ -} - -#ifndef __CRIS__ -/* The CRIS ABI specifies structure elements to have byte - alignment only, so it completely overrides this functions, - which assumes "natural" alignment and padding. */ - -/* Perform machine independent ffi_cif preparation, then call - machine dependent routine. */ - -#if defined(X86_DARWIN) && !defined __x86_64__ - -static inline bool -struct_on_stack( - int size) -{ - if (size > 8) - return true; - - /* This is not what the ABI says, but is what is really implemented */ - switch (size) - { - case 1: - case 2: - case 4: - case 8: - return false; - - default: - return true; - } -} - -#endif // defined(X86_DARWIN) && !defined __x86_64__ - -// Arguments' ffi_type->alignment must be nonzero. -ffi_status -ffi_prep_cif( -/*@out@*/ /*@partial@*/ ffi_cif* cif, - ffi_abi abi, - unsigned int nargs, -/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, -/*@dependent@*/ ffi_type** atypes) -{ - unsigned int bytes = 0; - unsigned int i; - ffi_type** ptr; - - if (cif == NULL) - return FFI_BAD_TYPEDEF; - - if (abi <= FFI_FIRST_ABI || abi > FFI_DEFAULT_ABI) - return FFI_BAD_ABI; - - cif->abi = abi; - cif->arg_types = atypes; - cif->nargs = nargs; - cif->rtype = rtype; - cif->flags = 0; - - /* Initialize the return type if necessary */ - /*@-usedef@*/ - if ((cif->rtype->size == 0) && (initialize_aggregate(cif->rtype) != FFI_OK)) - return FFI_BAD_TYPEDEF; - /*@=usedef@*/ - - /* Perform a sanity check on the return type */ - FFI_ASSERT_VALID_TYPE(cif->rtype); - - /* x86-64 and s390 stack space allocation is handled in prep_machdep. */ -#if !defined M68K && !defined __x86_64__ && !defined S390 && !defined PA - /* Make space for the return structure pointer */ - if (cif->rtype->type == FFI_TYPE_STRUCT -#ifdef SPARC - && (cif->abi != FFI_V9 || cif->rtype->size > 32) -#endif -#ifdef X86_DARWIN - && (struct_on_stack(cif->rtype->size)) -#endif - ) - bytes = STACK_ARG_SIZE(sizeof(void*)); -#endif - - for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++) - { - /* Initialize any uninitialized aggregate type definitions */ - if (((*ptr)->size == 0) && (initialize_aggregate((*ptr)) != FFI_OK)) - return FFI_BAD_TYPEDEF; - - if ((*ptr)->alignment == 0) - return FFI_BAD_TYPEDEF; - - /* Perform a sanity check on the argument type, do this - check after the initialization. */ - FFI_ASSERT_VALID_TYPE(*ptr); - -#if defined(X86_DARWIN) - { - int align = (*ptr)->alignment; - - if (align > 4) - align = 4; - - if ((align - 1) & bytes) - bytes = ALIGN(bytes, align); - - bytes += STACK_ARG_SIZE((*ptr)->size); - } -#elif !defined __x86_64__ && !defined S390 && !defined PA -#ifdef SPARC - if (((*ptr)->type == FFI_TYPE_STRUCT - && ((*ptr)->size > 16 || cif->abi != FFI_V9)) - || ((*ptr)->type == FFI_TYPE_LONGDOUBLE - && cif->abi != FFI_V9)) - bytes += sizeof(void*); - else -#endif - { - /* Add any padding if necessary */ - if (((*ptr)->alignment - 1) & bytes) - bytes = ALIGN(bytes, (*ptr)->alignment); - - bytes += STACK_ARG_SIZE((*ptr)->size); - } -#endif - } - - cif->bytes = bytes; - - /* Perform machine dependent cif processing */ - return ffi_prep_cif_machdep(cif); -} -#endif /* not __CRIS__ */ diff --git a/Modules/_ctypes/libffi_osx/include/ffi.h b/Modules/_ctypes/libffi_osx/include/ffi.h deleted file mode 100644 index c104a5c89350..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ffi.h +++ /dev/null @@ -1,355 +0,0 @@ -/* -----------------------------------------------------------------*-C-*- - libffi PyOBJC - Copyright (c) 1996-2003 Red Hat, Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - - ----------------------------------------------------------------------- */ - -/* ------------------------------------------------------------------- - The basic API is described in the README file. - - The raw API is designed to bypass some of the argument packing - and unpacking on architectures for which it can be avoided. - - The closure API allows interpreted functions to be packaged up - inside a C function pointer, so that they can be called as C functions, - with no understanding on the client side that they are interpreted. - It can also be used in other cases in which it is necessary to package - up a user specified parameter and a function pointer as a single - function pointer. - - The closure API must be implemented in order to get its functionality, - e.g. for use by gij. Routines are provided to emulate the raw API - if the underlying platform doesn't allow faster implementation. - - More details on the raw and closure API can be found in: - - http://gcc.gnu.org/ml/java/1999-q3/msg00138.html - - and - - http://gcc.gnu.org/ml/java/1999-q3/msg00174.html - -------------------------------------------------------------------- */ - -#ifndef LIBFFI_H -#define LIBFFI_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* Specify which architecture libffi is configured for. */ -#ifdef MACOSX -# if defined(__i386__) || defined(__x86_64__) -# define X86_DARWIN -# elif defined(__ppc__) || defined(__ppc64__) -# define POWERPC_DARWIN -# else -# error "Unsupported MacOS X CPU type" -# endif -#else -#error "Unsupported OS type" -#endif - -/* ---- System configuration information --------------------------------- */ - -#include "ffitarget.h" -#include "fficonfig.h" - -#ifndef LIBFFI_ASM - -#include <stddef.h> -#include <limits.h> - -/* LONG_LONG_MAX is not always defined (not if STRICT_ANSI, for example). - But we can find it either under the correct ANSI name, or under GNU - C's internal name. */ -#ifdef LONG_LONG_MAX -# define FFI_LONG_LONG_MAX LONG_LONG_MAX -#else -# ifdef LLONG_MAX -# define FFI_LONG_LONG_MAX LLONG_MAX -# else -# ifdef __GNUC__ -# define FFI_LONG_LONG_MAX __LONG_LONG_MAX__ -# endif -# endif -#endif - -#if SCHAR_MAX == 127 -# define ffi_type_uchar ffi_type_uint8 -# define ffi_type_schar ffi_type_sint8 -#else -#error "char size not supported" -#endif - -#if SHRT_MAX == 32767 -# define ffi_type_ushort ffi_type_uint16 -# define ffi_type_sshort ffi_type_sint16 -#elif SHRT_MAX == 2147483647 -# define ffi_type_ushort ffi_type_uint32 -# define ffi_type_sshort ffi_type_sint32 -#else -#error "short size not supported" -#endif - -#if INT_MAX == 32767 -# define ffi_type_uint ffi_type_uint16 -# define ffi_type_sint ffi_type_sint16 -#elif INT_MAX == 2147483647 -# define ffi_type_uint ffi_type_uint32 -# define ffi_type_sint ffi_type_sint32 -#elif INT_MAX == 9223372036854775807 -# define ffi_type_uint ffi_type_uint64 -# define ffi_type_sint ffi_type_sint64 -#else -#error "int size not supported" -#endif - -#define ffi_type_ulong ffi_type_uint64 -#define ffi_type_slong ffi_type_sint64 - -#if LONG_MAX == 2147483647 -# if FFI_LONG_LONG_MAX != 9223372036854775807 -# error "no 64-bit data type supported" -# endif -#elif LONG_MAX != 9223372036854775807 -#error "long size not supported" -#endif - -/* The closure code assumes that this works on pointers, i.e. a size_t - can hold a pointer. */ - -typedef struct _ffi_type { - size_t size; - unsigned short alignment; - unsigned short type; -/*@null@*/ struct _ffi_type** elements; -} ffi_type; - -/* These are defined in types.c */ -extern ffi_type ffi_type_void; -extern ffi_type ffi_type_uint8; -extern ffi_type ffi_type_sint8; -extern ffi_type ffi_type_uint16; -extern ffi_type ffi_type_sint16; -extern ffi_type ffi_type_uint32; -extern ffi_type ffi_type_sint32; -extern ffi_type ffi_type_uint64; -extern ffi_type ffi_type_sint64; -extern ffi_type ffi_type_float; -extern ffi_type ffi_type_double; -extern ffi_type ffi_type_longdouble; -extern ffi_type ffi_type_pointer; - -typedef enum ffi_status { - FFI_OK = 0, - FFI_BAD_TYPEDEF, - FFI_BAD_ABI -} ffi_status; - -typedef unsigned FFI_TYPE; - -typedef struct ffi_cif { - ffi_abi abi; - unsigned nargs; -/*@dependent@*/ ffi_type** arg_types; -/*@dependent@*/ ffi_type* rtype; - unsigned bytes; - unsigned flags; -#ifdef FFI_EXTRA_CIF_FIELDS - FFI_EXTRA_CIF_FIELDS; -#endif -} ffi_cif; - -/* ---- Definitions for the raw API -------------------------------------- */ - -#ifndef FFI_SIZEOF_ARG -# if LONG_MAX == 2147483647 -# define FFI_SIZEOF_ARG 4 -# elif LONG_MAX == 9223372036854775807 -# define FFI_SIZEOF_ARG 8 -# endif -#endif - -typedef union { - ffi_sarg sint; - ffi_arg uint; - float flt; - char data[FFI_SIZEOF_ARG]; - void* ptr; -} ffi_raw; - -void -ffi_raw_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ ffi_raw* avalue); - -void -ffi_ptrarray_to_raw( - ffi_cif* cif, - void** args, - ffi_raw* raw); - -void -ffi_raw_to_ptrarray( - ffi_cif* cif, - ffi_raw* raw, - void** args); - -size_t -ffi_raw_size( - ffi_cif* cif); - -/* This is analogous to the raw API, except it uses Java parameter - packing, even on 64-bit machines. I.e. on 64-bit machines - longs and doubles are followed by an empty 64-bit word. */ -void -ffi_java_raw_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ ffi_raw* avalue); - -void -ffi_java_ptrarray_to_raw( - ffi_cif* cif, - void** args, - ffi_raw* raw); - -void -ffi_java_raw_to_ptrarray( - ffi_cif* cif, - ffi_raw* raw, - void** args); - -size_t -ffi_java_raw_size( - ffi_cif* cif); - -/* ---- Definitions for closures ----------------------------------------- */ - -#if FFI_CLOSURES - -typedef struct ffi_closure { - char tramp[FFI_TRAMPOLINE_SIZE]; - ffi_cif* cif; - void (*fun)(ffi_cif*,void*,void**,void*); - void* user_data; -} ffi_closure; - -ffi_status -ffi_prep_closure( - ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,void**,void*), - void* user_data); - -void ffi_closure_free(void *); -void *ffi_closure_alloc (size_t size, void **code); - -typedef struct ffi_raw_closure { - char tramp[FFI_TRAMPOLINE_SIZE]; - ffi_cif* cif; - -#if !FFI_NATIVE_RAW_API - /* if this is enabled, then a raw closure has the same layout - as a regular closure. We use this to install an intermediate - handler to do the transaltion, void** -> ffi_raw*. */ - void (*translate_args)(ffi_cif*,void*,void**,void*); - void* this_closure; -#endif - - void (*fun)(ffi_cif*,void*,ffi_raw*,void*); - void* user_data; -} ffi_raw_closure; - -ffi_status -ffi_prep_raw_closure( - ffi_raw_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,ffi_raw*,void*), - void* user_data); - -ffi_status -ffi_prep_java_raw_closure( - ffi_raw_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,ffi_raw*,void*), - void* user_data); - -#endif // FFI_CLOSURES - -/* ---- Public interface definition -------------------------------------- */ - -ffi_status -ffi_prep_cif( -/*@out@*/ /*@partial@*/ ffi_cif* cif, - ffi_abi abi, - unsigned int nargs, -/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, -/*@dependent@*/ ffi_type** atypes); - -void -ffi_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ void** avalue); - -/* Useful for eliminating compiler warnings */ -#define FFI_FN(f) ((void (*)(void))f) - -#endif // #ifndef LIBFFI_ASM -/* ---- Definitions shared with assembly code ---------------------------- */ - -/* If these change, update src/mips/ffitarget.h. */ -#define FFI_TYPE_VOID 0 -#define FFI_TYPE_INT 1 -#define FFI_TYPE_FLOAT 2 -#define FFI_TYPE_DOUBLE 3 - -#ifdef HAVE_LONG_DOUBLE -# define FFI_TYPE_LONGDOUBLE 4 -#else -# define FFI_TYPE_LONGDOUBLE FFI_TYPE_DOUBLE -#endif - -#define FFI_TYPE_UINT8 5 -#define FFI_TYPE_SINT8 6 -#define FFI_TYPE_UINT16 7 -#define FFI_TYPE_SINT16 8 -#define FFI_TYPE_UINT32 9 -#define FFI_TYPE_SINT32 10 -#define FFI_TYPE_UINT64 11 -#define FFI_TYPE_SINT64 12 -#define FFI_TYPE_STRUCT 13 -#define FFI_TYPE_POINTER 14 - -/* This should always refer to the last type code (for sanity checks) */ -#define FFI_TYPE_LAST FFI_TYPE_POINTER - -#ifdef __cplusplus -} -#endif - -#endif // #ifndef LIBFFI_H diff --git a/Modules/_ctypes/libffi_osx/include/ffi_common.h b/Modules/_ctypes/libffi_osx/include/ffi_common.h deleted file mode 100644 index 685a3580f4fe..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ffi_common.h +++ /dev/null @@ -1,102 +0,0 @@ -/* ----------------------------------------------------------------------- - ffi_common.h - Copyright (c) 1996 Red Hat, Inc. - - Common internal definitions and macros. Only necessary for building - libffi. - ----------------------------------------------------------------------- */ - -#ifndef FFI_COMMON_H -#define FFI_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "fficonfig.h" - -/* Do not move this. Some versions of AIX are very picky about where - this is positioned. */ -#ifdef __GNUC__ -# define alloca __builtin_alloca -#else -# if HAVE_ALLOCA_H -# include <alloca.h> -# else -# ifdef _AIX -# pragma alloca -# else -# ifndef alloca /* predefined by HP cc +Olibcalls */ -char* alloca(); -# endif -# endif -# endif -#endif - -/* Check for the existence of memcpy. */ -#if STDC_HEADERS -# include <string.h> -#else -# ifndef HAVE_MEMCPY -# define memcpy(d, s, n) bcopy((s), (d), (n)) -# endif -#endif - -/*#if defined(FFI_DEBUG) -#include <stdio.h> -#endif*/ - -#ifdef FFI_DEBUG -#include <stdio.h> - -/*@exits@*/ void -ffi_assert( -/*@temp@*/ char* expr, -/*@temp@*/ char* file, - int line); -void -ffi_stop_here(void); -void -ffi_type_test( -/*@temp@*/ /*@out@*/ ffi_type* a, -/*@temp@*/ char* file, - int line); - -# define FFI_ASSERT(x) ((x) ? (void)0 : ffi_assert(#x, __FILE__,__LINE__)) -# define FFI_ASSERT_AT(x, f, l) ((x) ? 0 : ffi_assert(#x, (f), (l))) -# define FFI_ASSERT_VALID_TYPE(x) ffi_type_test(x, __FILE__, __LINE__) -#else -# define FFI_ASSERT(x) -# define FFI_ASSERT_AT(x, f, l) -# define FFI_ASSERT_VALID_TYPE(x) -#endif // #ifdef FFI_DEBUG - -#define ALIGN(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1)) - -/* Perform machine dependent cif processing */ -ffi_status -ffi_prep_cif_machdep( - ffi_cif* cif); - -/* Extended cif, used in callback from assembly routine */ -typedef struct extended_cif { -/*@dependent@*/ ffi_cif* cif; -/*@dependent@*/ void* rvalue; -/*@dependent@*/ void** avalue; -} extended_cif; - -/* Terse sized type definitions. */ -typedef unsigned int UINT8 __attribute__((__mode__(__QI__))); -typedef signed int SINT8 __attribute__((__mode__(__QI__))); -typedef unsigned int UINT16 __attribute__((__mode__(__HI__))); -typedef signed int SINT16 __attribute__((__mode__(__HI__))); -typedef unsigned int UINT32 __attribute__((__mode__(__SI__))); -typedef signed int SINT32 __attribute__((__mode__(__SI__))); -typedef unsigned int UINT64 __attribute__((__mode__(__DI__))); -typedef signed int SINT64 __attribute__((__mode__(__DI__))); -typedef float FLOAT32; - -#ifdef __cplusplus -} -#endif - -#endif // #ifndef FFI_COMMON_H \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/fficonfig.h b/Modules/_ctypes/libffi_osx/include/fficonfig.h deleted file mode 100644 index 217249071dcf..000000000000 --- a/Modules/_ctypes/libffi_osx/include/fficonfig.h +++ /dev/null @@ -1,150 +0,0 @@ -/* Manually created fficonfig.h for Darwin on PowerPC or Intel - - This file is manually generated to do away with the need for autoconf and - therefore make it easier to cross-compile and build fat binaries. - - NOTE: This file was added by PyObjC. -*/ - -#ifndef MACOSX -#error "This file is only supported on Mac OS X" -#endif - -#if defined(__i386__) -# define BYTEORDER 1234 -# undef HOST_WORDS_BIG_ENDIAN -# undef WORDS_BIGENDIAN -# define SIZEOF_DOUBLE 8 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 - -#elif defined(__x86_64__) -# define BYTEORDER 1234 -# undef HOST_WORDS_BIG_ENDIAN -# undef WORDS_BIGENDIAN -# define SIZEOF_DOUBLE 8 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 - -#elif defined(__ppc__) -# define BYTEORDER 4321 -# define HOST_WORDS_BIG_ENDIAN 1 -# define WORDS_BIGENDIAN 1 -# define SIZEOF_DOUBLE 8 -# if __GNUC__ >= 4 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 -# else -# undef HAVE_LONG_DOUBLE -# define SIZEOF_LONG_DOUBLE 8 -# endif - -#elif defined(__ppc64__) -# define BYTEORDER 4321 -# define HOST_WORDS_BIG_ENDIAN 1 -# define WORDS_BIGENDIAN 1 -# define SIZEOF_DOUBLE 8 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 - -#else -#error "Unknown CPU type" -#endif - -/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP - systems. This function is required for `alloca.c' support on those systems. */ -#undef CRAY_STACKSEG_END - -/* Define to 1 if using `alloca.c'. */ -/* #undef C_ALLOCA */ - -/* Define to the flags needed for the .section .eh_frame directive. */ -#define EH_FRAME_FLAGS "aw" - -/* Define this if you want extra debugging. */ -/* #undef FFI_DEBUG */ - -/* Define this is you do not want support for the raw API. */ -#define FFI_NO_RAW_API 1 - -/* Define this if you do not want support for aggregate types. */ -/* #undef FFI_NO_STRUCTS */ - -/* Define to 1 if you have `alloca', as a function or macro. */ -#define HAVE_ALLOCA 1 - -/* Define to 1 if you have <alloca.h> and it should be used (not on Ultrix). */ -#define HAVE_ALLOCA_H 1 - -/* Define if your assembler supports .register. */ -/* #undef HAVE_AS_REGISTER_PSEUDO_OP */ - -/* Define if your assembler and linker support unaligned PC relative relocs. */ -/* #undef HAVE_AS_SPARC_UA_PCREL */ - -/* Define to 1 if you have the `memcpy' function. */ -#define HAVE_MEMCPY 1 - -/* Define if mmap with MAP_ANON(YMOUS) works. */ -#define HAVE_MMAP_ANON 1 - -/* Define if mmap of /dev/zero works. */ -/* #undef HAVE_MMAP_DEV_ZERO */ - -/* Define if read-only mmap of a plain file works. */ -#define HAVE_MMAP_FILE 1 - -/* Define if .eh_frame sections should be read-only. */ -/* #undef HAVE_RO_EH_FRAME */ - -/* Define to 1 if your C compiler doesn't accept -c and -o together. */ -/* #undef NO_MINUS_C_MINUS_O */ - -/* Name of package */ -#define PACKAGE "libffi" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "http://gcc.gnu.org/bugs.html" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "libffi" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "libffi 2.1" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "libffi" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "2.1" - -/* If using the C implementation of alloca, define if you know the - direction of stack growth for your system; otherwise it will be - automatically deduced at run-time. - STACK_DIRECTION > 0 => grows toward higher addresses - STACK_DIRECTION < 0 => grows toward lower addresses - STACK_DIRECTION = 0 => direction of growth unknown */ -/* #undef STACK_DIRECTION */ - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Define this if you are using Purify and want to suppress spurious messages. */ -/* #undef USING_PURIFY */ - -/* Version number of package */ -#define VERSION "2.1-pyobjc" - -#ifdef HAVE_HIDDEN_VISIBILITY_ATTRIBUTE -# ifdef LIBFFI_ASM -# define FFI_HIDDEN(name) .hidden name -# else -# define FFI_HIDDEN __attribute__((visibility ("hidden"))) -# endif -#else -# ifdef LIBFFI_ASM -# define FFI_HIDDEN(name) -# else -# define FFI_HIDDEN -# endif -#endif \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/ffitarget.h b/Modules/_ctypes/libffi_osx/include/ffitarget.h deleted file mode 100644 index faaa30de6f6f..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ffitarget.h +++ /dev/null @@ -1,13 +0,0 @@ -/* Dispatch to the right ffitarget file. This file is PyObjC specific; in a - normal build, the build environment copies the file to the right location or - sets up the right include flags. We want to do neither because that would - make building fat binaries harder. -*/ - -#if defined(__i386__) || defined(__x86_64__) -#include "x86-ffitarget.h" -#elif defined(__ppc__) || defined(__ppc64__) -#include "ppc-ffitarget.h" -#else -#error "Unsupported CPU type" -#endif \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h b/Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h deleted file mode 100644 index 231842199056..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h +++ /dev/null @@ -1,104 +0,0 @@ -/* -----------------------------------------------------------------*-C-*- - ppc-ffitarget.h - Copyright (c) 1996-2003 Red Hat, Inc. - Target configuration macros for PowerPC. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#ifndef LIBFFI_TARGET_H -#define LIBFFI_TARGET_H - -/* ---- System specific configurations ----------------------------------- */ - -#if (defined(POWERPC) && defined(__powerpc64__)) || \ - (defined(POWERPC_DARWIN) && defined(__ppc64__)) -#define POWERPC64 -#endif - -#ifndef LIBFFI_ASM - -typedef unsigned long ffi_arg; -typedef signed long ffi_sarg; - -typedef enum ffi_abi { - FFI_FIRST_ABI = 0, - -#ifdef POWERPC - FFI_SYSV, - FFI_GCC_SYSV, - FFI_LINUX64, -# ifdef POWERPC64 - FFI_DEFAULT_ABI = FFI_LINUX64, -# else - FFI_DEFAULT_ABI = FFI_GCC_SYSV, -# endif -#endif - -#ifdef POWERPC_AIX - FFI_AIX, - FFI_DARWIN, - FFI_DEFAULT_ABI = FFI_AIX, -#endif - -#ifdef POWERPC_DARWIN - FFI_AIX, - FFI_DARWIN, - FFI_DEFAULT_ABI = FFI_DARWIN, -#endif - -#ifdef POWERPC_FREEBSD - FFI_SYSV, - FFI_GCC_SYSV, - FFI_LINUX64, - FFI_DEFAULT_ABI = FFI_SYSV, -#endif - - FFI_LAST_ABI = FFI_DEFAULT_ABI + 1 -} ffi_abi; - -#endif // #ifndef LIBFFI_ASM - -/* ---- Definitions for closures ----------------------------------------- */ - -#define FFI_CLOSURES 1 -#define FFI_NATIVE_RAW_API 0 - -/* Needed for FFI_SYSV small structure returns. */ -#define FFI_SYSV_TYPE_SMALL_STRUCT (FFI_TYPE_LAST) - -#if defined(POWERPC64) /*|| defined(POWERPC_AIX)*/ -# define FFI_TRAMPOLINE_SIZE 48 -#elif defined(POWERPC_AIX) -# define FFI_TRAMPOLINE_SIZE 24 -#else -# define FFI_TRAMPOLINE_SIZE 40 -#endif - -#ifndef LIBFFI_ASM -# if defined(POWERPC_DARWIN) || defined(POWERPC_AIX) -typedef struct ffi_aix_trampoline_struct { - void* code_pointer; /* Pointer to ffi_closure_ASM */ - void* toc; /* TOC */ - void* static_chain; /* Pointer to closure */ -} ffi_aix_trampoline_struct; -# endif -#endif // #ifndef LIBFFI_ASM - -#endif // #ifndef LIBFFI_TARGET_H \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h b/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h deleted file mode 100644 index 55c2b6c50cd9..000000000000 --- a/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h +++ /dev/null @@ -1,88 +0,0 @@ -/* -----------------------------------------------------------------*-C-*- - x86-ffitarget.h - Copyright (c) 1996-2003 Red Hat, Inc. - Target configuration macros for x86 and x86-64. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - - ----------------------------------------------------------------------- */ - -#ifndef LIBFFI_TARGET_H -#define LIBFFI_TARGET_H - -/* ---- System specific configurations ----------------------------------- */ - -#if defined(X86_64) && defined(__i386__) -# undef X86_64 -# define X86 -#endif - -#if defined(__x86_64__) -# ifndef X86_64 -# define X86_64 -# endif -#endif - -/* ---- Generic type definitions ----------------------------------------- */ - -#ifndef LIBFFI_ASM - -typedef unsigned long ffi_arg; -typedef signed long ffi_sarg; - -typedef enum ffi_abi { - FFI_FIRST_ABI = 0, - - /* ---- Intel x86 Win32 ---------- */ -#ifdef X86_WIN32 - FFI_SYSV, - FFI_STDCALL, - /* TODO: Add fastcall support for the sake of completeness */ - FFI_DEFAULT_ABI = FFI_SYSV, -#endif - - /* ---- Intel x86 and AMD x86-64 - */ -#if !defined(X86_WIN32) && (defined(__i386__) || defined(__x86_64__)) - FFI_SYSV, - FFI_UNIX64, /* Unix variants all use the same ABI for x86-64 */ -# ifdef __i386__ - FFI_DEFAULT_ABI = FFI_SYSV, -# else - FFI_DEFAULT_ABI = FFI_UNIX64, -# endif -#endif - - FFI_LAST_ABI = FFI_DEFAULT_ABI + 1 -} ffi_abi; - -#endif // #ifndef LIBFFI_ASM - -/* ---- Definitions for closures ----------------------------------------- */ - -#define FFI_CLOSURES 1 - -#if defined(X86_64) || (defined(__x86_64__) && defined(X86_DARWIN)) -# define FFI_TRAMPOLINE_SIZE 24 -# define FFI_NATIVE_RAW_API 0 -#else -# define FFI_TRAMPOLINE_SIZE 10 -# define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ -#endif - -#endif // #ifndef LIBFFI_TARGET_H \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S b/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S deleted file mode 100644 index f143dbd28c3d..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S +++ /dev/null @@ -1,365 +0,0 @@ -#if defined(__ppc__) || defined(__ppc64__) - -/* ----------------------------------------------------------------------- - ppc-darwin.S - Copyright (c) 2000 John Hornkvist - Copyright (c) 2004 Free Software Foundation, Inc. - - PowerPC Assembly glue. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define LIBFFI_ASM - -#include <fficonfig.h> -#include <ffi.h> -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - -.text - .align 2 -.globl _ffi_prep_args - -.text - .align 2 -.globl _ffi_call_DARWIN - -.text - .align 2 -_ffi_call_DARWIN: -LFB0: - mr r12,r8 /* We only need r12 until the call, - so it doesn't have to be saved. */ - -LFB1: - /* Save the old stack pointer as AP. */ - mr r8,r1 - -LCFI0: -#if defined(__ppc64__) - /* Allocate the stack space we need. - r4 (size of input data) - 48 bytes (linkage area) - 40 bytes (saved registers) - 8 bytes (extra FPR) - r4 + 96 bytes total - */ - - addi r4,r4,-96 // Add our overhead. - li r0,-32 // Align to 32 bytes. - and r4,r4,r0 -#endif - stgux r1,r1,r4 // Grow the stack. - mflr r9 - - /* Save registers we use. */ -#if defined(__ppc64__) - std r27,-40(r8) -#endif - stg r28,MODE_CHOICE(-16,-32)(r8) - stg r29,MODE_CHOICE(-12,-24)(r8) - stg r30,MODE_CHOICE(-8,-16)(r8) - stg r31,MODE_CHOICE(-4,-8)(r8) - stg r9,SF_RETURN(r8) /* return address */ -#if !defined(POWERPC_DARWIN) /* TOC unused in OS X */ - stg r2,MODE_CHOICE(20,40)(r1) -#endif - -LCFI1: -#if defined(__ppc64__) - mr r27,r3 // our extended_cif -#endif - /* Save arguments over call. */ - mr r31,r5 /* flags, */ - mr r30,r6 /* rvalue, */ - mr r29,r7 /* function address, */ - mr r28,r8 /* our AP. */ - -LCFI2: - /* Call ffi_prep_args. */ - mr r4,r1 - li r9,0 - mtctr r12 /* r12 holds address of _ffi_prep_args. */ - bctrl -#if !defined(POWERPC_DARWIN) /* TOC unused in OS X */ - lg r2,MODE_CHOICE(20,40)(r1) -#endif - - /* Now do the call. - Set up cr1 with bits 4-7 of the flags. */ - mtcrf 0x40,r31 - - /* Load all those argument registers. - We have set up a nice stack frame, just load it into registers. */ - lg r3,SF_ARG1(r1) - lg r4,SF_ARG2(r1) - lg r5,SF_ARG3(r1) - lg r6,SF_ARG4(r1) - nop - lg r7,SF_ARG5(r1) - lg r8,SF_ARG6(r1) - lg r9,SF_ARG7(r1) - lg r10,SF_ARG8(r1) - - /* Load all the FP registers. */ - bf 6,L2 /* No floats to load. */ -#if defined(__ppc64__) - lfd f1,MODE_CHOICE(-16,-40)-(14*8)(r28) - lfd f2,MODE_CHOICE(-16,-40)-(13*8)(r28) - lfd f3,MODE_CHOICE(-16,-40)-(12*8)(r28) - lfd f4,MODE_CHOICE(-16,-40)-(11*8)(r28) - nop - lfd f5,MODE_CHOICE(-16,-40)-(10*8)(r28) - lfd f6,MODE_CHOICE(-16,-40)-(9*8)(r28) - lfd f7,MODE_CHOICE(-16,-40)-(8*8)(r28) - lfd f8,MODE_CHOICE(-16,-40)-(7*8)(r28) - nop - lfd f9,MODE_CHOICE(-16,-40)-(6*8)(r28) - lfd f10,MODE_CHOICE(-16,-40)-(5*8)(r28) - lfd f11,MODE_CHOICE(-16,-40)-(4*8)(r28) - lfd f12,MODE_CHOICE(-16,-40)-(3*8)(r28) - nop - lfd f13,MODE_CHOICE(-16,-40)-(2*8)(r28) - lfd f14,MODE_CHOICE(-16,-40)-(1*8)(r28) -#elif defined(__ppc__) - lfd f1,MODE_CHOICE(-16,-40)-(13*8)(r28) - lfd f2,MODE_CHOICE(-16,-40)-(12*8)(r28) - lfd f3,MODE_CHOICE(-16,-40)-(11*8)(r28) - lfd f4,MODE_CHOICE(-16,-40)-(10*8)(r28) - nop - lfd f5,MODE_CHOICE(-16,-40)-(9*8)(r28) - lfd f6,MODE_CHOICE(-16,-40)-(8*8)(r28) - lfd f7,MODE_CHOICE(-16,-40)-(7*8)(r28) - lfd f8,MODE_CHOICE(-16,-40)-(6*8)(r28) - nop - lfd f9,MODE_CHOICE(-16,-40)-(5*8)(r28) - lfd f10,MODE_CHOICE(-16,-40)-(4*8)(r28) - lfd f11,MODE_CHOICE(-16,-40)-(3*8)(r28) - lfd f12,MODE_CHOICE(-16,-40)-(2*8)(r28) - nop - lfd f13,MODE_CHOICE(-16,-40)-(1*8)(r28) -#else -#error undefined architecture -#endif - -L2: - mr r12,r29 // Put the target address in r12 as specified. - mtctr r12 // Get the address to call into CTR. - nop - nop - bctrl // Make the call. - - // Deal with the return value. -#if defined(__ppc64__) - mtcrf 0x3,r31 // flags in cr6 and cr7 - bt 27,L(st_return_value) -#elif defined(__ppc__) - mtcrf 0x1,r31 // flags in cr7 -#else -#error undefined architecture -#endif - - bt 30,L(done_return_value) - bt 29,L(fp_return_value) - stg r3,0(r30) -#if defined(__ppc__) - bf 28,L(done_return_value) // Store the second long if necessary. - stg r4,4(r30) -#endif - // Fall through - -L(done_return_value): - lg r1,0(r1) // Restore stack pointer. - // Restore the registers we used. - lg r9,SF_RETURN(r1) // return address - lg r31,MODE_CHOICE(-4,-8)(r1) - mtlr r9 - lg r30,MODE_CHOICE(-8,-16)(r1) - lg r29,MODE_CHOICE(-12,-24)(r1) - lg r28,MODE_CHOICE(-16,-32)(r1) -#if defined(__ppc64__) - ld r27,-40(r1) -#endif - blr - -#if defined(__ppc64__) -L(st_return_value): - // Grow the stack enough to fit the registers. Leave room for 8 args - // to trample the 1st 8 slots in param area. - stgu r1,-SF_ROUND(280)(r1) // 64 + 104 + 48 + 64 - - // Store GPRs - std r3,SF_ARG9(r1) - std r4,SF_ARG10(r1) - std r5,SF_ARG11(r1) - std r6,SF_ARG12(r1) - nop - std r7,SF_ARG13(r1) - std r8,SF_ARG14(r1) - std r9,SF_ARG15(r1) - std r10,SF_ARG16(r1) - - // Store FPRs - nop - bf 26,L(call_struct_to_ram_form) - stfd f1,SF_ARG17(r1) - stfd f2,SF_ARG18(r1) - stfd f3,SF_ARG19(r1) - stfd f4,SF_ARG20(r1) - nop - stfd f5,SF_ARG21(r1) - stfd f6,SF_ARG22(r1) - stfd f7,SF_ARG23(r1) - stfd f8,SF_ARG24(r1) - nop - stfd f9,SF_ARG25(r1) - stfd f10,SF_ARG26(r1) - stfd f11,SF_ARG27(r1) - stfd f12,SF_ARG28(r1) - nop - stfd f13,SF_ARG29(r1) - -L(call_struct_to_ram_form): - ld r3,0(r27) // extended_cif->cif* - ld r3,16(r3) // ffi_cif->rtype* - addi r4,r1,SF_ARG9 // stored GPRs - addi r6,r1,SF_ARG17 // stored FPRs - li r5,0 // GPR size ptr (NULL) - li r7,0 // FPR size ptr (NULL) - li r8,0 // FPR count ptr (NULL) - li r10,0 // struct offset (NULL) - mr r9,r30 // return area - bl Lffi64_struct_to_ram_form$stub - lg r1,0(r1) // Restore stack pointer. - b L(done_return_value) -#endif - -L(fp_return_value): - /* Do we have long double to store? */ - bf 31,L(fd_return_value) - stfd f1,0(r30) - stfd f2,8(r30) - b L(done_return_value) - -L(fd_return_value): - /* Do we have double to store? */ - bf 28,L(float_return_value) - stfd f1,0(r30) - b L(done_return_value) - -L(float_return_value): - /* We only have a float to store. */ - stfs f1,0(r30) - b L(done_return_value) - -LFE1: -/* END(_ffi_call_DARWIN) */ - -/* Provide a null definition of _ffi_call_AIX. */ -.text - .align 2 -.globl _ffi_call_AIX -.text - .align 2 -_ffi_call_AIX: - blr -/* END(_ffi_call_AIX) */ - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 ; Length of Common Information Entry -LSCIE1: - .long 0x0 ; CIE Identifier Tag - .byte 0x1 ; CIE Version - .ascii "zR\0" ; CIE Augmentation - .byte 0x1 ; uleb128 0x1; CIE Code Alignment Factor - .byte 0x7c ; sleb128 -4; CIE Data Alignment Factor - .byte 0x41 ; CIE RA Column - .byte 0x1 ; uleb128 0x1; Augmentation size - .byte 0x10 ; FDE Encoding (pcrel) - .byte 0xc ; DW_CFA_def_cfa - .byte 0x1 ; uleb128 0x1 - .byte 0x0 ; uleb128 0x0 - .align LOG2_GPR_BYTES -LECIE1: -.globl _ffi_call_DARWIN.eh -_ffi_call_DARWIN.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 ; FDE Length - -LASFDE1: - .long LASFDE1-EH_frame1 ; FDE CIE offset - .g_long LFB0-. ; FDE initial location - .set L$set$3,LFE1-LFB0 - .g_long L$set$3 ; FDE address range - .byte 0x0 ; uleb128 0x0; Augmentation size - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$4,LCFI0-LFB1 - .long L$set$4 - .byte 0xd ; DW_CFA_def_cfa_register - .byte 0x08 ; uleb128 0x08 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$5,LCFI1-LCFI0 - .long L$set$5 - .byte 0x11 ; DW_CFA_offset_extended_sf - .byte 0x41 ; uleb128 0x41 - .byte 0x7e ; sleb128 -2 - .byte 0x9f ; DW_CFA_offset, column 0x1f - .byte 0x1 ; uleb128 0x1 - .byte 0x9e ; DW_CFA_offset, column 0x1e - .byte 0x2 ; uleb128 0x2 - .byte 0x9d ; DW_CFA_offset, column 0x1d - .byte 0x3 ; uleb128 0x3 - .byte 0x9c ; DW_CFA_offset, column 0x1c - .byte 0x4 ; uleb128 0x4 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$6,LCFI2-LCFI1 - .long L$set$6 - .byte 0xd ; DW_CFA_def_cfa_register - .byte 0x1c ; uleb128 0x1c - .align LOG2_GPR_BYTES -LEFDE1: - -#if defined(__ppc64__) -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi64_struct_to_ram_form$stub: - .indirect_symbol _ffi64_struct_to_ram_form - mflr r0 - bcl 20,31,LO$ffi64_struct_to_ram_form - -LO$ffi64_struct_to_ram_form: - mflr r11 - addis r11,r11,ha16(L_ffi64_struct_to_ram_form$lazy_ptr - LO$ffi64_struct_to_ram_form) - mtlr r0 - lgu r12,lo16(L_ffi64_struct_to_ram_form$lazy_ptr - LO$ffi64_struct_to_ram_form)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi64_struct_to_ram_form$lazy_ptr: - .indirect_symbol _ffi64_struct_to_ram_form - .g_long dyld_stub_binding_helper - -#endif // __ppc64__ -#endif // __ppc__ || __ppc64__ diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h b/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h deleted file mode 100644 index cf4bd50f93c3..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h +++ /dev/null @@ -1,85 +0,0 @@ -/* ----------------------------------------------------------------------- - ppc-darwin.h - Copyright (c) 2002, 2003, 2004, Free Software Foundation, - Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define L(x) x - -#define SF_ARG9 MODE_CHOICE(56,112) -#define SF_ARG10 MODE_CHOICE(60,120) -#define SF_ARG11 MODE_CHOICE(64,128) -#define SF_ARG12 MODE_CHOICE(68,136) -#define SF_ARG13 MODE_CHOICE(72,144) -#define SF_ARG14 MODE_CHOICE(76,152) -#define SF_ARG15 MODE_CHOICE(80,160) -#define SF_ARG16 MODE_CHOICE(84,168) -#define SF_ARG17 MODE_CHOICE(88,176) -#define SF_ARG18 MODE_CHOICE(92,184) -#define SF_ARG19 MODE_CHOICE(96,192) -#define SF_ARG20 MODE_CHOICE(100,200) -#define SF_ARG21 MODE_CHOICE(104,208) -#define SF_ARG22 MODE_CHOICE(108,216) -#define SF_ARG23 MODE_CHOICE(112,224) -#define SF_ARG24 MODE_CHOICE(116,232) -#define SF_ARG25 MODE_CHOICE(120,240) -#define SF_ARG26 MODE_CHOICE(124,248) -#define SF_ARG27 MODE_CHOICE(128,256) -#define SF_ARG28 MODE_CHOICE(132,264) -#define SF_ARG29 MODE_CHOICE(136,272) - -#define ASM_NEEDS_REGISTERS 4 -#define NUM_GPR_ARG_REGISTERS 8 -#define NUM_FPR_ARG_REGISTERS 13 - -#define FFI_TYPE_1_BYTE(x) ((x) == FFI_TYPE_UINT8 || (x) == FFI_TYPE_SINT8) -#define FFI_TYPE_2_BYTE(x) ((x) == FFI_TYPE_UINT16 || (x) == FFI_TYPE_SINT16) -#define FFI_TYPE_4_BYTE(x) \ - ((x) == FFI_TYPE_UINT32 || (x) == FFI_TYPE_SINT32 ||\ - (x) == FFI_TYPE_INT || (x) == FFI_TYPE_FLOAT) - -#if !defined(LIBFFI_ASM) - -enum { - FLAG_RETURNS_NOTHING = 1 << (31 - 30), // cr7 - FLAG_RETURNS_FP = 1 << (31 - 29), - FLAG_RETURNS_64BITS = 1 << (31 - 28), - FLAG_RETURNS_128BITS = 1 << (31 - 31), - - FLAG_RETURNS_STRUCT = 1 << (31 - 27), // cr6 - FLAG_STRUCT_CONTAINS_FP = 1 << (31 - 26), - - FLAG_ARG_NEEDS_COPY = 1 << (31 - 7), - FLAG_FP_ARGUMENTS = 1 << (31 - 6), // cr1.eq; specified by ABI - FLAG_4_GPR_ARGUMENTS = 1 << (31 - 5), - FLAG_RETVAL_REFERENCE = 1 << (31 - 4) -}; - -#if defined(__ppc64__) -void ffi64_struct_to_ram_form(const ffi_type*, const char*, unsigned int*, - const char*, unsigned int*, unsigned int*, char*, unsigned int*); -void ffi64_struct_to_reg_form(const ffi_type*, const char*, unsigned int*, - unsigned int*, char*, unsigned int*, char*, unsigned int*); -bool ffi64_stret_needs_ptr(const ffi_type* inType, - unsigned short*, unsigned short*); -#endif - -#endif // !defined(LIBFFI_ASM) \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S b/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S deleted file mode 100644 index c3d30c25254c..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S +++ /dev/null @@ -1,308 +0,0 @@ -#if defined(__ppc__) - -/* ----------------------------------------------------------------------- - ppc-darwin_closure.S - Copyright (c) 2002, 2003, 2004, Free Software Foundation, - Inc. based on ppc_closure.S - - PowerPC Assembly glue. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define LIBFFI_ASM - -#include <ffi.h> -#include <ppc-ffitarget.h> // for FFI_TRAMPOLINE_SIZE -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - - .file "ppc-darwin_closure.S" -.text - .align LOG2_GPR_BYTES - .globl _ffi_closure_ASM - -.text - .align LOG2_GPR_BYTES - -_ffi_closure_ASM: -LFB1: - mflr r0 // Save return address - stg r0,SF_RETURN(r1) - -LCFI0: - /* 24/48 bytes (Linkage Area) - 32/64 bytes (outgoing parameter area, always reserved) - 104 bytes (13*8 from FPR) - 16/32 bytes (result) - 176/232 total bytes */ - - /* skip over caller save area and keep stack aligned to 16/32. */ - stgu r1,-SF_ROUND(176)(r1) - -LCFI1: - /* We want to build up an area for the parameters passed - in registers. (both floating point and integer) */ - - /* 176/256 bytes (callee stack frame aligned to 16/32) - 24/48 bytes (caller linkage area) - 200/304 (start of caller parameter area aligned to 4/8) - */ - - /* Save GPRs 3 - 10 (aligned to 4/8) - in the parents outgoing area. */ - stg r3,200(r1) - stg r4,204(r1) - stg r5,208(r1) - stg r6,212(r1) - stg r7,216(r1) - stg r8,220(r1) - stg r9,224(r1) - stg r10,228(r1) - - /* Save FPRs 1 - 13. (aligned to 8) */ - stfd f1,56(r1) - stfd f2,64(r1) - stfd f3,72(r1) - stfd f4,80(r1) - stfd f5,88(r1) - stfd f6,96(r1) - stfd f7,104(r1) - stfd f8,112(r1) - stfd f9,120(r1) - stfd f10,128(r1) - stfd f11,136(r1) - stfd f12,144(r1) - stfd f13,152(r1) - - // Set up registers for the routine that actually does the work. - mr r3,r11 // context pointer from the trampoline - addi r4,r1,160 // result storage - addi r5,r1,200 // saved GPRs - addi r6,r1,56 // saved FPRs - bl Lffi_closure_helper_DARWIN$stub - - /* Now r3 contains the return type. Use it to look up in a table - so we know how to deal with each type. */ - addi r5,r1,160 // Copy result storage pointer. - bl Lget_ret_type0_addr // Get pointer to Lret_type0 into LR. - mflr r4 // Move to r4. - slwi r3,r3,4 // Multiply return type by 16. - add r3,r3,r4 // Add contents of table to table address. - mtctr r3 - bctr - -LFE1: -/* Each of the ret_typeX code fragments has to be exactly 16 bytes long - (4 instructions). For cache effectiveness we align to a 16 byte boundary - first. */ - .align 4 - nop - nop - nop - -Lget_ret_type0_addr: - blrl - -/* case FFI_TYPE_VOID */ -Lret_type0: - b Lfinish - nop - nop - nop - -/* case FFI_TYPE_INT */ -Lret_type1: - lwz r3,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_FLOAT */ -Lret_type2: - lfs f1,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_DOUBLE */ -Lret_type3: - lfd f1,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_LONGDOUBLE */ -Lret_type4: - lfd f1,0(r5) - lfd f2,8(r5) - b Lfinish - nop - -/* case FFI_TYPE_UINT8 */ -Lret_type5: - lbz r3,3(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_SINT8 */ -Lret_type6: - lbz r3,3(r5) - extsb r3,r3 - b Lfinish - nop - -/* case FFI_TYPE_UINT16 */ -Lret_type7: - lhz r3,2(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_SINT16 */ -Lret_type8: - lha r3,2(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_UINT32 */ -Lret_type9: // same as Lret_type1 - lwz r3,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_SINT32 */ -Lret_type10: // same as Lret_type1 - lwz r3,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_UINT64 */ -Lret_type11: - lwz r3,0(r5) - lwz r4,4(r5) - b Lfinish - nop - -/* case FFI_TYPE_SINT64 */ -Lret_type12: // same as Lret_type11 - lwz r3,0(r5) - lwz r4,4(r5) - b Lfinish - nop - -/* case FFI_TYPE_STRUCT */ -Lret_type13: - b Lfinish - nop - nop - nop - -/* End 16-byte aligned cases */ -/* case FFI_TYPE_POINTER */ -// This case assumes that FFI_TYPE_POINTER == FFI_TYPE_LAST. If more types -// are added in future, the following code will need to be updated and -// padded to 16 bytes. -Lret_type14: - lg r3,0(r5) - // fall through - -/* case done */ -Lfinish: - addi r1,r1,SF_ROUND(176) // Restore stack pointer. - lg r0,SF_RETURN(r1) // Restore return address. - mtlr r0 // Restore link register. - blr - -/* END(ffi_closure_ASM) */ - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 ; Length of Common Information Entry -LSCIE1: - .long 0x0 ; CIE Identifier Tag - .byte 0x1 ; CIE Version - .ascii "zR\0" ; CIE Augmentation - .byte 0x1 ; uleb128 0x1; CIE Code Alignment Factor - .byte 0x7c ; sleb128 -4; CIE Data Alignment Factor - .byte 0x41 ; CIE RA Column - .byte 0x1 ; uleb128 0x1; Augmentation size - .byte 0x10 ; FDE Encoding (pcrel) - .byte 0xc ; DW_CFA_def_cfa - .byte 0x1 ; uleb128 0x1 - .byte 0x0 ; uleb128 0x0 - .align LOG2_GPR_BYTES -LECIE1: -.globl _ffi_closure_ASM.eh -_ffi_closure_ASM.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 ; FDE Length - -LASFDE1: - .long LASFDE1-EH_frame1 ; FDE CIE offset - .g_long LFB1-. ; FDE initial location - .set L$set$3,LFE1-LFB1 - .g_long L$set$3 ; FDE address range - .byte 0x0 ; uleb128 0x0; Augmentation size - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$3,LCFI1-LCFI0 - .long L$set$3 - .byte 0xe ; DW_CFA_def_cfa_offset - .byte 176,1 ; uleb128 176 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$4,LCFI0-LFB1 - .long L$set$4 - .byte 0x11 ; DW_CFA_offset_extended_sf - .byte 0x41 ; uleb128 0x41 - .byte 0x7e ; sleb128 -2 - .align LOG2_GPR_BYTES - -LEFDE1: -.data - .align LOG2_GPR_BYTES -LDFCM0: -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi_closure_helper_DARWIN$stub: - .indirect_symbol _ffi_closure_helper_DARWIN - mflr r0 - bcl 20,31,LO$ffi_closure_helper_DARWIN - -LO$ffi_closure_helper_DARWIN: - mflr r11 - addis r11,r11,ha16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN) - mtlr r0 - lgu r12,lo16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi_closure_helper_DARWIN$lazy_ptr: - .indirect_symbol _ffi_closure_helper_DARWIN - .g_long dyld_stub_binding_helper - - -#endif // __ppc__ diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c deleted file mode 100644 index 8953d5fda358..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c +++ /dev/null @@ -1,1776 +0,0 @@ -#if defined(__ppc__) || defined(__ppc64__) - -/* ----------------------------------------------------------------------- - ffi.c - Copyright (c) 1998 Geoffrey Keating - - PowerPC Foreign Function Interface - - Darwin ABI support (c) 2001 John Hornkvist - AIX ABI support (c) 2002 Free Software Foundation, Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdbool.h> -#include <stdio.h> -#include <stdlib.h> -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - -#if 0 -#if defined(POWERPC_DARWIN) -#include <libkern/OSCacheControl.h> // for sys_icache_invalidate() -#endif - -#else - -#pragma weak sys_icache_invalidate -extern void sys_icache_invalidate(void *start, size_t len); - -#endif - - -extern void ffi_closure_ASM(void); - -// The layout of a function descriptor. A C function pointer really -// points to one of these. -typedef struct aix_fd_struct { - void* code_pointer; - void* toc; -} aix_fd; - -/* ffi_prep_args is called by the assembly routine once stack space - has been allocated for the function's arguments. - - The stack layout we want looks like this: - - | Return address from ffi_call_DARWIN | higher addresses - |--------------------------------------------| - | Previous backchain pointer 4/8 | stack pointer here - |--------------------------------------------|-\ <<< on entry to - | Saved r28-r31 (4/8)*4 | | ffi_call_DARWIN - |--------------------------------------------| | - | Parameters (at least 8*(4/8)=32/64) | | (176) +112 - +288 - |--------------------------------------------| | - | Space for GPR2 4/8 | | - |--------------------------------------------| | stack | - | Reserved (4/8)*2 | | grows | - |--------------------------------------------| | down V - | Space for callee's LR 4/8 | | - |--------------------------------------------| | lower addresses - | Saved CR 4/8 | | - |--------------------------------------------| | stack pointer here - | Current backchain pointer 4/8 | | during - |--------------------------------------------|-/ <<< ffi_call_DARWIN - - Note: ppc64 CR is saved in the low word of a long on the stack. -*/ - -/*@-exportheader@*/ -void -ffi_prep_args( - extended_cif* inEcif, - unsigned *const stack) -/*@=exportheader@*/ -{ - /* Copy the ecif to a local var so we can trample the arg. - BC note: test this with GP later for possible problems... */ - volatile extended_cif* ecif = inEcif; - - const unsigned bytes = ecif->cif->bytes; - const unsigned flags = ecif->cif->flags; - - /* Cast the stack arg from int* to long*. sizeof(long) == 4 in 32-bit mode - and 8 in 64-bit mode. */ - unsigned long *const longStack = (unsigned long *const)stack; - - /* 'stacktop' points at the previous backchain pointer. */ -#if defined(__ppc64__) - // In ppc-darwin.s, an extra 96 bytes is reserved for the linkage area, - // saved registers, and an extra FPR. - unsigned long *const stacktop = - (unsigned long *)(unsigned long)((char*)longStack + bytes + 96); -#elif defined(__ppc__) - unsigned long *const stacktop = longStack + (bytes / sizeof(long)); -#else -#error undefined architecture -#endif - - /* 'fpr_base' points at the space for fpr1, and grows upwards as - we use FPR registers. */ - double* fpr_base = (double*)(stacktop - ASM_NEEDS_REGISTERS) - - NUM_FPR_ARG_REGISTERS; - -#if defined(__ppc64__) - // 64-bit saves an extra register, and uses an extra FPR. Knock fpr_base - // down a couple pegs. - fpr_base -= 2; -#endif - - unsigned int fparg_count = 0; - - /* 'next_arg' grows up as we put parameters in it. */ - unsigned long* next_arg = longStack + 6; /* 6 reserved positions. */ - - int i; - double double_tmp; - void** p_argv = ecif->avalue; - unsigned long gprvalue; - ffi_type** ptr = ecif->cif->arg_types; - - /* Check that everything starts aligned properly. */ - FFI_ASSERT(stack == SF_ROUND(stack)); - FFI_ASSERT(stacktop == SF_ROUND(stacktop)); - FFI_ASSERT(bytes == SF_ROUND(bytes)); - - /* Deal with return values that are actually pass-by-reference. - Rule: - Return values are referenced by r3, so r4 is the first parameter. */ - - if (flags & FLAG_RETVAL_REFERENCE) - *next_arg++ = (unsigned long)(char*)ecif->rvalue; - - /* Now for the arguments. */ - for (i = ecif->cif->nargs; i > 0; i--, ptr++, p_argv++) - { - switch ((*ptr)->type) - { - /* If a floating-point parameter appears before all of the general- - purpose registers are filled, the corresponding GPRs that match - the size of the floating-point parameter are shadowed for the - benefit of vararg and pre-ANSI functions. */ - case FFI_TYPE_FLOAT: - double_tmp = *(float*)*p_argv; - - if (fparg_count < NUM_FPR_ARG_REGISTERS) - *fpr_base++ = double_tmp; - - *(double*)next_arg = double_tmp; - - next_arg++; - fparg_count++; - FFI_ASSERT(flags & FLAG_FP_ARGUMENTS); - - break; - - case FFI_TYPE_DOUBLE: - double_tmp = *(double*)*p_argv; - - if (fparg_count < NUM_FPR_ARG_REGISTERS) - *fpr_base++ = double_tmp; - - *(double*)next_arg = double_tmp; - - next_arg += MODE_CHOICE(2,1); - fparg_count++; - FFI_ASSERT(flags & FLAG_FP_ARGUMENTS); - - break; - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - case FFI_TYPE_LONGDOUBLE: -#if defined(__ppc64__) - if (fparg_count < NUM_FPR_ARG_REGISTERS) - *(long double*)fpr_base = *(long double*)*p_argv; -#elif defined(__ppc__) - if (fparg_count < NUM_FPR_ARG_REGISTERS - 1) - *(long double*)fpr_base = *(long double*)*p_argv; - else if (fparg_count == NUM_FPR_ARG_REGISTERS - 1) - *(double*)fpr_base = *(double*)*p_argv; -#else -#error undefined architecture -#endif - - *(long double*)next_arg = *(long double*)*p_argv; - fparg_count += 2; - fpr_base += 2; - next_arg += MODE_CHOICE(4,2); - FFI_ASSERT(flags & FLAG_FP_ARGUMENTS); - - break; -#endif // FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: -#if defined(__ppc64__) - gprvalue = *(long long*)*p_argv; - goto putgpr; -#elif defined(__ppc__) - *(long long*)next_arg = *(long long*)*p_argv; - next_arg += 2; - break; -#else -#error undefined architecture -#endif - - case FFI_TYPE_POINTER: - gprvalue = *(unsigned long*)*p_argv; - goto putgpr; - - case FFI_TYPE_UINT8: - gprvalue = *(unsigned char*)*p_argv; - goto putgpr; - - case FFI_TYPE_SINT8: - gprvalue = *(signed char*)*p_argv; - goto putgpr; - - case FFI_TYPE_UINT16: - gprvalue = *(unsigned short*)*p_argv; - goto putgpr; - - case FFI_TYPE_SINT16: - gprvalue = *(signed short*)*p_argv; - goto putgpr; - - case FFI_TYPE_STRUCT: - { -#if defined(__ppc64__) - unsigned int gprSize = 0; - unsigned int fprSize = 0; - - ffi64_struct_to_reg_form(*ptr, (char*)*p_argv, NULL, &fparg_count, - (char*)next_arg, &gprSize, (char*)fpr_base, &fprSize); - next_arg += gprSize / sizeof(long); - fpr_base += fprSize / sizeof(double); - -#elif defined(__ppc__) - char* dest_cpy = (char*)next_arg; - - /* Structures that match the basic modes (QI 1 byte, HI 2 bytes, - SI 4 bytes) are aligned as if they were those modes. - Structures with 3 byte in size are padded upwards. */ - unsigned size_al = (*ptr)->size; - - /* If the first member of the struct is a double, then align - the struct to double-word. */ - if ((*ptr)->elements[0]->type == FFI_TYPE_DOUBLE) - size_al = ALIGN((*ptr)->size, 8); - - if (ecif->cif->abi == FFI_DARWIN) - { - if (size_al < 3) - dest_cpy += 4 - size_al; - } - - memcpy((char*)dest_cpy, (char*)*p_argv, size_al); - next_arg += (size_al + 3) / 4; -#else -#error undefined architecture -#endif - break; - } - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - gprvalue = *(unsigned*)*p_argv; - -putgpr: - *next_arg++ = gprvalue; - break; - - default: - break; - } - } - - /* Check that we didn't overrun the stack... */ - //FFI_ASSERT(gpr_base <= stacktop - ASM_NEEDS_REGISTERS); - //FFI_ASSERT((unsigned *)fpr_base - // <= stacktop - ASM_NEEDS_REGISTERS - NUM_GPR_ARG_REGISTERS); - //FFI_ASSERT(flags & FLAG_4_GPR_ARGUMENTS || intarg_count <= 4); -} - -#if defined(__ppc64__) - -bool -ffi64_struct_contains_fp( - const ffi_type* inType) -{ - bool containsFP = false; - unsigned int i; - - for (i = 0; inType->elements[i] != NULL && !containsFP; i++) - { - if (inType->elements[i]->type == FFI_TYPE_FLOAT || - inType->elements[i]->type == FFI_TYPE_DOUBLE || - inType->elements[i]->type == FFI_TYPE_LONGDOUBLE) - containsFP = true; - else if (inType->elements[i]->type == FFI_TYPE_STRUCT) - containsFP = ffi64_struct_contains_fp(inType->elements[i]); - } - - return containsFP; -} - -#endif // defined(__ppc64__) - -/* Perform machine dependent cif processing. */ -ffi_status -ffi_prep_cif_machdep( - ffi_cif* cif) -{ - /* All this is for the DARWIN ABI. */ - int i; - ffi_type** ptr; - int intarg_count = 0; - int fparg_count = 0; - unsigned int flags = 0; - unsigned int size_al = 0; - - /* All the machine-independent calculation of cif->bytes will be wrong. - Redo the calculation for DARWIN. */ - - /* Space for the frame pointer, callee's LR, CR, etc, and for - the asm's temp regs. */ - unsigned int bytes = (6 + ASM_NEEDS_REGISTERS) * sizeof(long); - - /* Return value handling. The rules are as follows: - - 32-bit (or less) integer values are returned in gpr3; - - Structures of size <= 4 bytes also returned in gpr3; - - 64-bit integer values and structures between 5 and 8 bytes are - returned in gpr3 and gpr4; - - Single/double FP values are returned in fpr1; - - Long double FP (if not equivalent to double) values are returned in - fpr1 and fpr2; - - Larger structures values are allocated space and a pointer is passed - as the first argument. */ - switch (cif->rtype->type) - { -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - case FFI_TYPE_LONGDOUBLE: - flags |= FLAG_RETURNS_128BITS; - flags |= FLAG_RETURNS_FP; - break; -#endif // FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_DOUBLE: - flags |= FLAG_RETURNS_64BITS; - /* Fall through. */ - case FFI_TYPE_FLOAT: - flags |= FLAG_RETURNS_FP; - break; - -#if defined(__ppc64__) - case FFI_TYPE_POINTER: -#endif - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - flags |= FLAG_RETURNS_64BITS; - break; - - case FFI_TYPE_STRUCT: - { -#if defined(__ppc64__) - - if (ffi64_stret_needs_ptr(cif->rtype, NULL, NULL)) - { - flags |= FLAG_RETVAL_REFERENCE; - flags |= FLAG_RETURNS_NOTHING; - intarg_count++; - } - else - { - flags |= FLAG_RETURNS_STRUCT; - - if (ffi64_struct_contains_fp(cif->rtype)) - flags |= FLAG_STRUCT_CONTAINS_FP; - } - -#elif defined(__ppc__) - - flags |= FLAG_RETVAL_REFERENCE; - flags |= FLAG_RETURNS_NOTHING; - intarg_count++; - -#else -#error undefined architecture -#endif - break; - } - - case FFI_TYPE_VOID: - flags |= FLAG_RETURNS_NOTHING; - break; - - default: - /* Returns 32-bit integer, or similar. Nothing to do here. */ - break; - } - - /* The first NUM_GPR_ARG_REGISTERS words of integer arguments, and the - first NUM_FPR_ARG_REGISTERS fp arguments, go in registers; the rest - goes on the stack. Structures are passed as a pointer to a copy of - the structure. Stuff on the stack needs to keep proper alignment. */ - for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++) - { - switch ((*ptr)->type) - { - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - fparg_count++; - /* If this FP arg is going on the stack, it must be - 8-byte-aligned. */ - if (fparg_count > NUM_FPR_ARG_REGISTERS - && intarg_count % 2 != 0) - intarg_count++; - break; - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - case FFI_TYPE_LONGDOUBLE: - fparg_count += 2; - /* If this FP arg is going on the stack, it must be - 8-byte-aligned. */ - - if ( -#if defined(__ppc64__) - fparg_count > NUM_FPR_ARG_REGISTERS + 1 -#elif defined(__ppc__) - fparg_count > NUM_FPR_ARG_REGISTERS -#else -#error undefined architecture -#endif - && intarg_count % 2 != 0) - intarg_count++; - - intarg_count += 2; - break; -#endif // FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - /* 'long long' arguments are passed as two words, but - either both words must fit in registers or both go - on the stack. If they go on the stack, they must - be 8-byte-aligned. */ - if (intarg_count == NUM_GPR_ARG_REGISTERS - 1 - || (intarg_count >= NUM_GPR_ARG_REGISTERS - && intarg_count % 2 != 0)) - intarg_count++; - - intarg_count += MODE_CHOICE(2,1); - - break; - - case FFI_TYPE_STRUCT: - size_al = (*ptr)->size; - /* If the first member of the struct is a double, then align - the struct to double-word. */ - if ((*ptr)->elements[0]->type == FFI_TYPE_DOUBLE) - size_al = ALIGN((*ptr)->size, 8); - -#if defined(__ppc64__) - // Look for FP struct members. - unsigned int j; - - for (j = 0; (*ptr)->elements[j] != NULL; j++) - { - if ((*ptr)->elements[j]->type == FFI_TYPE_FLOAT || - (*ptr)->elements[j]->type == FFI_TYPE_DOUBLE) - { - fparg_count++; - - if (fparg_count > NUM_FPR_ARG_REGISTERS) - intarg_count++; - } - else if ((*ptr)->elements[j]->type == FFI_TYPE_LONGDOUBLE) - { - fparg_count += 2; - - if (fparg_count > NUM_FPR_ARG_REGISTERS + 1) - intarg_count += 2; - } - else - intarg_count++; - } -#elif defined(__ppc__) - intarg_count += (size_al + 3) / 4; -#else -#error undefined architecture -#endif - - break; - - default: - /* Everything else is passed as a 4/8-byte word in a GPR, either - the object itself or a pointer to it. */ - intarg_count++; - break; - } - } - - /* Space for the FPR registers, if needed. */ - if (fparg_count != 0) - { - flags |= FLAG_FP_ARGUMENTS; -#if defined(__ppc64__) - bytes += (NUM_FPR_ARG_REGISTERS + 1) * sizeof(double); -#elif defined(__ppc__) - bytes += NUM_FPR_ARG_REGISTERS * sizeof(double); -#else -#error undefined architecture -#endif - } - - /* Stack space. */ -#if defined(__ppc64__) - if ((intarg_count + fparg_count) > NUM_GPR_ARG_REGISTERS) - bytes += (intarg_count + fparg_count) * sizeof(long); -#elif defined(__ppc__) - if ((intarg_count + 2 * fparg_count) > NUM_GPR_ARG_REGISTERS) - bytes += (intarg_count + 2 * fparg_count) * sizeof(long); -#else -#error undefined architecture -#endif - else - bytes += NUM_GPR_ARG_REGISTERS * sizeof(long); - - /* The stack space allocated needs to be a multiple of 16/32 bytes. */ - bytes = SF_ROUND(bytes); - - cif->flags = flags; - cif->bytes = bytes; - - return FFI_OK; -} - -/*@-declundef@*/ -/*@-exportheader@*/ -extern void -ffi_call_AIX( -/*@out@*/ extended_cif*, - unsigned, - unsigned, -/*@out@*/ unsigned*, - void (*fn)(void), - void (*fn2)(extended_cif*, unsigned *const)); - -extern void -ffi_call_DARWIN( -/*@out@*/ extended_cif*, - unsigned long, - unsigned, -/*@out@*/ unsigned*, - void (*fn)(void), - void (*fn2)(extended_cif*, unsigned *const)); -/*@=declundef@*/ -/*@=exportheader@*/ - -void -ffi_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ void** avalue) -{ - extended_cif ecif; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return - value address then we need to make one. */ - if ((rvalue == NULL) && - (cif->rtype->type == FFI_TYPE_STRUCT)) - { - /*@-sysunrecog@*/ - ecif.rvalue = alloca(cif->rtype->size); - /*@=sysunrecog@*/ - } - else - ecif.rvalue = rvalue; - - switch (cif->abi) - { - case FFI_AIX: - /*@-usedef@*/ - ffi_call_AIX(&ecif, -cif->bytes, - cif->flags, ecif.rvalue, fn, ffi_prep_args); - /*@=usedef@*/ - break; - - case FFI_DARWIN: - /*@-usedef@*/ - ffi_call_DARWIN(&ecif, -(long)cif->bytes, - cif->flags, ecif.rvalue, fn, ffi_prep_args); - /*@=usedef@*/ - break; - - default: - FFI_ASSERT(0); - break; - } -} - -/* here I'd like to add the stack frame layout we use in darwin_closure.S - and aix_clsoure.S - - SP previous -> +---------------------------------------+ <--- child frame - | back chain to caller 4 | - +---------------------------------------+ 4 - | saved CR 4 | - +---------------------------------------+ 8 - | saved LR 4 | - +---------------------------------------+ 12 - | reserved for compilers 4 | - +---------------------------------------+ 16 - | reserved for binders 4 | - +---------------------------------------+ 20 - | saved TOC pointer 4 | - +---------------------------------------+ 24 - | always reserved 8*4=32 (previous GPRs)| - | according to the linkage convention | - | from AIX | - +---------------------------------------+ 56 - | our FPR area 13*8=104 | - | f1 | - | . | - | f13 | - +---------------------------------------+ 160 - | result area 8 | - +---------------------------------------+ 168 - | alignement to the next multiple of 16 | -SP current --> +---------------------------------------+ 176 <- parent frame - | back chain to caller 4 | - +---------------------------------------+ 180 - | saved CR 4 | - +---------------------------------------+ 184 - | saved LR 4 | - +---------------------------------------+ 188 - | reserved for compilers 4 | - +---------------------------------------+ 192 - | reserved for binders 4 | - +---------------------------------------+ 196 - | saved TOC pointer 4 | - +---------------------------------------+ 200 - | always reserved 8*4=32 we store our | - | GPRs here | - | r3 | - | . | - | r10 | - +---------------------------------------+ 232 - | overflow part | - +---------------------------------------+ xxx - | ???? | - +---------------------------------------+ xxx -*/ - -#if !defined(POWERPC_DARWIN) - -#define MIN_LINE_SIZE 32 - -static void -flush_icache( - char* addr) -{ -#ifndef _AIX - __asm__ volatile ( - "dcbf 0,%0\n" - "sync\n" - "icbi 0,%0\n" - "sync\n" - "isync" - : : "r" (addr) : "memory"); -#endif -} - -static void -flush_range( - char* addr, - int size) -{ - int i; - - for (i = 0; i < size; i += MIN_LINE_SIZE) - flush_icache(addr + i); - - flush_icache(addr + size - 1); -} - -#endif // !defined(POWERPC_DARWIN) - -ffi_status -ffi_prep_closure( - ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*, void*, void**, void*), - void* user_data) -{ - switch (cif->abi) - { - case FFI_DARWIN: - { - FFI_ASSERT (cif->abi == FFI_DARWIN); - - unsigned int* tramp = (unsigned int*)&closure->tramp[0]; - -#if defined(__ppc64__) - tramp[0] = 0x7c0802a6; // mflr r0 - tramp[1] = 0x429f0005; // bcl 20,31,+0x8 - tramp[2] = 0x7d6802a6; // mflr r11 - tramp[3] = 0x7c0803a6; // mtlr r0 - tramp[4] = 0xe98b0018; // ld r12,24(r11) - tramp[5] = 0x7d8903a6; // mtctr r12 - tramp[6] = 0xe96b0020; // ld r11,32(r11) - tramp[7] = 0x4e800420; // bctr - *(unsigned long*)&tramp[8] = (unsigned long)ffi_closure_ASM; - *(unsigned long*)&tramp[10] = (unsigned long)closure; -#elif defined(__ppc__) - tramp[0] = 0x7c0802a6; // mflr r0 - tramp[1] = 0x429f0005; // bcl 20,31,+0x8 - tramp[2] = 0x7d6802a6; // mflr r11 - tramp[3] = 0x7c0803a6; // mtlr r0 - tramp[4] = 0x818b0018; // lwz r12,24(r11) - tramp[5] = 0x7d8903a6; // mtctr r12 - tramp[6] = 0x816b001c; // lwz r11,28(r11) - tramp[7] = 0x4e800420; // bctr - tramp[8] = (unsigned long)ffi_closure_ASM; - tramp[9] = (unsigned long)closure; -#else -#error undefined architecture -#endif - - closure->cif = cif; - closure->fun = fun; - closure->user_data = user_data; - - // Flush the icache. Only necessary on Darwin. -#if defined(POWERPC_DARWIN) - sys_icache_invalidate(closure->tramp, FFI_TRAMPOLINE_SIZE); -#else - flush_range(closure->tramp, FFI_TRAMPOLINE_SIZE); -#endif - - break; - } - - case FFI_AIX: - { - FFI_ASSERT (cif->abi == FFI_AIX); - - ffi_aix_trampoline_struct* tramp_aix = - (ffi_aix_trampoline_struct*)(closure->tramp); - aix_fd* fd = (aix_fd*)(void*)ffi_closure_ASM; - - tramp_aix->code_pointer = fd->code_pointer; - tramp_aix->toc = fd->toc; - tramp_aix->static_chain = closure; - closure->cif = cif; - closure->fun = fun; - closure->user_data = user_data; - break; - } - - default: - return FFI_BAD_ABI; - } - - return FFI_OK; -} - -#if defined(__ppc__) - typedef double ldbits[2]; - - typedef union - { - ldbits lb; - long double ld; - } ldu; -#endif - -typedef union -{ - float f; - double d; -} ffi_dblfl; - -/* The trampoline invokes ffi_closure_ASM, and on entry, r11 holds the - address of the closure. After storing the registers that could possibly - contain parameters to be passed into the stack frame and setting up space - for a return value, ffi_closure_ASM invokes the following helper function - to do most of the work. */ -int -ffi_closure_helper_DARWIN( - ffi_closure* closure, - void* rvalue, - unsigned long* pgr, - ffi_dblfl* pfr) -{ - /* rvalue is the pointer to space for return value in closure assembly - pgr is the pointer to where r3-r10 are stored in ffi_closure_ASM - pfr is the pointer to where f1-f13 are stored in ffi_closure_ASM. */ - -#if defined(__ppc__) - ldu temp_ld; -#endif - - double temp; - unsigned int i; - unsigned int nf = 0; /* number of FPRs already used. */ - unsigned int ng = 0; /* number of GPRs already used. */ - ffi_cif* cif = closure->cif; - long avn = cif->nargs; - void** avalue = alloca(cif->nargs * sizeof(void*)); - ffi_type** arg_types = cif->arg_types; - - /* Copy the caller's structure return value address so that the closure - returns the data directly to the caller. */ -#if defined(__ppc64__) - if (cif->rtype->type == FFI_TYPE_STRUCT && - ffi64_stret_needs_ptr(cif->rtype, NULL, NULL)) -#elif defined(__ppc__) - if (cif->rtype->type == FFI_TYPE_STRUCT) -#else -#error undefined architecture -#endif - { - rvalue = (void*)*pgr; - pgr++; - ng++; - } - - /* Grab the addresses of the arguments from the stack frame. */ - for (i = 0; i < avn; i++) - { - switch (arg_types[i]->type) - { - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT8: - avalue[i] = (char*)pgr + MODE_CHOICE(3,7); - ng++; - pgr++; - break; - - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT16: - avalue[i] = (char*)pgr + MODE_CHOICE(2,6); - ng++; - pgr++; - break; - -#if defined(__ppc__) - case FFI_TYPE_POINTER: -#endif - case FFI_TYPE_SINT32: - case FFI_TYPE_UINT32: - avalue[i] = (char*)pgr + MODE_CHOICE(0,4); - ng++; - pgr++; - - break; - - case FFI_TYPE_STRUCT: - if (cif->abi == FFI_DARWIN) - { -#if defined(__ppc64__) - unsigned int gprSize = 0; - unsigned int fprSize = 0; - unsigned int savedFPRSize = fprSize; - - avalue[i] = alloca(arg_types[i]->size); - ffi64_struct_to_ram_form(arg_types[i], (const char*)pgr, - &gprSize, (const char*)pfr, &fprSize, &nf, avalue[i], NULL); - - ng += gprSize / sizeof(long); - pgr += gprSize / sizeof(long); - pfr += (fprSize - savedFPRSize) / sizeof(double); - -#elif defined(__ppc__) - /* Structures that match the basic modes (QI 1 byte, HI 2 bytes, - SI 4 bytes) are aligned as if they were those modes. */ - unsigned int size_al = size_al = arg_types[i]->size; - - /* If the first member of the struct is a double, then align - the struct to double-word. */ - if (arg_types[i]->elements[0]->type == FFI_TYPE_DOUBLE) - size_al = ALIGN(arg_types[i]->size, 8); - - if (size_al < 3) - avalue[i] = (void*)pgr + MODE_CHOICE(4,8) - size_al; - else - avalue[i] = (void*)pgr; - - ng += (size_al + 3) / sizeof(long); - pgr += (size_al + 3) / sizeof(long); -#else -#error undefined architecture -#endif - } - - break; - -#if defined(__ppc64__) - case FFI_TYPE_POINTER: -#endif - case FFI_TYPE_SINT64: - case FFI_TYPE_UINT64: - /* Long long ints are passed in 1 or 2 GPRs. */ - avalue[i] = pgr; - ng += MODE_CHOICE(2,1); - pgr += MODE_CHOICE(2,1); - - break; - - case FFI_TYPE_FLOAT: - /* A float value consumes a GPR. - There are 13 64-bit floating point registers. */ - if (nf < NUM_FPR_ARG_REGISTERS) - { - temp = pfr->d; - pfr->f = (float)temp; - avalue[i] = pfr; - pfr++; - } - else - avalue[i] = pgr; - - nf++; - ng++; - pgr++; - break; - - case FFI_TYPE_DOUBLE: - /* A double value consumes one or two GPRs. - There are 13 64bit floating point registers. */ - if (nf < NUM_FPR_ARG_REGISTERS) - { - avalue[i] = pfr; - pfr++; - } - else - avalue[i] = pgr; - - nf++; - ng += MODE_CHOICE(2,1); - pgr += MODE_CHOICE(2,1); - - break; - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_LONGDOUBLE: -#if defined(__ppc64__) - if (nf < NUM_FPR_ARG_REGISTERS) - { - avalue[i] = pfr; - pfr += 2; - } -#elif defined(__ppc__) - /* A long double value consumes 2/4 GPRs and 2 FPRs. - There are 13 64bit floating point registers. */ - if (nf < NUM_FPR_ARG_REGISTERS - 1) - { - avalue[i] = pfr; - pfr += 2; - } - /* Here we have the situation where one part of the long double - is stored in fpr13 and the other part is already on the stack. - We use a union to pass the long double to avalue[i]. */ - else if (nf == NUM_FPR_ARG_REGISTERS - 1) - { - memcpy (&temp_ld.lb[0], pfr, sizeof(temp_ld.lb[0])); - memcpy (&temp_ld.lb[1], pgr + 2, sizeof(temp_ld.lb[1])); - avalue[i] = &temp_ld.ld; - } -#else -#error undefined architecture -#endif - else - avalue[i] = pgr; - - nf += 2; - ng += MODE_CHOICE(4,2); - pgr += MODE_CHOICE(4,2); - - break; - -#endif /* FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE */ - - default: - FFI_ASSERT(0); - break; - } - } - - (closure->fun)(cif, rvalue, avalue, closure->user_data); - - /* Tell ffi_closure_ASM to perform return type promotions. */ - return cif->rtype->type; -} - -#if defined(__ppc64__) - -/* ffi64_struct_to_ram_form - - Rebuild a struct's natural layout from buffers of concatenated registers. - Return the number of registers used. - inGPRs[0-7] == r3, inFPRs[0-7] == f1 ... -*/ -void -ffi64_struct_to_ram_form( - const ffi_type* inType, - const char* inGPRs, - unsigned int* ioGPRMarker, - const char* inFPRs, - unsigned int* ioFPRMarker, - unsigned int* ioFPRsUsed, - char* outStruct, // caller-allocated - unsigned int* ioStructMarker) -{ - unsigned int srcGMarker = 0; - unsigned int srcFMarker = 0; - unsigned int savedFMarker = 0; - unsigned int fprsUsed = 0; - unsigned int savedFPRsUsed = 0; - unsigned int destMarker = 0; - - static unsigned int recurseCount = 0; - - if (ioGPRMarker) - srcGMarker = *ioGPRMarker; - - if (ioFPRMarker) - { - srcFMarker = *ioFPRMarker; - savedFMarker = srcFMarker; - } - - if (ioFPRsUsed) - { - fprsUsed = *ioFPRsUsed; - savedFPRsUsed = fprsUsed; - } - - if (ioStructMarker) - destMarker = *ioStructMarker; - - size_t i; - - switch (inType->size) - { - case 1: case 2: case 4: - srcGMarker += 8 - inType->size; - break; - - default: - break; - } - - for (i = 0; inType->elements[i] != NULL; i++) - { - switch (inType->elements[i]->type) - { - case FFI_TYPE_FLOAT: - srcFMarker = ALIGN(srcFMarker, 4); - srcGMarker = ALIGN(srcGMarker, 4); - destMarker = ALIGN(destMarker, 4); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - *(float*)&outStruct[destMarker] = - (float)*(double*)&inFPRs[srcFMarker]; - srcFMarker += 8; - fprsUsed++; - } - else - *(float*)&outStruct[destMarker] = - (float)*(double*)&inGPRs[srcGMarker]; - - srcGMarker += 4; - destMarker += 4; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (destMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 4)) - srcGMarker = ALIGN(srcGMarker, 8); - } - - break; - - case FFI_TYPE_DOUBLE: - srcFMarker = ALIGN(srcFMarker, 8); - destMarker = ALIGN(destMarker, 8); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - *(double*)&outStruct[destMarker] = - *(double*)&inFPRs[srcFMarker]; - srcFMarker += 8; - fprsUsed++; - } - else - *(double*)&outStruct[destMarker] = - *(double*)&inGPRs[srcGMarker]; - - destMarker += 8; - - // Skip next GPR - srcGMarker += 8; - srcGMarker = ALIGN(srcGMarker, 8); - - break; - - case FFI_TYPE_LONGDOUBLE: - destMarker = ALIGN(destMarker, 16); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - srcFMarker = ALIGN(srcFMarker, 8); - srcGMarker = ALIGN(srcGMarker, 8); - *(long double*)&outStruct[destMarker] = - *(long double*)&inFPRs[srcFMarker]; - srcFMarker += 16; - fprsUsed += 2; - } - else - { - srcFMarker = ALIGN(srcFMarker, 16); - srcGMarker = ALIGN(srcGMarker, 16); - *(long double*)&outStruct[destMarker] = - *(long double*)&inGPRs[srcGMarker]; - } - - destMarker += 16; - - // Skip next 2 GPRs - srcGMarker += 16; - srcGMarker = ALIGN(srcGMarker, 8); - - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - { - if (inType->alignment == 1) // chars only - { - if (inType->size == 1) - outStruct[destMarker++] = inGPRs[srcGMarker++]; - else if (inType->size == 2) - { - outStruct[destMarker++] = inGPRs[srcGMarker++]; - outStruct[destMarker++] = inGPRs[srcGMarker++]; - i++; - } - else - { - memcpy(&outStruct[destMarker], - &inGPRs[srcGMarker], inType->size); - srcGMarker += inType->size; - destMarker += inType->size; - i += inType->size - 1; - } - } - else // chars and other stuff - { - outStruct[destMarker++] = inGPRs[srcGMarker++]; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (srcGMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 4)) - srcGMarker = ALIGN(srcGMarker, inType->alignment); // was 8 - } - } - - break; - } - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - srcGMarker = ALIGN(srcGMarker, 2); - destMarker = ALIGN(destMarker, 2); - - *(short*)&outStruct[destMarker] = - *(short*)&inGPRs[srcGMarker]; - srcGMarker += 2; - destMarker += 2; - - break; - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - srcGMarker = ALIGN(srcGMarker, 4); - destMarker = ALIGN(destMarker, 4); - - *(int*)&outStruct[destMarker] = - *(int*)&inGPRs[srcGMarker]; - srcGMarker += 4; - destMarker += 4; - - break; - - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - srcGMarker = ALIGN(srcGMarker, 8); - destMarker = ALIGN(destMarker, 8); - - *(long long*)&outStruct[destMarker] = - *(long long*)&inGPRs[srcGMarker]; - srcGMarker += 8; - destMarker += 8; - - break; - - case FFI_TYPE_STRUCT: - recurseCount++; - ffi64_struct_to_ram_form(inType->elements[i], inGPRs, - &srcGMarker, inFPRs, &srcFMarker, &fprsUsed, - outStruct, &destMarker); - recurseCount--; - break; - - default: - FFI_ASSERT(0); // unknown element type - break; - } - } - - srcGMarker = ALIGN(srcGMarker, inType->alignment); - - // Take care of the special case for 16-byte structs, but not for - // nested structs. - if (recurseCount == 0 && srcGMarker == 16) - { - *(long double*)&outStruct[0] = *(long double*)&inGPRs[0]; - srcFMarker = savedFMarker; - fprsUsed = savedFPRsUsed; - } - - if (ioGPRMarker) - *ioGPRMarker = ALIGN(srcGMarker, 8); - - if (ioFPRMarker) - *ioFPRMarker = srcFMarker; - - if (ioFPRsUsed) - *ioFPRsUsed = fprsUsed; - - if (ioStructMarker) - *ioStructMarker = ALIGN(destMarker, 8); -} - -/* ffi64_struct_to_reg_form - - Copy a struct's elements into buffers that can be sliced into registers. - Return the sizes of the output buffers in bytes. Pass NULL buffer pointers - to calculate size only. - outGPRs[0-7] == r3, outFPRs[0-7] == f1 ... -*/ -void -ffi64_struct_to_reg_form( - const ffi_type* inType, - const char* inStruct, - unsigned int* ioStructMarker, - unsigned int* ioFPRsUsed, - char* outGPRs, // caller-allocated - unsigned int* ioGPRSize, - char* outFPRs, // caller-allocated - unsigned int* ioFPRSize) -{ - size_t i; - unsigned int srcMarker = 0; - unsigned int destGMarker = 0; - unsigned int destFMarker = 0; - unsigned int savedFMarker = 0; - unsigned int fprsUsed = 0; - unsigned int savedFPRsUsed = 0; - - static unsigned int recurseCount = 0; - - if (ioStructMarker) - srcMarker = *ioStructMarker; - - if (ioFPRsUsed) - { - fprsUsed = *ioFPRsUsed; - savedFPRsUsed = fprsUsed; - } - - if (ioGPRSize) - destGMarker = *ioGPRSize; - - if (ioFPRSize) - { - destFMarker = *ioFPRSize; - savedFMarker = destFMarker; - } - - switch (inType->size) - { - case 1: case 2: case 4: - destGMarker += 8 - inType->size; - break; - - default: - break; - } - - for (i = 0; inType->elements[i] != NULL; i++) - { - switch (inType->elements[i]->type) - { - // Shadow floating-point types in GPRs for vararg and pre-ANSI - // functions. - case FFI_TYPE_FLOAT: - // Nudge markers to next 4/8-byte boundary - srcMarker = ALIGN(srcMarker, 4); - destGMarker = ALIGN(destGMarker, 4); - destFMarker = ALIGN(destFMarker, 8); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - if (outFPRs != NULL && inStruct != NULL) - *(double*)&outFPRs[destFMarker] = - (double)*(float*)&inStruct[srcMarker]; - - destFMarker += 8; - fprsUsed++; - } - - if (outGPRs != NULL && inStruct != NULL) - *(double*)&outGPRs[destGMarker] = - (double)*(float*)&inStruct[srcMarker]; - - srcMarker += 4; - destGMarker += 4; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (srcMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 4)) - destGMarker = ALIGN(destGMarker, 8); - } - - break; - - case FFI_TYPE_DOUBLE: - srcMarker = ALIGN(srcMarker, 8); - destFMarker = ALIGN(destFMarker, 8); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - if (outFPRs != NULL && inStruct != NULL) - *(double*)&outFPRs[destFMarker] = - *(double*)&inStruct[srcMarker]; - - destFMarker += 8; - fprsUsed++; - } - - if (outGPRs != NULL && inStruct != NULL) - *(double*)&outGPRs[destGMarker] = - *(double*)&inStruct[srcMarker]; - - srcMarker += 8; - - // Skip next GPR - destGMarker += 8; - destGMarker = ALIGN(destGMarker, 8); - - break; - - case FFI_TYPE_LONGDOUBLE: - srcMarker = ALIGN(srcMarker, 16); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - destFMarker = ALIGN(destFMarker, 8); - destGMarker = ALIGN(destGMarker, 8); - - if (outFPRs != NULL && inStruct != NULL) - *(long double*)&outFPRs[destFMarker] = - *(long double*)&inStruct[srcMarker]; - - if (outGPRs != NULL && inStruct != NULL) - *(long double*)&outGPRs[destGMarker] = - *(long double*)&inStruct[srcMarker]; - - destFMarker += 16; - fprsUsed += 2; - } - else - { - destGMarker = ALIGN(destGMarker, 16); - - if (outGPRs != NULL && inStruct != NULL) - *(long double*)&outGPRs[destGMarker] = - *(long double*)&inStruct[srcMarker]; - } - - srcMarker += 16; - destGMarker += 16; // Skip next 2 GPRs - destGMarker = ALIGN(destGMarker, 8); // was 16 - - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - if (inType->alignment == 1) // bytes only - { - if (inType->size == 1) - { - if (outGPRs != NULL && inStruct != NULL) - outGPRs[destGMarker] = inStruct[srcMarker]; - - srcMarker++; - destGMarker++; - } - else if (inType->size == 2) - { - if (outGPRs != NULL && inStruct != NULL) - { - outGPRs[destGMarker] = inStruct[srcMarker]; - outGPRs[destGMarker + 1] = inStruct[srcMarker + 1]; - } - - srcMarker += 2; - destGMarker += 2; - - i++; - } - else - { - if (outGPRs != NULL && inStruct != NULL) - { - // Avoid memcpy for small chunks. - if (inType->size <= sizeof(long)) - *(long*)&outGPRs[destGMarker] = - *(long*)&inStruct[srcMarker]; - else - memcpy(&outGPRs[destGMarker], - &inStruct[srcMarker], inType->size); - } - - srcMarker += inType->size; - destGMarker += inType->size; - i += inType->size - 1; - } - } - else // bytes and other stuff - { - if (outGPRs != NULL && inStruct != NULL) - outGPRs[destGMarker] = inStruct[srcMarker]; - - srcMarker++; - destGMarker++; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (destGMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 4)) - destGMarker = ALIGN(destGMarker, inType->alignment); // was 8 - } - } - - break; - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - srcMarker = ALIGN(srcMarker, 2); - destGMarker = ALIGN(destGMarker, 2); - - if (outGPRs != NULL && inStruct != NULL) - *(short*)&outGPRs[destGMarker] = - *(short*)&inStruct[srcMarker]; - - srcMarker += 2; - destGMarker += 2; - - if (inType->elements[i + 1] == NULL) - destGMarker = ALIGN(destGMarker, inType->alignment); - - break; - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - srcMarker = ALIGN(srcMarker, 4); - destGMarker = ALIGN(destGMarker, 4); - - if (outGPRs != NULL && inStruct != NULL) - *(int*)&outGPRs[destGMarker] = - *(int*)&inStruct[srcMarker]; - - srcMarker += 4; - destGMarker += 4; - - break; - - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - srcMarker = ALIGN(srcMarker, 8); - destGMarker = ALIGN(destGMarker, 8); - - if (outGPRs != NULL && inStruct != NULL) - *(long long*)&outGPRs[destGMarker] = - *(long long*)&inStruct[srcMarker]; - - srcMarker += 8; - destGMarker += 8; - - if (inType->elements[i + 1] == NULL) - destGMarker = ALIGN(destGMarker, inType->alignment); - - break; - - case FFI_TYPE_STRUCT: - recurseCount++; - ffi64_struct_to_reg_form(inType->elements[i], - inStruct, &srcMarker, &fprsUsed, outGPRs, - &destGMarker, outFPRs, &destFMarker); - recurseCount--; - break; - - default: - FFI_ASSERT(0); - break; - } - } - - destGMarker = ALIGN(destGMarker, inType->alignment); - - // Take care of the special case for 16-byte structs, but not for - // nested structs. - if (recurseCount == 0 && destGMarker == 16) - { - if (outGPRs != NULL && inStruct != NULL) - *(long double*)&outGPRs[0] = *(long double*)&inStruct[0]; - - destFMarker = savedFMarker; - fprsUsed = savedFPRsUsed; - } - - if (ioStructMarker) - *ioStructMarker = ALIGN(srcMarker, 8); - - if (ioFPRsUsed) - *ioFPRsUsed = fprsUsed; - - if (ioGPRSize) - *ioGPRSize = ALIGN(destGMarker, 8); - - if (ioFPRSize) - *ioFPRSize = ALIGN(destFMarker, 8); -} - -/* ffi64_stret_needs_ptr - - Determine whether a returned struct needs a pointer in r3 or can fit - in registers. -*/ - -bool -ffi64_stret_needs_ptr( - const ffi_type* inType, - unsigned short* ioGPRCount, - unsigned short* ioFPRCount) -{ - // Obvious case first- struct is larger than combined FPR size. - if (inType->size > 14 * 8) - return true; - - // Now the struct can physically fit in registers, determine if it - // also fits logically. - bool needsPtr = false; - unsigned short gprsUsed = 0; - unsigned short fprsUsed = 0; - size_t i; - - if (ioGPRCount) - gprsUsed = *ioGPRCount; - - if (ioFPRCount) - fprsUsed = *ioFPRCount; - - for (i = 0; inType->elements[i] != NULL && !needsPtr; i++) - { - switch (inType->elements[i]->type) - { - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - gprsUsed++; - fprsUsed++; - - if (fprsUsed > 13) - needsPtr = true; - - break; - - case FFI_TYPE_LONGDOUBLE: - gprsUsed += 2; - fprsUsed += 2; - - if (fprsUsed > 14) - needsPtr = true; - - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - { - gprsUsed++; - - if (gprsUsed > 8) - { - needsPtr = true; - break; - } - - if (inType->elements[i + 1] == NULL) // last byte in the struct - break; - - // Count possible contiguous bytes ahead, up to 8. - unsigned short j; - - for (j = 1; j < 8; j++) - { - if (inType->elements[i + j] == NULL || - !FFI_TYPE_1_BYTE(inType->elements[i + j]->type)) - break; - } - - i += j - 1; // allow for i++ before the test condition - - break; - } - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - gprsUsed++; - - if (gprsUsed > 8) - needsPtr = true; - - break; - - case FFI_TYPE_STRUCT: - needsPtr = ffi64_stret_needs_ptr( - inType->elements[i], &gprsUsed, &fprsUsed); - - break; - - default: - FFI_ASSERT(0); - break; - } - } - - if (ioGPRCount) - *ioGPRCount = gprsUsed; - - if (ioFPRCount) - *ioFPRCount = fprsUsed; - - return needsPtr; -} - -/* ffi64_data_size - - Calculate the size in bytes of an ffi type. -*/ - -unsigned int -ffi64_data_size( - const ffi_type* inType) -{ - unsigned int size = 0; - - switch (inType->type) - { - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - size = 1; - break; - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - size = 2; - break; - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_FLOAT: - size = 4; - break; - - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - case FFI_TYPE_DOUBLE: - size = 8; - break; - - case FFI_TYPE_LONGDOUBLE: - size = 16; - break; - - case FFI_TYPE_STRUCT: - ffi64_struct_to_reg_form( - inType, NULL, NULL, NULL, NULL, &size, NULL, NULL); - break; - - case FFI_TYPE_VOID: - break; - - default: - FFI_ASSERT(0); - break; - } - - return size; -} - -#endif /* defined(__ppc64__) */ -#endif /* __ppc__ || __ppc64__ */ diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S b/Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S deleted file mode 100644 index 7162fa1dda65..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S +++ /dev/null @@ -1,418 +0,0 @@ -#if defined(__ppc64__) - -/* ----------------------------------------------------------------------- - ppc64-darwin_closure.S - Copyright (c) 2002, 2003, 2004, Free Software Foundation, - Inc. based on ppc_closure.S - - PowerPC Assembly glue. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define LIBFFI_ASM - -#include <ffi.h> -#include <ppc-ffitarget.h> // for FFI_TRAMPOLINE_SIZE -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - - .file "ppc64-darwin_closure.S" -.text - .align LOG2_GPR_BYTES - .globl _ffi_closure_ASM - -.text - .align LOG2_GPR_BYTES - -_ffi_closure_ASM: -LFB1: - mflr r0 - stg r0,SF_RETURN(r1) // save return address - - // Save GPRs 3 - 10 (aligned to 8) in the parents outgoing area. - stg r3,SF_ARG1(r1) - stg r4,SF_ARG2(r1) - stg r5,SF_ARG3(r1) - stg r6,SF_ARG4(r1) - stg r7,SF_ARG5(r1) - stg r8,SF_ARG6(r1) - stg r9,SF_ARG7(r1) - stg r10,SF_ARG8(r1) - -LCFI0: -/* 48 bytes (Linkage Area) - 64 bytes (outgoing parameter area, always reserved) - 112 bytes (14*8 for incoming FPR) - ? bytes (result) - 112 bytes (14*8 for outgoing FPR) - 16 bytes (2 saved registers) - 352 + ? total bytes -*/ - - std r31,-8(r1) // Save registers we use. - std r30,-16(r1) - mr r30,r1 // Save the old SP. - mr r31,r11 // Save the ffi_closure around ffi64_data_size. - - // Calculate the space we need. - stdu r1,-SF_MINSIZE(r1) - ld r3,FFI_TRAMPOLINE_SIZE(r31) // ffi_closure->cif* - ld r3,16(r3) // ffi_cif->rtype* - bl Lffi64_data_size$stub - ld r1,0(r1) - - addi r3,r3,352 // Add our overhead. - neg r3,r3 - li r0,-32 // Align to 32 bytes. - and r3,r3,r0 - stdux r1,r1,r3 // Grow the stack. - - mr r11,r31 // Copy the ffi_closure back. - -LCFI1: - // We want to build up an area for the parameters passed - // in registers. (both floating point and integer) - -/* 320 bytes (callee stack frame aligned to 32) - 48 bytes (caller linkage area) - 368 (start of caller parameter area aligned to 8) -*/ - - // Save FPRs 1 - 14. (aligned to 8) - stfd f1,112(r1) - stfd f2,120(r1) - stfd f3,128(r1) - stfd f4,136(r1) - stfd f5,144(r1) - stfd f6,152(r1) - stfd f7,160(r1) - stfd f8,168(r1) - stfd f9,176(r1) - stfd f10,184(r1) - stfd f11,192(r1) - stfd f12,200(r1) - stfd f13,208(r1) - stfd f14,216(r1) - - // Set up registers for the routine that actually does the work. - mr r3,r11 // context pointer from the trampoline - addi r4,r1,224 // result storage - addi r5,r30,SF_ARG1 // saved GPRs - addi r6,r1,112 // saved FPRs - bl Lffi_closure_helper_DARWIN$stub - - // Look the proper starting point in table - // by using return type as an offset. - addi r5,r1,224 // Get pointer to results area. - bl Lget_ret_type0_addr // Get pointer to Lret_type0 into LR. - mflr r4 // Move to r4. - slwi r3,r3,4 // Now multiply return type by 16. - add r3,r3,r4 // Add contents of table to table address. - mtctr r3 - bctr - -LFE1: - // Each of the ret_typeX code fragments has to be exactly 16 bytes long - // (4 instructions). For cache effectiveness we align to a 16 byte - // boundary first. - .align 4 - nop - nop - nop - -Lget_ret_type0_addr: - blrl - -// case FFI_TYPE_VOID -Lret_type0: - b Lfinish - nop - nop - nop - -// case FFI_TYPE_INT -Lret_type1: - lwz r3,4(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_FLOAT -Lret_type2: - lfs f1,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_DOUBLE -Lret_type3: - lfd f1,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_LONGDOUBLE -Lret_type4: - lfd f1,0(r5) - lfd f2,8(r5) - b Lfinish - nop - -// case FFI_TYPE_UINT8 -Lret_type5: - lbz r3,7(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT8 -Lret_type6: - lbz r3,7(r5) - extsb r3,r3 - b Lfinish - nop - -// case FFI_TYPE_UINT16 -Lret_type7: - lhz r3,6(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT16 -Lret_type8: - lha r3,6(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_UINT32 -Lret_type9: // same as Lret_type1 - lwz r3,4(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT32 -Lret_type10: // same as Lret_type1 - lwz r3,4(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_UINT64 -Lret_type11: - ld r3,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT64 -Lret_type12: // same as Lret_type11 - ld r3,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_STRUCT -Lret_type13: - b Lret_struct - nop - nop - nop - -// ** End 16-byte aligned cases ** -// case FFI_TYPE_POINTER -// This case assumes that FFI_TYPE_POINTER == FFI_TYPE_LAST. If more types -// are added in future, the following code will need to be updated and -// padded to 16 bytes. -Lret_type14: - lg r3,0(r5) - b Lfinish - -// copy struct into registers -Lret_struct: - ld r31,FFI_TRAMPOLINE_SIZE(r31) // ffi_closure->cif* - ld r3,16(r31) // ffi_cif->rtype* - ld r31,24(r31) // ffi_cif->flags - mr r4,r5 // copy struct* to 2nd arg - addi r7,r1,SF_ARG9 // GPR return area - addi r9,r30,-16-(14*8) // FPR return area - li r5,0 // struct offset ptr (NULL) - li r6,0 // FPR used count ptr (NULL) - li r8,0 // GPR return area size ptr (NULL) - li r10,0 // FPR return area size ptr (NULL) - bl Lffi64_struct_to_reg_form$stub - - // Load GPRs - ld r3,SF_ARG9(r1) - ld r4,SF_ARG10(r1) - ld r5,SF_ARG11(r1) - ld r6,SF_ARG12(r1) - nop - ld r7,SF_ARG13(r1) - ld r8,SF_ARG14(r1) - ld r9,SF_ARG15(r1) - ld r10,SF_ARG16(r1) - nop - - // Load FPRs - mtcrf 0x2,r31 - bf 26,Lfinish - lfd f1,-16-(14*8)(r30) - lfd f2,-16-(13*8)(r30) - lfd f3,-16-(12*8)(r30) - lfd f4,-16-(11*8)(r30) - nop - lfd f5,-16-(10*8)(r30) - lfd f6,-16-(9*8)(r30) - lfd f7,-16-(8*8)(r30) - lfd f8,-16-(7*8)(r30) - nop - lfd f9,-16-(6*8)(r30) - lfd f10,-16-(5*8)(r30) - lfd f11,-16-(4*8)(r30) - lfd f12,-16-(3*8)(r30) - nop - lfd f13,-16-(2*8)(r30) - lfd f14,-16-(1*8)(r30) - // Fall through - -// case done -Lfinish: - lg r1,0(r1) // Restore stack pointer. - ld r31,-8(r1) // Restore registers we used. - ld r30,-16(r1) - lg r0,SF_RETURN(r1) // Get return address. - mtlr r0 // Reset link register. - blr - -// END(ffi_closure_ASM) - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 ; Length of Common Information Entry -LSCIE1: - .long 0x0 ; CIE Identifier Tag - .byte 0x1 ; CIE Version - .ascii "zR\0" ; CIE Augmentation - .byte 0x1 ; uleb128 0x1; CIE Code Alignment Factor - .byte 0x7c ; sleb128 -4; CIE Data Alignment Factor - .byte 0x41 ; CIE RA Column - .byte 0x1 ; uleb128 0x1; Augmentation size - .byte 0x10 ; FDE Encoding (pcrel) - .byte 0xc ; DW_CFA_def_cfa - .byte 0x1 ; uleb128 0x1 - .byte 0x0 ; uleb128 0x0 - .align LOG2_GPR_BYTES -LECIE1: -.globl _ffi_closure_ASM.eh -_ffi_closure_ASM.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 ; FDE Length - -LASFDE1: - .long LASFDE1-EH_frame1 ; FDE CIE offset - .g_long LFB1-. ; FDE initial location - .set L$set$3,LFE1-LFB1 - .g_long L$set$3 ; FDE address range - .byte 0x0 ; uleb128 0x0; Augmentation size - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$3,LCFI1-LCFI0 - .long L$set$3 - .byte 0xe ; DW_CFA_def_cfa_offset - .byte 176,1 ; uleb128 176 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$4,LCFI0-LFB1 - .long L$set$4 - .byte 0x11 ; DW_CFA_offset_extended_sf - .byte 0x41 ; uleb128 0x41 - .byte 0x7e ; sleb128 -2 - .align LOG2_GPR_BYTES - -LEFDE1: -.data - .align LOG2_GPR_BYTES -LDFCM0: -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi_closure_helper_DARWIN$stub: - .indirect_symbol _ffi_closure_helper_DARWIN - mflr r0 - bcl 20,31,LO$ffi_closure_helper_DARWIN - -LO$ffi_closure_helper_DARWIN: - mflr r11 - addis r11,r11,ha16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN) - mtlr r0 - lgu r12,lo16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi_closure_helper_DARWIN$lazy_ptr: - .indirect_symbol _ffi_closure_helper_DARWIN - .g_long dyld_stub_binding_helper - -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi64_struct_to_reg_form$stub: - .indirect_symbol _ffi64_struct_to_reg_form - mflr r0 - bcl 20,31,LO$ffi64_struct_to_reg_form - -LO$ffi64_struct_to_reg_form: - mflr r11 - addis r11,r11,ha16(L_ffi64_struct_to_reg_form$lazy_ptr - LO$ffi64_struct_to_reg_form) - mtlr r0 - lgu r12,lo16(L_ffi64_struct_to_reg_form$lazy_ptr - LO$ffi64_struct_to_reg_form)(r11) - mtctr r12 - bctr - -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi64_data_size$stub: - .indirect_symbol _ffi64_data_size - mflr r0 - bcl 20,31,LO$ffi64_data_size - -LO$ffi64_data_size: - mflr r11 - addis r11,r11,ha16(L_ffi64_data_size$lazy_ptr - LO$ffi64_data_size) - mtlr r0 - lgu r12,lo16(L_ffi64_data_size$lazy_ptr - LO$ffi64_data_size)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi64_struct_to_reg_form$lazy_ptr: - .indirect_symbol _ffi64_struct_to_reg_form - .g_long dyld_stub_binding_helper - -L_ffi64_data_size$lazy_ptr: - .indirect_symbol _ffi64_data_size - .g_long dyld_stub_binding_helper - -#endif // __ppc64__ diff --git a/Modules/_ctypes/libffi_osx/types.c b/Modules/_ctypes/libffi_osx/types.c deleted file mode 100644 index 44806aeeb75d..000000000000 --- a/Modules/_ctypes/libffi_osx/types.c +++ /dev/null @@ -1,115 +0,0 @@ -/* ----------------------------------------------------------------------- - types.c - Copyright (c) 1996, 1998 Red Hat, Inc. - - Predefined ffi_types needed by libffi. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -/* Type definitions */ -#define FFI_INTEGRAL_TYPEDEF(n, s, a, t) \ - ffi_type ffi_type_##n = { s, a, t, NULL } -#define FFI_AGGREGATE_TYPEDEF(n, e) \ - ffi_type ffi_type_##n = { 0, 0, FFI_TYPE_STRUCT, e } - -FFI_INTEGRAL_TYPEDEF(uint8, 1, 1, FFI_TYPE_UINT8); -FFI_INTEGRAL_TYPEDEF(sint8, 1, 1, FFI_TYPE_SINT8); -FFI_INTEGRAL_TYPEDEF(uint16, 2, 2, FFI_TYPE_UINT16); -FFI_INTEGRAL_TYPEDEF(sint16, 2, 2, FFI_TYPE_SINT16); -FFI_INTEGRAL_TYPEDEF(uint32, 4, 4, FFI_TYPE_UINT32); -FFI_INTEGRAL_TYPEDEF(sint32, 4, 4, FFI_TYPE_SINT32); -FFI_INTEGRAL_TYPEDEF(float, 4, 4, FFI_TYPE_FLOAT); - -/* Size and alignment are fake here. They must not be 0. */ -FFI_INTEGRAL_TYPEDEF(void, 1, 1, FFI_TYPE_VOID); - -#if defined ALPHA || defined SPARC64 || defined X86_64 || \ - defined S390X || defined IA64 || defined POWERPC64 -FFI_INTEGRAL_TYPEDEF(pointer, 8, 8, FFI_TYPE_POINTER); -#else -FFI_INTEGRAL_TYPEDEF(pointer, 4, 4, FFI_TYPE_POINTER); -#endif - -#if defined X86 || defined ARM || defined M68K || defined(X86_DARWIN) - -# ifdef X86_64 - FFI_INTEGRAL_TYPEDEF(uint64, 8, 8, FFI_TYPE_UINT64); - FFI_INTEGRAL_TYPEDEF(sint64, 8, 8, FFI_TYPE_SINT64); -# else - FFI_INTEGRAL_TYPEDEF(uint64, 8, 4, FFI_TYPE_UINT64); - FFI_INTEGRAL_TYPEDEF(sint64, 8, 4, FFI_TYPE_SINT64); -# endif - -#elif defined(POWERPC_DARWIN) -FFI_INTEGRAL_TYPEDEF(uint64, 8, 8, FFI_TYPE_UINT64); -FFI_INTEGRAL_TYPEDEF(sint64, 8, 8, FFI_TYPE_SINT64); -#elif defined SH -FFI_INTEGRAL_TYPEDEF(uint64, 8, 4, FFI_TYPE_UINT64); -FFI_INTEGRAL_TYPEDEF(sint64, 8, 4, FFI_TYPE_SINT64); -#else -FFI_INTEGRAL_TYPEDEF(uint64, 8, 8, FFI_TYPE_UINT64); -FFI_INTEGRAL_TYPEDEF(sint64, 8, 8, FFI_TYPE_SINT64); -#endif - -#if defined X86 || defined X86_WIN32 || defined M68K || defined(X86_DARWIN) - -# if defined X86_WIN32 || defined X86_64 - FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(double, 8, 4, FFI_TYPE_DOUBLE); -# endif - -# ifdef X86_DARWIN - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(longdouble, 12, 4, FFI_TYPE_LONGDOUBLE); -# endif - -#elif defined ARM || defined SH || defined POWERPC_AIX -FFI_INTEGRAL_TYPEDEF(double, 8, 4, FFI_TYPE_DOUBLE); -FFI_INTEGRAL_TYPEDEF(longdouble, 8, 4, FFI_TYPE_LONGDOUBLE); -#elif defined POWERPC_DARWIN -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); - -# if __GNUC__ >= 4 - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(longdouble, 8, 8, FFI_TYPE_LONGDOUBLE); -# endif - -#elif defined SPARC -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); - -# ifdef SPARC64 - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 8, FFI_TYPE_LONGDOUBLE); -# endif - -#elif defined X86_64 || defined POWERPC64 -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); -FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -#else -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); -FFI_INTEGRAL_TYPEDEF(longdouble, 8, 8, FFI_TYPE_LONGDOUBLE); -#endif \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/x86/darwin64.S b/Modules/_ctypes/libffi_osx/x86/darwin64.S deleted file mode 100644 index 1286d33f83e0..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/darwin64.S +++ /dev/null @@ -1,417 +0,0 @@ -/* ----------------------------------------------------------------------- - darwin64.S - Copyright (c) 2006 Free Software Foundation, Inc. - derived from unix64.S - - x86-64 Foreign Function Interface for Darwin. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#ifdef __x86_64__ -#define LIBFFI_ASM -#include <fficonfig.h> -#include <ffi.h> - - .file "darwin64.S" -.text - -/* ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags, - void *raddr, void (*fnaddr)()); - - Bit o trickiness here -- ARGS+BYTES is the base of the stack frame - for this function. This has been allocated by ffi_call. We also - deallocate some of the stack that has been alloca'd. */ - - .align 3 - .globl _ffi_call_unix64 - -_ffi_call_unix64: -LUW0: - movq (%rsp), %r10 /* Load return address. */ - movq %rdi, %r12 /* Save a copy of the register area. */ - leaq (%rdi, %rsi), %rax /* Find local stack base. */ - movq %rdx, (%rax) /* Save flags. */ - movq %rcx, 8(%rax) /* Save raddr. */ - movq %rbp, 16(%rax) /* Save old frame pointer. */ - movq %r10, 24(%rax) /* Relocate return address. */ - movq %rax, %rbp /* Finalize local stack frame. */ -LUW1: - /* movq %rdi, %r10 // Save a copy of the register area. */ - movq %r12, %r10 - movq %r8, %r11 /* Save a copy of the target fn. */ - movl %r9d, %eax /* Set number of SSE registers. */ - - /* Load up all argument registers. */ - movq (%r10), %rdi - movq 8(%r10), %rsi - movq 16(%r10), %rdx - movq 24(%r10), %rcx - movq 32(%r10), %r8 - movq 40(%r10), %r9 - testl %eax, %eax - jnz Lload_sse -Lret_from_load_sse: - - /* Deallocate the reg arg area. */ - leaq 176(%r10), %rsp - - /* Call the user function. */ - call *%r11 - - /* Deallocate stack arg area; local stack frame in redzone. */ - leaq 24(%rbp), %rsp - - movq 0(%rbp), %rcx /* Reload flags. */ - movq 8(%rbp), %rdi /* Reload raddr. */ - movq 16(%rbp), %rbp /* Reload old frame pointer. */ -LUW2: - - /* The first byte of the flags contains the FFI_TYPE. */ - movzbl %cl, %r10d - leaq Lstore_table(%rip), %r11 - movslq (%r11, %r10, 4), %r10 - addq %r11, %r10 - jmp *%r10 - -Lstore_table: - .long Lst_void-Lstore_table /* FFI_TYPE_VOID */ - .long Lst_sint32-Lstore_table /* FFI_TYPE_INT */ - .long Lst_float-Lstore_table /* FFI_TYPE_FLOAT */ - .long Lst_double-Lstore_table /* FFI_TYPE_DOUBLE */ - .long Lst_ldouble-Lstore_table /* FFI_TYPE_LONGDOUBLE */ - .long Lst_uint8-Lstore_table /* FFI_TYPE_UINT8 */ - .long Lst_sint8-Lstore_table /* FFI_TYPE_SINT8 */ - .long Lst_uint16-Lstore_table /* FFI_TYPE_UINT16 */ - .long Lst_sint16-Lstore_table /* FFI_TYPE_SINT16 */ - .long Lst_uint32-Lstore_table /* FFI_TYPE_UINT32 */ - .long Lst_sint32-Lstore_table /* FFI_TYPE_SINT32 */ - .long Lst_int64-Lstore_table /* FFI_TYPE_UINT64 */ - .long Lst_int64-Lstore_table /* FFI_TYPE_SINT64 */ - .long Lst_struct-Lstore_table /* FFI_TYPE_STRUCT */ - .long Lst_int64-Lstore_table /* FFI_TYPE_POINTER */ - - .text - .align 3 -Lst_void: - ret - .align 3 -Lst_uint8: - movzbq %al, %rax - movq %rax, (%rdi) - ret - .align 3 -Lst_sint8: - movsbq %al, %rax - movq %rax, (%rdi) - ret - .align 3 -Lst_uint16: - movzwq %ax, %rax - movq %rax, (%rdi) - .align 3 -Lst_sint16: - movswq %ax, %rax - movq %rax, (%rdi) - ret - .align 3 -Lst_uint32: - movl %eax, %eax - movq %rax, (%rdi) - .align 3 -Lst_sint32: - cltq - movq %rax, (%rdi) - ret - .align 3 -Lst_int64: - movq %rax, (%rdi) - ret - .align 3 -Lst_float: - movss %xmm0, (%rdi) - ret - .align 3 -Lst_double: - movsd %xmm0, (%rdi) - ret -Lst_ldouble: - fstpt (%rdi) - ret - .align 3 -Lst_struct: - leaq -20(%rsp), %rsi /* Scratch area in redzone. */ - - /* We have to locate the values now, and since we don't want to - write too much data into the user's return value, we spill the - value to a 16 byte scratch area first. Bits 8, 9, and 10 - control where the values are located. Only one of the three - bits will be set; see ffi_prep_cif_machdep for the pattern. */ - movd %xmm0, %r10 - movd %xmm1, %r11 - testl $0x100, %ecx - cmovnz %rax, %rdx - cmovnz %r10, %rax - testl $0x200, %ecx - cmovnz %r10, %rdx - testl $0x400, %ecx - cmovnz %r10, %rax - cmovnz %r11, %rdx - movq %rax, (%rsi) - movq %rdx, 8(%rsi) - - /* Bits 12-31 contain the true size of the structure. Copy from - the scratch area to the true destination. */ - shrl $12, %ecx - rep movsb - ret - - /* Many times we can avoid loading any SSE registers at all. - It's not worth an indirect jump to load the exact set of - SSE registers needed; zero or all is a good compromise. */ - .align 3 -LUW3: -Lload_sse: - movdqa 48(%r10), %xmm0 - movdqa 64(%r10), %xmm1 - movdqa 80(%r10), %xmm2 - movdqa 96(%r10), %xmm3 - movdqa 112(%r10), %xmm4 - movdqa 128(%r10), %xmm5 - movdqa 144(%r10), %xmm6 - movdqa 160(%r10), %xmm7 - jmp Lret_from_load_sse - -LUW4: - .align 3 - .globl _ffi_closure_unix64 - -_ffi_closure_unix64: -LUW5: - /* The carry flag is set by the trampoline iff SSE registers - are used. Don't clobber it before the branch instruction. */ - leaq -200(%rsp), %rsp -LUW6: - movq %rdi, (%rsp) - movq %rsi, 8(%rsp) - movq %rdx, 16(%rsp) - movq %rcx, 24(%rsp) - movq %r8, 32(%rsp) - movq %r9, 40(%rsp) - jc Lsave_sse -Lret_from_save_sse: - - movq %r10, %rdi - leaq 176(%rsp), %rsi - movq %rsp, %rdx - leaq 208(%rsp), %rcx - call _ffi_closure_unix64_inner - - /* Deallocate stack frame early; return value is now in redzone. */ - addq $200, %rsp -LUW7: - - /* The first byte of the return value contains the FFI_TYPE. */ - movzbl %al, %r10d - leaq Lload_table(%rip), %r11 - movslq (%r11, %r10, 4), %r10 - addq %r11, %r10 - jmp *%r10 - -Lload_table: - .long Lld_void-Lload_table /* FFI_TYPE_VOID */ - .long Lld_int32-Lload_table /* FFI_TYPE_INT */ - .long Lld_float-Lload_table /* FFI_TYPE_FLOAT */ - .long Lld_double-Lload_table /* FFI_TYPE_DOUBLE */ - .long Lld_ldouble-Lload_table /* FFI_TYPE_LONGDOUBLE */ - .long Lld_int8-Lload_table /* FFI_TYPE_UINT8 */ - .long Lld_int8-Lload_table /* FFI_TYPE_SINT8 */ - .long Lld_int16-Lload_table /* FFI_TYPE_UINT16 */ - .long Lld_int16-Lload_table /* FFI_TYPE_SINT16 */ - .long Lld_int32-Lload_table /* FFI_TYPE_UINT32 */ - .long Lld_int32-Lload_table /* FFI_TYPE_SINT32 */ - .long Lld_int64-Lload_table /* FFI_TYPE_UINT64 */ - .long Lld_int64-Lload_table /* FFI_TYPE_SINT64 */ - .long Lld_struct-Lload_table /* FFI_TYPE_STRUCT */ - .long Lld_int64-Lload_table /* FFI_TYPE_POINTER */ - - .text - .align 3 -Lld_void: - ret - .align 3 -Lld_int8: - movzbl -24(%rsp), %eax - ret - .align 3 -Lld_int16: - movzwl -24(%rsp), %eax - ret - .align 3 -Lld_int32: - movl -24(%rsp), %eax - ret - .align 3 -Lld_int64: - movq -24(%rsp), %rax - ret - .align 3 -Lld_float: - movss -24(%rsp), %xmm0 - ret - .align 3 -Lld_double: - movsd -24(%rsp), %xmm0 - ret - .align 3 -Lld_ldouble: - fldt -24(%rsp) - ret - .align 3 -Lld_struct: - /* There are four possibilities here, %rax/%rdx, %xmm0/%rax, - %rax/%xmm0, %xmm0/%xmm1. We collapse two by always loading - both rdx and xmm1 with the second word. For the remaining, - bit 8 set means xmm0 gets the second word, and bit 9 means - that rax gets the second word. */ - movq -24(%rsp), %rcx - movq -16(%rsp), %rdx - movq -16(%rsp), %xmm1 - testl $0x100, %eax - cmovnz %rdx, %rcx - movd %rcx, %xmm0 - testl $0x200, %eax - movq -24(%rsp), %rax - cmovnz %rdx, %rax - ret - - /* See the comment above Lload_sse; the same logic applies here. */ - .align 3 -LUW8: -Lsave_sse: - movdqa %xmm0, 48(%rsp) - movdqa %xmm1, 64(%rsp) - movdqa %xmm2, 80(%rsp) - movdqa %xmm3, 96(%rsp) - movdqa %xmm4, 112(%rsp) - movdqa %xmm5, 128(%rsp) - movdqa %xmm6, 144(%rsp) - movdqa %xmm7, 160(%rsp) - jmp Lret_from_save_sse - -LUW9: -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 /* CIE Length */ - .long L$set$0 -LSCIE1: - .long 0x0 /* CIE Identifier Tag */ - .byte 0x1 /* CIE Version */ - .ascii "zR\0" /* CIE Augmentation */ - .byte 0x1 /* uleb128 0x1; CIE Code Alignment Factor */ - .byte 0x78 /* sleb128 -8; CIE Data Alignment Factor */ - .byte 0x10 /* CIE RA Column */ - .byte 0x1 /* uleb128 0x1; Augmentation size */ - .byte 0x10 /* FDE Encoding (pcrel sdata4) */ - .byte 0xc /* DW_CFA_def_cfa, %rsp offset 8 */ - .byte 0x7 /* uleb128 0x7 */ - .byte 0x8 /* uleb128 0x8 */ - .byte 0x90 /* DW_CFA_offset, column 0x10 */ - .byte 0x1 - .align 3 -LECIE1: - .globl _ffi_call_unix64.eh -_ffi_call_unix64.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 /* FDE Length */ - .long L$set$1 -LASFDE1: - .long LASFDE1-EH_frame1 /* FDE CIE offset */ - .quad LUW0-. /* FDE initial location */ - .set L$set$2,LUW4-LUW0 /* FDE address range */ - .quad L$set$2 - .byte 0x0 /* Augmentation size */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$3,LUW1-LUW0 - .long L$set$3 - - /* New stack frame based off rbp. This is an itty bit of unwind - trickery in that the CFA *has* changed. There is no easy way - to describe it correctly on entry to the function. Fortunately, - it doesn't matter too much since at all points we can correctly - unwind back to ffi_call. Note that the location to which we - moved the return address is (the new) CFA-8, so from the - perspective of the unwind info, it hasn't moved. */ - .byte 0xc /* DW_CFA_def_cfa, %rbp offset 32 */ - .byte 0x6 - .byte 0x20 - .byte 0x80+6 /* DW_CFA_offset, %rbp offset 2*-8 */ - .byte 0x2 - .byte 0xa /* DW_CFA_remember_state */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$4,LUW2-LUW1 - .long L$set$4 - .byte 0xc /* DW_CFA_def_cfa, %rsp offset 8 */ - .byte 0x7 - .byte 0x8 - .byte 0xc0+6 /* DW_CFA_restore, %rbp */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$5,LUW3-LUW2 - .long L$set$5 - .byte 0xb /* DW_CFA_restore_state */ - - .align 3 -LEFDE1: - .globl _ffi_closure_unix64.eh -_ffi_closure_unix64.eh: -LSFDE3: - .set L$set$6,LEFDE3-LASFDE3 /* FDE Length */ - .long L$set$6 -LASFDE3: - .long LASFDE3-EH_frame1 /* FDE CIE offset */ - .quad LUW5-. /* FDE initial location */ - .set L$set$7,LUW9-LUW5 /* FDE address range */ - .quad L$set$7 - .byte 0x0 /* Augmentation size */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$8,LUW6-LUW5 - .long L$set$8 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 208,1 /* uleb128 208 */ - .byte 0xa /* DW_CFA_remember_state */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$9,LUW7-LUW6 - .long L$set$9 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$10,LUW8-LUW7 - .long L$set$10 - .byte 0xb /* DW_CFA_restore_state */ - - .align 3 -LEFDE3: - .subsections_via_symbols - -#endif /* __x86_64__ */ diff --git a/Modules/_ctypes/libffi_osx/x86/x86-darwin.S b/Modules/_ctypes/libffi_osx/x86/x86-darwin.S deleted file mode 100644 index 925a84131661..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/x86-darwin.S +++ /dev/null @@ -1,422 +0,0 @@ -#ifdef __i386__ -/* ----------------------------------------------------------------------- - darwin.S - Copyright (c) 1996, 1998, 2001, 2002, 2003 Red Hat, Inc. - - X86 Foreign Function Interface - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -/* - * This file is based on sysv.S and then hacked up by Ronald who hasn't done - * assembly programming in 8 years. - */ - -#ifndef __x86_64__ - -#define LIBFFI_ASM -#include <fficonfig.h> -#include <ffi.h> - -#ifdef PyObjC_STRICT_DEBUGGING - /* XXX: Debugging of stack alignment, to be removed */ -#define ASSERT_STACK_ALIGNED movdqa -16(%esp), %xmm0 -#else -#define ASSERT_STACK_ALIGNED -#endif - -.text - -.globl _ffi_prep_args - - .align 4 -.globl _ffi_call_SYSV - -_ffi_call_SYSV: -LFB1: - pushl %ebp -LCFI0: - movl %esp,%ebp -LCFI1: - subl $8,%esp - /* Make room for all of the new args. */ - movl 16(%ebp),%ecx - subl %ecx,%esp - - movl %esp,%eax - - /* Place all of the ffi_prep_args in position */ - subl $8,%esp - pushl 12(%ebp) - pushl %eax - call *8(%ebp) - - /* Return stack to previous state and call the function */ - addl $16,%esp - - call *28(%ebp) - - /* Remove the space we pushed for the args */ - movl 16(%ebp),%ecx - addl %ecx,%esp - - /* Load %ecx with the return type code */ - movl 20(%ebp),%ecx - - /* If the return value pointer is NULL, assume no return value. */ - cmpl $0,24(%ebp) - jne Lretint - - /* Even if there is no space for the return value, we are - obliged to handle floating-point values. */ - cmpl $FFI_TYPE_FLOAT,%ecx - jne Lnoretval - fstp %st(0) - - jmp Lepilogue - -Lretint: - cmpl $FFI_TYPE_INT,%ecx - jne Lretfloat - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - jmp Lepilogue - -Lretfloat: - cmpl $FFI_TYPE_FLOAT,%ecx - jne Lretdouble - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstps (%ecx) - jmp Lepilogue - -Lretdouble: - cmpl $FFI_TYPE_DOUBLE,%ecx - jne Lretlongdouble - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpl (%ecx) - jmp Lepilogue - -Lretlongdouble: - cmpl $FFI_TYPE_LONGDOUBLE,%ecx - jne Lretint64 - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpt (%ecx) - jmp Lepilogue - -Lretint64: - cmpl $FFI_TYPE_SINT64,%ecx - jne Lretstruct1b - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - movl %edx,4(%ecx) - jmp Lepilogue - -Lretstruct1b: - cmpl $FFI_TYPE_SINT8,%ecx - jne Lretstruct2b - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movb %al,0(%ecx) - jmp Lepilogue - -Lretstruct2b: - cmpl $FFI_TYPE_SINT16,%ecx - jne Lretstruct - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movw %ax,0(%ecx) - jmp Lepilogue - -Lretstruct: - cmpl $FFI_TYPE_STRUCT,%ecx - jne Lnoretval - /* Nothing to do! */ - addl $4,%esp - popl %ebp - ret - -Lnoretval: -Lepilogue: - addl $8,%esp - movl %ebp,%esp - popl %ebp - ret -LFE1: -.ffi_call_SYSV_end: - - .align 4 -FFI_HIDDEN (ffi_closure_SYSV) -.globl _ffi_closure_SYSV - -_ffi_closure_SYSV: -LFB2: - pushl %ebp -LCFI2: - movl %esp, %ebp -LCFI3: - subl $56, %esp - leal -40(%ebp), %edx - movl %edx, -12(%ebp) /* resp */ - leal 8(%ebp), %edx - movl %edx, 4(%esp) /* args = __builtin_dwarf_cfa () */ - leal -12(%ebp), %edx - movl %edx, (%esp) /* &resp */ - movl %ebx, 8(%esp) -LCFI7: - call L_ffi_closure_SYSV_inner$stub - movl 8(%esp), %ebx - movl -12(%ebp), %ecx - cmpl $FFI_TYPE_INT, %eax - je Lcls_retint - cmpl $FFI_TYPE_FLOAT, %eax - je Lcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je Lcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je Lcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je Lcls_retllong - cmpl $FFI_TYPE_UINT8, %eax - je Lcls_retstruct1 - cmpl $FFI_TYPE_SINT8, %eax - je Lcls_retstruct1 - cmpl $FFI_TYPE_UINT16, %eax - je Lcls_retstruct2 - cmpl $FFI_TYPE_SINT16, %eax - je Lcls_retstruct2 - cmpl $FFI_TYPE_STRUCT, %eax - je Lcls_retstruct -Lcls_epilogue: - movl %ebp, %esp - popl %ebp - ret -Lcls_retint: - movl (%ecx), %eax - jmp Lcls_epilogue -Lcls_retfloat: - flds (%ecx) - jmp Lcls_epilogue -Lcls_retdouble: - fldl (%ecx) - jmp Lcls_epilogue -Lcls_retldouble: - fldt (%ecx) - jmp Lcls_epilogue -Lcls_retllong: - movl (%ecx), %eax - movl 4(%ecx), %edx - jmp Lcls_epilogue -Lcls_retstruct1: - movsbl (%ecx), %eax - jmp Lcls_epilogue -Lcls_retstruct2: - movswl (%ecx), %eax - jmp Lcls_epilogue -Lcls_retstruct: - lea -8(%ebp),%esp - movl %ebp, %esp - popl %ebp - ret $4 -LFE2: - -#if !FFI_NO_RAW_API - -#define RAW_CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) & ~3) -#define RAW_CLOSURE_FUN_OFFSET (RAW_CLOSURE_CIF_OFFSET + 4) -#define RAW_CLOSURE_USER_DATA_OFFSET (RAW_CLOSURE_FUN_OFFSET + 4) -#define CIF_FLAGS_OFFSET 20 - - .align 4 -FFI_HIDDEN (ffi_closure_raw_SYSV) -.globl _ffi_closure_raw_SYSV - -_ffi_closure_raw_SYSV: -LFB3: - pushl %ebp -LCFI4: - movl %esp, %ebp -LCFI5: - pushl %esi -LCFI6: - subl $36, %esp - movl RAW_CLOSURE_CIF_OFFSET(%eax), %esi /* closure->cif */ - movl RAW_CLOSURE_USER_DATA_OFFSET(%eax), %edx /* closure->user_data */ - movl %edx, 12(%esp) /* user_data */ - leal 8(%ebp), %edx /* __builtin_dwarf_cfa () */ - movl %edx, 8(%esp) /* raw_args */ - leal -24(%ebp), %edx - movl %edx, 4(%esp) /* &res */ - movl %esi, (%esp) /* cif */ - call *RAW_CLOSURE_FUN_OFFSET(%eax) /* closure->fun */ - movl CIF_FLAGS_OFFSET(%esi), %eax /* rtype */ - cmpl $FFI_TYPE_INT, %eax - je Lrcls_retint - cmpl $FFI_TYPE_FLOAT, %eax - je Lrcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je Lrcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je Lrcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je Lrcls_retllong -Lrcls_epilogue: - addl $36, %esp - popl %esi - popl %ebp - ret -Lrcls_retint: - movl -24(%ebp), %eax - jmp Lrcls_epilogue -Lrcls_retfloat: - flds -24(%ebp) - jmp Lrcls_epilogue -Lrcls_retdouble: - fldl -24(%ebp) - jmp Lrcls_epilogue -Lrcls_retldouble: - fldt -24(%ebp) - jmp Lrcls_epilogue -Lrcls_retllong: - movl -24(%ebp), %eax - movl -20(%ebp), %edx - jmp Lrcls_epilogue -LFE3: -#endif - -.section __IMPORT,__jump_table,symbol_stubs,self_modifying_code+pure_instructions,5 -L_ffi_closure_SYSV_inner$stub: - .indirect_symbol _ffi_closure_SYSV_inner - hlt ; hlt ; hlt ; hlt ; hlt - - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 -LSCIE1: - .long 0x0 - .byte 0x1 - .ascii "zR\0" - .byte 0x1 - .byte 0x7c - .byte 0x8 - .byte 0x1 - .byte 0x10 - .byte 0xc - .byte 0x5 - .byte 0x4 - .byte 0x88 - .byte 0x1 - .align 2 -LECIE1: -.globl _ffi_call_SYSV.eh -_ffi_call_SYSV.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 -LASFDE1: - .long LASFDE1-EH_frame1 - .long LFB1-. - .set L$set$2,LFE1-LFB1 - .long L$set$2 - .byte 0x0 - .byte 0x4 - .set L$set$3,LCFI0-LFB1 - .long L$set$3 - .byte 0xe - .byte 0x8 - .byte 0x84 - .byte 0x2 - .byte 0x4 - .set L$set$4,LCFI1-LCFI0 - .long L$set$4 - .byte 0xd - .byte 0x4 - .align 2 -LEFDE1: -.globl _ffi_closure_SYSV.eh -_ffi_closure_SYSV.eh: -LSFDE2: - .set L$set$5,LEFDE2-LASFDE2 - .long L$set$5 -LASFDE2: - .long LASFDE2-EH_frame1 - .long LFB2-. - .set L$set$6,LFE2-LFB2 - .long L$set$6 - .byte 0x0 - .byte 0x4 - .set L$set$7,LCFI2-LFB2 - .long L$set$7 - .byte 0xe - .byte 0x8 - .byte 0x84 - .byte 0x2 - .byte 0x4 - .set L$set$8,LCFI3-LCFI2 - .long L$set$8 - .byte 0xd - .byte 0x4 - .align 2 -LEFDE2: - -#if !FFI_NO_RAW_API - -.globl _ffi_closure_raw_SYSV.eh -_ffi_closure_raw_SYSV.eh: -LSFDE3: - .set L$set$10,LEFDE3-LASFDE3 - .long L$set$10 -LASFDE3: - .long LASFDE3-EH_frame1 - .long LFB3-. - .set L$set$11,LFE3-LFB3 - .long L$set$11 - .byte 0x0 - .byte 0x4 - .set L$set$12,LCFI4-LFB3 - .long L$set$12 - .byte 0xe - .byte 0x8 - .byte 0x84 - .byte 0x2 - .byte 0x4 - .set L$set$13,LCFI5-LCFI4 - .long L$set$13 - .byte 0xd - .byte 0x4 - .byte 0x4 - .set L$set$14,LCFI6-LCFI5 - .long L$set$14 - .byte 0x85 - .byte 0x3 - .align 2 -LEFDE3: - -#endif - -#endif /* ifndef __x86_64__ */ - -#endif /* defined __i386__ */ diff --git a/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c b/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c deleted file mode 100644 index 8e7d01648802..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c +++ /dev/null @@ -1,737 +0,0 @@ -#ifdef __x86_64__ - -/* ----------------------------------------------------------------------- - x86-ffi64.c - Copyright (c) 2002 Bo Thorsen <bo at suse.de> - - x86-64 Foreign Function Interface - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdlib.h> -#include <stdarg.h> - -#define MAX_GPR_REGS 6 -#define MAX_SSE_REGS 8 - -typedef struct RegisterArgs { - /* Registers for argument passing. */ - UINT64 gpr[MAX_GPR_REGS]; - __int128_t sse[MAX_SSE_REGS]; -} RegisterArgs; - -extern void -ffi_call_unix64( - void* args, - unsigned long bytes, - unsigned flags, - void* raddr, - void (*fnaddr)(void), - unsigned ssecount); - -/* All reference to register classes here is identical to the code in - gcc/config/i386/i386.c. Do *not* change one without the other. */ - -/* Register class used for passing given 64bit part of the argument. - These represent classes as documented by the PS ABI, with the exception - of SSESF, SSEDF classes, that are basically SSE class, just gcc will - use SF or DFmode move instead of DImode to avoid reformating penalties. - - Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves - whenever possible (upper half does contain padding). */ -enum x86_64_reg_class -{ - X86_64_NO_CLASS, - X86_64_INTEGER_CLASS, - X86_64_INTEGERSI_CLASS, - X86_64_SSE_CLASS, - X86_64_SSESF_CLASS, - X86_64_SSEDF_CLASS, - X86_64_SSEUP_CLASS, - X86_64_X87_CLASS, - X86_64_X87UP_CLASS, - X86_64_COMPLEX_X87_CLASS, - X86_64_MEMORY_CLASS -}; - -#define MAX_CLASSES 4 -#define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS) - -/* x86-64 register passing implementation. See x86-64 ABI for details. Goal - of this code is to classify each 8bytes of incoming argument by the register - class and assign registers accordingly. */ - -/* Return the union class of CLASS1 and CLASS2. - See the x86-64 PS ABI for details. */ -static enum x86_64_reg_class -merge_classes( - enum x86_64_reg_class class1, - enum x86_64_reg_class class2) -{ - /* Rule #1: If both classes are equal, this is the resulting class. */ - if (class1 == class2) - return class1; - - /* Rule #2: If one of the classes is NO_CLASS, the resulting class is - the other class. */ - if (class1 == X86_64_NO_CLASS) - return class2; - - if (class2 == X86_64_NO_CLASS) - return class1; - - /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */ - if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS) - return X86_64_MEMORY_CLASS; - - /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */ - if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS) - || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS)) - return X86_64_INTEGERSI_CLASS; - - if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS - || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS) - return X86_64_INTEGER_CLASS; - - /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class, - MEMORY is used. */ - if (class1 == X86_64_X87_CLASS - || class1 == X86_64_X87UP_CLASS - || class1 == X86_64_COMPLEX_X87_CLASS - || class2 == X86_64_X87_CLASS - || class2 == X86_64_X87UP_CLASS - || class2 == X86_64_COMPLEX_X87_CLASS) - return X86_64_MEMORY_CLASS; - - /* Rule #6: Otherwise class SSE is used. */ - return X86_64_SSE_CLASS; -} - -/* Classify the argument of type TYPE and mode MODE. - CLASSES will be filled by the register class used to pass each word - of the operand. The number of words is returned. In case the parameter - should be passed in memory, 0 is returned. As a special case for zero - sized containers, classes[0] will be NO_CLASS and 1 is returned. - - See the x86-64 PS ABI for details. */ - -static int -classify_argument( - ffi_type* type, - enum x86_64_reg_class classes[], - size_t byte_offset) -{ - switch (type->type) - { - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - case FFI_TYPE_POINTER: -#if 0 - if (byte_offset + type->size <= 4) - classes[0] = X86_64_INTEGERSI_CLASS; - else - classes[0] = X86_64_INTEGER_CLASS; - - return 1; -#else - { - int size = byte_offset + type->size; - - if (size <= 4) - { - classes[0] = X86_64_INTEGERSI_CLASS; - return 1; - } - else if (size <= 8) - { - classes[0] = X86_64_INTEGER_CLASS; - return 1; - } - else if (size <= 12) - { - classes[0] = X86_64_INTEGER_CLASS; - classes[1] = X86_64_INTEGERSI_CLASS; - return 2; - } - else if (size <= 16) - { - classes[0] = classes[1] = X86_64_INTEGERSI_CLASS; - return 2; - } - else - FFI_ASSERT (0); - } -#endif - - case FFI_TYPE_FLOAT: - if (byte_offset == 0) - classes[0] = X86_64_SSESF_CLASS; - else - classes[0] = X86_64_SSE_CLASS; - - return 1; - - case FFI_TYPE_DOUBLE: - classes[0] = X86_64_SSEDF_CLASS; - return 1; - - case FFI_TYPE_LONGDOUBLE: - classes[0] = X86_64_X87_CLASS; - classes[1] = X86_64_X87UP_CLASS; - return 2; - - case FFI_TYPE_STRUCT: - { - ffi_type** ptr; - int i; - enum x86_64_reg_class subclasses[MAX_CLASSES]; - const int UNITS_PER_WORD = 8; - int words = - (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD; - - /* If the struct is larger than 16 bytes, pass it on the stack. */ - if (type->size > 16) - return 0; - - for (i = 0; i < words; i++) - classes[i] = X86_64_NO_CLASS; - - /* Merge the fields of structure. */ - for (ptr = type->elements; *ptr != NULL; ptr++) - { - int num, pos; - - byte_offset = ALIGN(byte_offset, (*ptr)->alignment); - - num = classify_argument(*ptr, subclasses, byte_offset % 8); - - if (num == 0) - return 0; - - pos = byte_offset / 8; - - for (i = 0; i < num; i++) - { - classes[i + pos] = - merge_classes(subclasses[i], classes[i + pos]); - } - - byte_offset += (*ptr)->size; - } - - if (words > 2) - { - /* When size > 16 bytes, if the first one isn't - X86_64_SSE_CLASS or any other ones aren't - X86_64_SSEUP_CLASS, everything should be passed in - memory. */ - if (classes[0] != X86_64_SSE_CLASS) - return 0; - - for (i = 1; i < words; i++) - if (classes[i] != X86_64_SSEUP_CLASS) - return 0; - } - - - /* Final merger cleanup. */ - for (i = 0; i < words; i++) - { - /* If one class is MEMORY, everything should be passed in - memory. */ - if (classes[i] == X86_64_MEMORY_CLASS) - return 0; - - /* The X86_64_SSEUP_CLASS should be always preceded by - X86_64_SSE_CLASS. */ - if (classes[i] == X86_64_SSEUP_CLASS - && classes[i - 1] != X86_64_SSE_CLASS - && classes[i - 1] != X86_64_SSEUP_CLASS) - { - FFI_ASSERT(i != 0); - classes[i] = X86_64_SSE_CLASS; - } - - /* X86_64_X87UP_CLASS should be preceded by X86_64_X87_CLASS. */ - if (classes[i] == X86_64_X87UP_CLASS - && classes[i - 1] != X86_64_X87_CLASS) - { - FFI_ASSERT(i != 0); - classes[i] = X86_64_SSE_CLASS; - } - } - - return words; - } - - default: - FFI_ASSERT(0); - } - - return 0; /* Never reached. */ -} - -/* Examine the argument and return set number of register required in each - class. Return zero if parameter should be passed in memory, otherwise - the number of registers. */ -static int -examine_argument( - ffi_type* type, - enum x86_64_reg_class classes[MAX_CLASSES], - _Bool in_return, - int* pngpr, - int* pnsse) -{ - int n = classify_argument(type, classes, 0); - int ngpr = 0; - int nsse = 0; - int i; - - if (n == 0) - return 0; - - for (i = 0; i < n; ++i) - { - switch (classes[i]) - { - case X86_64_INTEGER_CLASS: - case X86_64_INTEGERSI_CLASS: - ngpr++; - break; - - case X86_64_SSE_CLASS: - case X86_64_SSESF_CLASS: - case X86_64_SSEDF_CLASS: - nsse++; - break; - - case X86_64_NO_CLASS: - case X86_64_SSEUP_CLASS: - break; - - case X86_64_X87_CLASS: - case X86_64_X87UP_CLASS: - case X86_64_COMPLEX_X87_CLASS: - return in_return != 0; - - default: - abort(); - } - } - - *pngpr = ngpr; - *pnsse = nsse; - - return n; -} - -/* Perform machine dependent cif processing. */ -ffi_status -ffi_prep_cif_machdep( - ffi_cif* cif) -{ - int gprcount = 0; - int ssecount = 0; - int flags = cif->rtype->type; - int i, avn, n, ngpr, nsse; - enum x86_64_reg_class classes[MAX_CLASSES]; - size_t bytes; - - if (flags != FFI_TYPE_VOID) - { - n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); - - if (n == 0) - { - /* The return value is passed in memory. A pointer to that - memory is the first argument. Allocate a register for it. */ - gprcount++; - - /* We don't have to do anything in asm for the return. */ - flags = FFI_TYPE_VOID; - } - else if (flags == FFI_TYPE_STRUCT) - { - /* Mark which registers the result appears in. */ - _Bool sse0 = SSE_CLASS_P(classes[0]); - _Bool sse1 = n == 2 && SSE_CLASS_P(classes[1]); - - if (sse0 && !sse1) - flags |= 1 << 8; - else if (!sse0 && sse1) - flags |= 1 << 9; - else if (sse0 && sse1) - flags |= 1 << 10; - - /* Mark the true size of the structure. */ - flags |= cif->rtype->size << 12; - } - } - - /* Go over all arguments and determine the way they should be passed. - If it's in a register and there is space for it, let that be so. If - not, add it's size to the stack byte count. */ - for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++) - { - if (examine_argument(cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0 - || gprcount + ngpr > MAX_GPR_REGS - || ssecount + nsse > MAX_SSE_REGS) - { - long align = cif->arg_types[i]->alignment; - - if (align < 8) - align = 8; - - bytes = ALIGN(bytes, align); - bytes += cif->arg_types[i]->size; - } - else - { - gprcount += ngpr; - ssecount += nsse; - } - } - - if (ssecount) - flags |= 1 << 11; - - cif->flags = flags; - cif->bytes = bytes; - cif->bytes = ALIGN(bytes,8); - - return FFI_OK; -} - -void -ffi_call( - ffi_cif* cif, - void (*fn)(void), - void* rvalue, - void** avalue) -{ - enum x86_64_reg_class classes[MAX_CLASSES]; - char* stack; - char* argp; - ffi_type** arg_types; - int gprcount, ssecount, ngpr, nsse, i, avn; - _Bool ret_in_memory; - RegisterArgs* reg_args; - - /* Can't call 32-bit mode from 64-bit mode. */ - FFI_ASSERT(cif->abi == FFI_UNIX64); - - /* If the return value is a struct and we don't have a return value - address then we need to make one. Note the setting of flags to - VOID above in ffi_prep_cif_machdep. */ - ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT - && (cif->flags & 0xff) == FFI_TYPE_VOID); - - if (rvalue == NULL && ret_in_memory) - rvalue = alloca (cif->rtype->size); - - /* Allocate the space for the arguments, plus 4 words of temp space. */ - stack = alloca(sizeof(RegisterArgs) + cif->bytes + 4 * 8); - reg_args = (RegisterArgs*)stack; - argp = stack + sizeof(RegisterArgs); - - gprcount = ssecount = 0; - - /* If the return value is passed in memory, add the pointer as the - first integer argument. */ - if (ret_in_memory) - reg_args->gpr[gprcount++] = (long) rvalue; - - avn = cif->nargs; - arg_types = cif->arg_types; - - for (i = 0; i < avn; ++i) - { - size_t size = arg_types[i]->size; - int n; - - n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse); - - if (n == 0 - || gprcount + ngpr > MAX_GPR_REGS - || ssecount + nsse > MAX_SSE_REGS) - { - long align = arg_types[i]->alignment; - - /* Stack arguments are *always* at least 8 byte aligned. */ - if (align < 8) - align = 8; - - /* Pass this argument in memory. */ - argp = (void *) ALIGN (argp, align); - memcpy (argp, avalue[i], size); - argp += size; - } - else - { /* The argument is passed entirely in registers. */ - char *a = (char *) avalue[i]; - int j; - - for (j = 0; j < n; j++, a += 8, size -= 8) - { - switch (classes[j]) - { - case X86_64_INTEGER_CLASS: - case X86_64_INTEGERSI_CLASS: - reg_args->gpr[gprcount] = 0; - switch (arg_types[i]->type) { - case FFI_TYPE_SINT8: - { - int8_t shortval = *(int8_t*)a; - int64_t actval = (int64_t)shortval; - reg_args->gpr[gprcount] = actval; - /*memcpy (®_args->gpr[gprcount], &actval, 8);*/ - break; - } - - case FFI_TYPE_SINT16: - { - int16_t shortval = *(int16_t*)a; - int64_t actval = (int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - case FFI_TYPE_SINT32: - { - int32_t shortval = *(int32_t*)a; - int64_t actval = (int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - case FFI_TYPE_UINT8: - { - u_int8_t shortval = *(u_int8_t*)a; - u_int64_t actval = (u_int64_t)shortval; - /*memcpy (®_args->gpr[gprcount], &actval, 8);*/ - reg_args->gpr[gprcount] = actval; - break; - } - - case FFI_TYPE_UINT16: - { - u_int16_t shortval = *(u_int16_t*)a; - u_int64_t actval = (u_int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - case FFI_TYPE_UINT32: - { - u_int32_t shortval = *(u_int32_t*)a; - u_int64_t actval = (u_int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - default: - //memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8); - reg_args->gpr[gprcount] = *(int64_t*)a; - } - gprcount++; - break; - - case X86_64_SSE_CLASS: - case X86_64_SSEDF_CLASS: - reg_args->sse[ssecount++] = *(UINT64 *) a; - break; - - case X86_64_SSESF_CLASS: - reg_args->sse[ssecount++] = *(UINT32 *) a; - break; - - default: - abort(); - } - } - } - } - - ffi_call_unix64 (stack, cif->bytes + sizeof(RegisterArgs), - cif->flags, rvalue, fn, ssecount); -} - -extern void ffi_closure_unix64(void); - -ffi_status -ffi_prep_closure( - ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*, void*, void**, void*), - void* user_data) -{ - volatile unsigned short* tramp; - - if (cif->abi != FFI_UNIX64) - return FFI_BAD_ABI; - - tramp = (volatile unsigned short*)&closure->tramp[0]; - - tramp[0] = 0xbb49; /* mov <code>, %r11 */ - *(void* volatile*)&tramp[1] = ffi_closure_unix64; - tramp[5] = 0xba49; /* mov <data>, %r10 */ - *(void* volatile*)&tramp[6] = closure; - - /* Set the carry bit if the function uses any sse registers. - This is clc or stc, together with the first byte of the jmp. */ - tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8; - tramp[11] = 0xe3ff; /* jmp *%r11 */ - - closure->cif = cif; - closure->fun = fun; - closure->user_data = user_data; - - return FFI_OK; -} - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wmissing-prototypes" -int -ffi_closure_unix64_inner( - ffi_closure* closure, - void* rvalue, - RegisterArgs* reg_args, - char* argp) -#pragma clang diagnostic pop -{ - ffi_cif* cif = closure->cif; - void** avalue = alloca(cif->nargs * sizeof(void *)); - ffi_type** arg_types; - long i, avn; - int gprcount = 0; - int ssecount = 0; - int ngpr, nsse; - int ret; - - ret = cif->rtype->type; - - if (ret != FFI_TYPE_VOID) - { - enum x86_64_reg_class classes[MAX_CLASSES]; - int n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); - - if (n == 0) - { - /* The return value goes in memory. Arrange for the closure - return value to go directly back to the original caller. */ - rvalue = (void *) reg_args->gpr[gprcount++]; - - /* We don't have to do anything in asm for the return. */ - ret = FFI_TYPE_VOID; - } - else if (ret == FFI_TYPE_STRUCT && n == 2) - { - /* Mark which register the second word of the structure goes in. */ - _Bool sse0 = SSE_CLASS_P (classes[0]); - _Bool sse1 = SSE_CLASS_P (classes[1]); - - if (!sse0 && sse1) - ret |= 1 << 8; - else if (sse0 && !sse1) - ret |= 1 << 9; - } - } - - avn = cif->nargs; - arg_types = cif->arg_types; - - for (i = 0; i < avn; ++i) - { - enum x86_64_reg_class classes[MAX_CLASSES]; - int n; - - n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse); - - if (n == 0 - || gprcount + ngpr > MAX_GPR_REGS - || ssecount + nsse > MAX_SSE_REGS) - { - long align = arg_types[i]->alignment; - - /* Stack arguments are *always* at least 8 byte aligned. */ - if (align < 8) - align = 8; - - /* Pass this argument in memory. */ - argp = (void *) ALIGN (argp, align); - avalue[i] = argp; - argp += arg_types[i]->size; - } - -#if !defined(X86_DARWIN) - /* If the argument is in a single register, or two consecutive - registers, then we can use that address directly. */ - else if (n == 1 || (n == 2 && - SSE_CLASS_P (classes[0]) == SSE_CLASS_P (classes[1]))) - { - // The argument is in a single register. - if (SSE_CLASS_P (classes[0])) - { - avalue[i] = ®_args->sse[ssecount]; - ssecount += n; - } - else - { - avalue[i] = ®_args->gpr[gprcount]; - gprcount += n; - } - } -#endif - - /* Otherwise, allocate space to make them consecutive. */ - else - { - char *a = alloca (16); - int j; - - avalue[i] = a; - - for (j = 0; j < n; j++, a += 8) - { - if (SSE_CLASS_P (classes[j])) - memcpy (a, ®_args->sse[ssecount++], 8); - else - memcpy (a, ®_args->gpr[gprcount++], 8); - } - } - } - - /* Invoke the closure. */ - closure->fun (cif, rvalue, avalue, closure->user_data); - - /* Tell assembly how to perform return type promotions. */ - return ret; -} - -#endif /* __x86_64__ */ diff --git a/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c b/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c deleted file mode 100644 index 706ea0f51206..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c +++ /dev/null @@ -1,438 +0,0 @@ -#ifdef __i386__ -/* ----------------------------------------------------------------------- - ffi.c - Copyright (c) 1996, 1998, 1999, 2001 Red Hat, Inc. - Copyright (c) 2002 Ranjit Mathew - Copyright (c) 2002 Bo Thorsen - Copyright (c) 2002 Roger Sayle - - x86 Foreign Function Interface - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdlib.h> - -/* ffi_prep_args is called by the assembly routine once stack space - has been allocated for the function's arguments */ - -void ffi_prep_args(char *stack, extended_cif *ecif); - -void ffi_prep_args(char *stack, extended_cif *ecif) -{ - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - - argp = stack; - - if (ecif->cif->flags == FFI_TYPE_STRUCT) - { - *(void **) argp = ecif->rvalue; - argp += 4; - } - - p_argv = ecif->avalue; - - for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; - i != 0; - i--, p_arg++) - { - size_t z; - - /* Align if necessary */ - if ((sizeof(int) - 1) & (unsigned) argp) - argp = (char *) ALIGN(argp, sizeof(int)); - - z = (*p_arg)->size; - if (z < sizeof(int)) - { - z = sizeof(int); - switch ((*p_arg)->type) - { - case FFI_TYPE_SINT8: - *(signed int *) argp = (signed int)*(SINT8 *)(* p_argv); - break; - - case FFI_TYPE_UINT8: - *(unsigned int *) argp = (unsigned int)*(UINT8 *)(* p_argv); - break; - - case FFI_TYPE_SINT16: - *(signed int *) argp = (signed int)*(SINT16 *)(* p_argv); - break; - - case FFI_TYPE_UINT16: - *(unsigned int *) argp = (unsigned int)*(UINT16 *)(* p_argv); - break; - - case FFI_TYPE_SINT32: - *(signed int *) argp = (signed int)*(SINT32 *)(* p_argv); - break; - - case FFI_TYPE_UINT32: - *(unsigned int *) argp = (unsigned int)*(UINT32 *)(* p_argv); - break; - - case FFI_TYPE_STRUCT: - *(unsigned int *) argp = (unsigned int)*(UINT32 *)(* p_argv); - break; - - default: - FFI_ASSERT(0); - } - } - else - { - memcpy(argp, *p_argv, z); - } - p_argv++; - argp += z; - } - - return; -} - -/* Perform machine dependent cif processing */ -ffi_status ffi_prep_cif_machdep(ffi_cif *cif) -{ - /* Set the return type flag */ - switch (cif->rtype->type) - { - case FFI_TYPE_VOID: -#ifdef X86 - case FFI_TYPE_STRUCT: - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: -#endif - - case FFI_TYPE_SINT64: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: - cif->flags = (unsigned) cif->rtype->type; - break; - - case FFI_TYPE_UINT64: - cif->flags = FFI_TYPE_SINT64; - break; - -#ifndef X86 - case FFI_TYPE_STRUCT: - if (cif->rtype->size == 1) - { - cif->flags = FFI_TYPE_SINT8; /* same as char size */ - } - else if (cif->rtype->size == 2) - { - cif->flags = FFI_TYPE_SINT16; /* same as short size */ - } - else if (cif->rtype->size == 4) - { - cif->flags = FFI_TYPE_INT; /* same as int type */ - } - else if (cif->rtype->size == 8) - { - cif->flags = FFI_TYPE_SINT64; /* same as int64 type */ - } - else - { - cif->flags = FFI_TYPE_STRUCT; - } - break; -#endif - - default: - cif->flags = FFI_TYPE_INT; - break; - } - -#ifdef X86_DARWIN - cif->bytes = (cif->bytes + 15) & ~0xF; -#endif - - return FFI_OK; -} - -extern void ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, - unsigned, unsigned, unsigned *, void (*fn)()); - -#ifdef X86_WIN32 -extern void ffi_call_STDCALL(void (*)(char *, extended_cif *), extended_cif *, - unsigned, unsigned, unsigned *, void (*fn)()); - -#endif /* X86_WIN32 */ - -void ffi_call(ffi_cif *cif, void (*fn)(), void *rvalue, void **avalue) -{ - extended_cif ecif; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ - - if ((rvalue == NULL) && - (cif->flags == FFI_TYPE_STRUCT)) - { - ecif.rvalue = alloca(cif->rtype->size); - } - else - ecif.rvalue = rvalue; - - - switch (cif->abi) - { - case FFI_SYSV: - ffi_call_SYSV(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, - fn); - break; -#ifdef X86_WIN32 - case FFI_STDCALL: - ffi_call_STDCALL(ffi_prep_args, &ecif, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; -#endif /* X86_WIN32 */ - default: - FFI_ASSERT(0); - break; - } -} - - -/** private members **/ - -static void ffi_prep_incoming_args_SYSV (char *stack, void **ret, - void** args, ffi_cif* cif); -void FFI_HIDDEN ffi_closure_SYSV (ffi_closure *) -__attribute__ ((regparm(1))); -unsigned int FFI_HIDDEN ffi_closure_SYSV_inner (ffi_closure *, void **, void *) -__attribute__ ((regparm(1))); -void FFI_HIDDEN ffi_closure_raw_SYSV (ffi_raw_closure *) -__attribute__ ((regparm(1))); - -/* This function is jumped to by the trampoline */ - -unsigned int FFI_HIDDEN -ffi_closure_SYSV_inner (closure, respp, args) -ffi_closure *closure; -void **respp; -void *args; -{ - // our various things... - ffi_cif *cif; - void **arg_area; - - cif = closure->cif; - arg_area = (void**) alloca (cif->nargs * sizeof (void*)); - - /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding - * value on the stack; and if the function returns - * a structure, it will re-set RESP to point to the - * structure return address. */ - - ffi_prep_incoming_args_SYSV(args, respp, arg_area, cif); - - (closure->fun) (cif, *respp, arg_area, closure->user_data); - - return cif->flags; -} - -static void -ffi_prep_incoming_args_SYSV(char *stack, void **rvalue, void **avalue, - ffi_cif *cif) -{ - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - - argp = stack; - - if ( cif->flags == FFI_TYPE_STRUCT ) { - *rvalue = *(void **) argp; - argp += 4; - } - - p_argv = avalue; - - for (i = cif->nargs, p_arg = cif->arg_types; (i != 0); i--, p_arg++) - { - size_t z; - - /* Align if necessary */ - if ((sizeof(int) - 1) & (unsigned) argp) { - argp = (char *) ALIGN(argp, sizeof(int)); - } - - z = (*p_arg)->size; - - /* because we're little endian, this is what it turns into. */ - - *p_argv = (void*) argp; - - p_argv++; - argp += z; - } - - return; -} - -/* How to make a trampoline. Derived from gcc/config/i386/i386.c. */ - -#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX) \ -({ unsigned char *__tramp = (unsigned char*)(TRAMP); \ -unsigned int __fun = (unsigned int)(FUN); \ -unsigned int __ctx = (unsigned int)(CTX); \ -unsigned int __dis = __fun - (__ctx + FFI_TRAMPOLINE_SIZE); \ -*(unsigned char*) &__tramp[0] = 0xb8; \ -*(unsigned int*) &__tramp[1] = __ctx; /* movl __ctx, %eax */ \ -*(unsigned char *) &__tramp[5] = 0xe9; \ -*(unsigned int*) &__tramp[6] = __dis; /* jmp __fun */ \ -}) - - -/* the cif must already be prep'ed */ -ffi_status -ffi_prep_closure (ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,void**,void*), - void *user_data) -{ - if (cif->abi != FFI_SYSV) - return FFI_BAD_ABI; - - FFI_INIT_TRAMPOLINE (&closure->tramp[0], \ - &ffi_closure_SYSV, \ - (void*)closure); - - closure->cif = cif; - closure->user_data = user_data; - closure->fun = fun; - - return FFI_OK; -} - -/* ------- Native raw API support -------------------------------- */ - -#if !FFI_NO_RAW_API - -ffi_status -ffi_prep_raw_closure_loc (ffi_raw_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,ffi_raw*,void*), - void *user_data, - void *codeloc) -{ - int i; - - FFI_ASSERT (cif->abi == FFI_SYSV); - - // we currently don't support certain kinds of arguments for raw - // closures. This should be implemented by a separate assembly language - // routine, since it would require argument processing, something we - // don't do now for performance. - - for (i = cif->nargs-1; i >= 0; i--) - { - FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_STRUCT); - FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_LONGDOUBLE); - } - - - FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_raw_SYSV, - codeloc); - - closure->cif = cif; - closure->user_data = user_data; - closure->fun = fun; - - return FFI_OK; -} - -static void -ffi_prep_args_raw(char *stack, extended_cif *ecif) -{ - memcpy (stack, ecif->avalue, ecif->cif->bytes); -} - -/* we borrow this routine from libffi (it must be changed, though, to - * actually call the function passed in the first argument. as of - * libffi-1.20, this is not the case.) - */ - -extern void -ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)()); - -#ifdef X86_WIN32 -extern void -ffi_call_STDCALL(void (*)(char *, extended_cif *), extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)()); -#endif /* X86_WIN32 */ - -void -ffi_raw_call(ffi_cif *cif, void (*fn)(), void *rvalue, ffi_raw *fake_avalue) -{ - extended_cif ecif; - void **avalue = (void **)fake_avalue; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ - - if ((rvalue == NULL) && - (cif->rtype->type == FFI_TYPE_STRUCT)) - { - ecif.rvalue = alloca(cif->rtype->size); - } - else - ecif.rvalue = rvalue; - - - switch (cif->abi) - { - case FFI_SYSV: - ffi_call_SYSV(ffi_prep_args_raw, &ecif, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; -#ifdef X86_WIN32 - case FFI_STDCALL: - ffi_call_STDCALL(ffi_prep_args_raw, &ecif, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; -#endif /* X86_WIN32 */ - default: - FFI_ASSERT(0); - break; - } -} - -#endif -#endif // __i386__ diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 769e27432f28..ac1721c061f0 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -52,7 +52,6 @@ def clean_lines(text): # OSX #Modules/_ctypes/darwin/*.c -#Modules/_ctypes/libffi_osx/*.c Modules/_scproxy.c # SystemConfiguration/SystemConfiguration.h # Windows diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index a324eafc52b2..6dcf61206619 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -13,11 +13,11 @@ # Excluded directories which are copies of external libraries: # don't check their coding style -EXCLUDE_DIRS = [os.path.join('Modules', '_ctypes', 'libffi_osx'), - os.path.join('Modules', '_ctypes', 'libffi_msvc'), - os.path.join('Modules', '_decimal', 'libmpdec'), - os.path.join('Modules', 'expat'), - os.path.join('Modules', 'zlib')] +EXCLUDE_DIRS = [ + os.path.join('Modules', '_decimal', 'libmpdec'), + os.path.join('Modules', 'expat'), + os.path.join('Modules', 'zlib'), + ] SRCDIR = sysconfig.get_config_var('srcdir') From webhook-mailer at python.org Wed Dec 28 14:43:25 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Wed, 28 Dec 2022 19:43:25 -0000 Subject: [Python-checkins] gh-94172: Update docs for params removed in 3.12 (#100431) Message-ID: <mailman.3121.1672256606.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9eca7235af84bb4cc29a54a16af8730b01878722 commit: 9eca7235af84bb4cc29a54a16af8730b01878722 branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-28T14:43:19-05:00 summary: gh-94172: Update docs for params removed in 3.12 (#100431) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: M Doc/library/ftplib.rst M Doc/library/http.client.rst M Doc/library/imaplib.rst M Doc/library/poplib.rst M Doc/library/smtplib.rst M Doc/whatsnew/3.12.rst M Misc/NEWS.d/3.12.0a2.rst diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index e5ba9eef4074..47054812f9f5 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -85,7 +85,8 @@ The module defines the following items: The *encoding* parameter was added, and the default was changed from Latin-1 to UTF-8 to follow :rfc:`2640`. -.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None, *, encoding='utf-8') +.. class:: FTP_TLS(host='', user='', passwd='', acct='', *, context=None, + timeout=None, source_address=None, encoding='utf-8') A :class:`FTP` subclass which adds TLS support to FTP as described in :rfc:`4217`. @@ -96,10 +97,6 @@ The module defines the following items: options, certificates and private keys into a single (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *context* -- they - can point to PEM-formatted private key and certificate chain files - (respectively) for the SSL connection. - .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -111,7 +108,6 @@ The module defines the following items: :data:`ssl.HAS_SNI`). .. deprecated:: 3.6 - *keyfile* and *certfile* are deprecated in favor of *context*. Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let :func:`ssl.create_default_context` select the system's trusted CA @@ -123,6 +119,9 @@ The module defines the following items: The *encoding* parameter was added, and the default was changed from Latin-1 to UTF-8 to follow :rfc:`2640`. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + Here's a sample session using the :class:`FTP_TLS` class:: >>> ftps = FTP_TLS('ftp.pureftpd.org') diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 06f92510a5e4..48582219695b 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -20,7 +20,7 @@ HTTPS protocols. It is normally not used directly --- the module .. seealso:: - The `Requests package <https://requests.readthedocs.io/en/master/>`_ + The `Requests package <https://requests.readthedocs.io/en/latest/>`_ is recommended for a higher-level HTTP client interface. .. note:: @@ -67,10 +67,9 @@ The module provides the following classes: *blocksize* parameter was added. -.. class:: HTTPSConnection(host, port=None, key_file=None, \ - cert_file=None[, timeout], \ - source_address=None, *, context=None, \ - check_hostname=None, blocksize=8192) +.. class:: HTTPSConnection(host, port=None, *[, timeout], \ + source_address=None, context=None, \ + blocksize=8192) A subclass of :class:`HTTPConnection` that uses SSL for communication with secure servers. Default port is ``443``. If *context* is specified, it @@ -96,6 +95,16 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. + .. deprecated:: 3.6 + *key_file* and *cert_file* are deprecated in favor of *context*. + Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let + :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. + + The *check_hostname* parameter is also deprecated; the + :attr:`ssl.SSLContext.check_hostname` attribute of *context* should + be used instead. + .. versionchanged:: 3.8 This class now enables TLS 1.3 :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or @@ -106,16 +115,9 @@ The module provides the following classes: ``http/1.1`` when no *context* is given. Custom *context* should set ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. - .. deprecated:: 3.6 - - *key_file* and *cert_file* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - The *check_hostname* parameter is also deprecated; the - :attr:`ssl.SSLContext.check_hostname` attribute of *context* should - be used instead. + .. versionchanged:: 3.12 + The deprecated *key_file*, *cert_file* and *check_hostname* parameters + have been removed. .. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) @@ -344,11 +346,11 @@ HTTPConnection Objects Set the host and the port for HTTP Connect Tunnelling. This allows running the connection through a proxy server. - The host and port arguments specify the endpoint of the tunneled connection + The *host* and *port* arguments specify the endpoint of the tunneled connection (i.e. the address included in the CONNECT request, *not* the address of the proxy server). - The headers argument should be a mapping of extra HTTP headers to send with + The *headers* argument should be a mapping of extra HTTP headers to send with the CONNECT request. For example, to tunnel through a HTTPS proxy server running locally on port diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 8c28fce99ff9..f1344ae9bb0a 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -84,8 +84,8 @@ Three exceptions are defined as attributes of the :class:`IMAP4` class: There's also a subclass for secure connections: -.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, keyfile=None, \ - certfile=None, ssl_context=None, timeout=None) +.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, *, ssl_context=None, \ + timeout=None) This is a subclass derived from :class:`IMAP4` that connects over an SSL encrypted socket (to use this class you need a socket module that was compiled @@ -96,12 +96,6 @@ There's also a subclass for secure connections: (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *ssl_context* - they - can point to PEM-formatted private key and certificate chain files for - the SSL connection. Note that the *keyfile*/*certfile* parameters are - mutually exclusive with *ssl_context*, a :class:`ValueError` is raised - if *keyfile*/*certfile* is provided along with *ssl_context*. - The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. If timeout is not given or is None, the global default socket timeout is used. @@ -124,6 +118,9 @@ There's also a subclass for secure connections: .. versionchanged:: 3.9 The optional *timeout* parameter was added. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + The second subclass allows for connections created by a child process: @@ -564,7 +561,7 @@ An :class:`IMAP4` instance has the following methods: ``search``, the searching *charset* argument is mandatory. There is also a ``uid thread`` command which corresponds to ``thread`` the way that ``uid search`` corresponds to ``search``. The ``thread`` command first searches the - mailbox for messages that match the given searching criteria using the charset + mailbox for messages that match the given searching criteria using the *charset* argument for the interpretation of strings in the searching criteria. It then returns the matching messages threaded according to the specified threading algorithm. diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index e22a2e1455e7..d8618ce9b60b 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -53,7 +53,7 @@ The :mod:`poplib` module provides two classes: If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket. -.. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) +.. class:: POP3_SSL(host, port=POP3_SSL_PORT, *, timeout=None, context=None) This is a subclass of :class:`POP3` that connects to the server over an SSL encrypted socket. If *port* is not specified, 995, the standard POP3-over-SSL @@ -63,10 +63,6 @@ The :mod:`poplib` module provides two classes: single (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *context* - they can - point to PEM-formatted private key and certificate chain files, - respectively, for the SSL connection. - .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL .. audit-event:: poplib.putline self,line poplib.POP3_SSL @@ -94,6 +90,9 @@ The :mod:`poplib` module provides two classes: If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + One exception is defined as an attribute of the :mod:`poplib` module: diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 4e50ad1568ed..2539c3d38832 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -66,18 +66,17 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). Support for the :keyword:`with` statement was added. .. versionchanged:: 3.3 - source_address argument was added. + *source_address* argument was added. .. versionadded:: 3.5 The SMTPUTF8 extension (:rfc:`6531`) is now supported. .. versionchanged:: 3.9 If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket + :class:`ValueError` to prevent the creation of a non-blocking socket. -.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \ - certfile=None [, timeout], context=None, \ - source_address=None) +.. class:: SMTP_SSL(host='', port=0, local_hostname=None, * [, timeout], \ + context=None, source_address=None) An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is @@ -90,15 +89,11 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). aspects of the secure connection. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *context*, and can - point to a PEM formatted private key and certificate chain file for the - SSL connection. - .. versionchanged:: 3.3 *context* was added. .. versionchanged:: 3.3 - source_address argument was added. + The *source_address* argument was added. .. versionchanged:: 3.4 The class now supports hostname check with @@ -116,13 +111,16 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + .. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, \ source_address=None[, timeout]) The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our :meth:`connect` method must support that as well as a regular host:port - server. The optional arguments local_hostname and source_address have the + server. The optional arguments *local_hostname* and *source_address* have the same meaning as they do in the :class:`SMTP` class. To specify a Unix socket, you must use an absolute path for *host*, starting with a '/'. @@ -360,7 +358,7 @@ An :class:`SMTP` instance has the following methods: be used as argument to the ``AUTH`` command; the valid values are those listed in the ``auth`` element of :attr:`esmtp_features`. - *authobject* must be a callable object taking an optional single argument: + *authobject* must be a callable object taking an optional single argument:: data = authobject(challenge=None) @@ -393,7 +391,7 @@ An :class:`SMTP` instance has the following methods: .. versionadded:: 3.5 -.. method:: SMTP.starttls(keyfile=None, certfile=None, context=None) +.. method:: SMTP.starttls(*, context=None) Put the SMTP connection in TLS (Transport Layer Security) mode. All SMTP commands that follow will be encrypted. You should then call :meth:`ehlo` @@ -416,6 +414,9 @@ An :class:`SMTP` instance has the following methods: :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 69f97debd694..2f50ece4dab3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -631,9 +631,11 @@ Removed <https://github.com/sphinx-contrib/sphinx-lint>`_. (Contributed by Julien Palard in :gh:`98179`.) -* Remove the *keyfile*, *certfile* and *check_hostname* parameters, deprecated - since Python 3.6, in modules: :mod:`ftplib`, :mod:`http.client`, - :mod:`imaplib`, :mod:`poplib` and :mod:`smtplib`. Use the *context* parameter +* Remove the *keyfile* and *certfile* parameters from the + :mod:`ftplib`, :mod:`imaplib`, :mod:`poplib` and :mod:`smtplib` modules, + and the *key_file*, *cert_file* and *check_hostname* parameters from the + :mod:`http.client` module, + all deprecated since Python 3.6. Use the *context* parameter (*ssl_context* in :mod:`imaplib`) instead. (Contributed by Victor Stinner in :gh:`94172`.) diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 4029856ee332..318f3f71f115 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -725,9 +725,11 @@ Fix handling of ``bytes`` :term:`path-like objects <path-like object>` in .. nonce: AXE2IZ .. section: Library -Remove the *keyfile*, *certfile* and *check_hostname* parameters, deprecated -since Python 3.6, in modules: :mod:`ftplib`, :mod:`http.client`, -:mod:`imaplib`, :mod:`poplib` and :mod:`smtplib`. Use the *context* +Remove the *keyfile* and *certfile* parameters from the +:mod:`ftplib`, :mod:`imaplib`, :mod:`poplib` and :mod:`smtplib` modules, +and the *key_file*, *cert_file* and *check_hostname* parameters from the +:mod:`http.client` module, +all deprecated since Python 3.6. Use the *context* parameter (*ssl_context* in :mod:`imaplib`) instead. Patch by Victor Stinner. From webhook-mailer at python.org Wed Dec 28 15:29:33 2022 From: webhook-mailer at python.org (rhettinger) Date: Wed, 28 Dec 2022 20:29:33 -0000 Subject: [Python-checkins] Restore early-out to factor(). Strengthen tests. (GH-100591) Message-ID: <mailman.3122.1672259374.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c4c5790120beabed83ce5855f18d209ab8324434 commit: c4c5790120beabed83ce5855f18d209ab8324434 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-28T12:29:27-08:00 summary: Restore early-out to factor(). Strengthen tests. (GH-100591) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index f1277dfdbdc6..5d6efabcef60 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -901,12 +901,14 @@ which incur interpreter overhead. "Prime factors of n." # factor(99) --> 3 3 11 for prime in sieve(math.isqrt(n) + 1): - while n >= prime: + while True: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient + if n == 1: + return if n >= 2: yield n @@ -1286,13 +1288,21 @@ which incur interpreter overhead. [3, 3] >>> list(factor(10)) [2, 5] - >>> list(factor(999953*999983)) + >>> list(factor(128_884_753_939)) # large prime + [128884753939] + >>> list(factor(999953 * 999983)) # large semiprime [999953, 999983] - >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(6 ** 20)) == [2] * 20 + [3] * 20 # large power + True + >>> list(factor(909_909_090_909)) # large multiterm composite + [3, 3, 7, 13, 13, 751, 113797] + >>> math.prod([3, 3, 7, 13, 13, 751, 113797]) + 909909090909 + >>> all(math.prod(factor(n)) == n for n in range(1, 2_000)) True - >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(2_000)) True - >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 15:38:03 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 20:38:03 -0000 Subject: [Python-checkins] Restore early-out to factor(). Strengthen tests. (GH-100591) Message-ID: <mailman.3123.1672259884.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9120450b253a718ae295d7e86b5087705c177eae commit: 9120450b253a718ae295d7e86b5087705c177eae branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T12:37:58-08:00 summary: Restore early-out to factor(). Strengthen tests. (GH-100591) (cherry picked from commit c4c5790120beabed83ce5855f18d209ab8324434) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 221863cbbcc5..93bc1f792f54 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -862,12 +862,14 @@ which incur interpreter overhead. "Prime factors of n." # factor(99) --> 3 3 11 for prime in sieve(math.isqrt(n) + 1): - while n >= prime: + while True: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient + if n == 1: + return if n >= 2: yield n @@ -1256,13 +1258,21 @@ which incur interpreter overhead. [3, 3] >>> list(factor(10)) [2, 5] - >>> list(factor(999953*999983)) + >>> list(factor(128_884_753_939)) # large prime + [128884753939] + >>> list(factor(999953 * 999983)) # large semiprime [999953, 999983] - >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(6 ** 20)) == [2] * 20 + [3] * 20 # large power + True + >>> list(factor(909_909_090_909)) # large multiterm composite + [3, 3, 7, 13, 13, 751, 113797] + >>> math.prod([3, 3, 7, 13, 13, 751, 113797]) + 909909090909 + >>> all(math.prod(factor(n)) == n for n in range(1, 2_000)) True - >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(2_000)) True - >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 16:24:59 2022 From: webhook-mailer at python.org (brandtbucher) Date: Wed, 28 Dec 2022 21:24:59 -0000 Subject: [Python-checkins] GH-100101: Clarify documentation of zip's strict option (GH-100103) Message-ID: <mailman.3124.1672262700.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cf1c09818032df3080c2cd9e7edb5f657213dc83 commit: cf1c09818032df3080c2cd9e7edb5f657213dc83 branch: main author: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> committer: brandtbucher <brandtbucher at gmail.com> date: 2022-12-28T13:24:50-08:00 summary: GH-100101: Clarify documentation of zip's strict option (GH-100103) files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2e988257d5d3..f00a072e6f41 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1923,14 +1923,24 @@ are always available. They are listed here in alphabetical order. >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) [('a', 1), ('b', 2), ('c', 3)] - Unlike the default behavior, it checks that the lengths of iterables are - identical, raising a :exc:`ValueError` if they aren't: - - >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True)) + Unlike the default behavior, it raises a :exc:`ValueError` if one iterable + is exhausted before the others: + + >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP + ... print(item) + ... + (0, 'fee') + (1, 'fi') + (2, 'fo') Traceback (most recent call last): ... ValueError: zip() argument 2 is longer than argument 1 + .. + This doctest is disabled because doctest does not support capturing + output and exceptions in the same code unit. + https://github.com/python/cpython/issues/65382 + Without the ``strict=True`` argument, any bug that results in iterables of different lengths will be silenced, possibly manifesting as a hard-to-find bug in another part of the program. From webhook-mailer at python.org Wed Dec 28 16:31:49 2022 From: webhook-mailer at python.org (jaraco) Date: Wed, 28 Dec 2022 21:31:49 -0000 Subject: [Python-checkins] gh-100585: Fixed a bug where importlib.resources.as_file was leaving file pointers open (GH-100586) Message-ID: <mailman.3125.1672263110.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f10f503b24a35a43910a632ee50c7568bedd6664 commit: f10f503b24a35a43910a632ee50c7568bedd6664 branch: main author: Samet YASLAN <sametyaslan at gmail.com> committer: jaraco <jaraco at jaraco.com> date: 2022-12-28T16:31:43-05:00 summary: gh-100585: Fixed a bug where importlib.resources.as_file was leaving file pointers open (GH-100586) * gh-100585: Fixed open fp bug in the imporlib module * Added news for gh-100585 files: A Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst M Lib/importlib/resources/_common.py diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index 8c0be7ee547e..92a37e2c12b8 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -155,5 +155,5 @@ def _write_contents(target, source): for item in source.iterdir(): _write_contents(child, item) else: - child.open('wb').write(source.read_bytes()) + child.write_bytes(source.read_bytes()) return child diff --git a/Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst b/Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst new file mode 100644 index 000000000000..0bf33b8bdd02 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst @@ -0,0 +1 @@ +Fixed a bug where importlib.resources.as_file was leaving file pointers open From webhook-mailer at python.org Wed Dec 28 16:32:24 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 21:32:24 -0000 Subject: [Python-checkins] GH-100101: Clarify documentation of zip's strict option (GH-100103) Message-ID: <mailman.3126.1672263145.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ac31120de7824581656de7734cbaa7333d555067 commit: ac31120de7824581656de7734cbaa7333d555067 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T13:32:19-08:00 summary: GH-100101: Clarify documentation of zip's strict option (GH-100103) (cherry picked from commit cf1c09818032df3080c2cd9e7edb5f657213dc83) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 304716194d22..c71ebd235cf3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1918,14 +1918,24 @@ are always available. They are listed here in alphabetical order. >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) [('a', 1), ('b', 2), ('c', 3)] - Unlike the default behavior, it checks that the lengths of iterables are - identical, raising a :exc:`ValueError` if they aren't: - - >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True)) + Unlike the default behavior, it raises a :exc:`ValueError` if one iterable + is exhausted before the others: + + >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP + ... print(item) + ... + (0, 'fee') + (1, 'fi') + (2, 'fo') Traceback (most recent call last): ... ValueError: zip() argument 2 is longer than argument 1 + .. + This doctest is disabled because doctest does not support capturing + output and exceptions in the same code unit. + https://github.com/python/cpython/issues/65382 + Without the ``strict=True`` argument, any bug that results in iterables of different lengths will be silenced, possibly manifesting as a hard-to-find bug in another part of the program. From webhook-mailer at python.org Wed Dec 28 16:32:50 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 21:32:50 -0000 Subject: [Python-checkins] GH-100101: Clarify documentation of zip's strict option (GH-100103) Message-ID: <mailman.3127.1672263171.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4217fafeac87d31ad7eb40ca0ec2bb9309333211 commit: 4217fafeac87d31ad7eb40ca0ec2bb9309333211 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T13:32:44-08:00 summary: GH-100101: Clarify documentation of zip's strict option (GH-100103) (cherry picked from commit cf1c09818032df3080c2cd9e7edb5f657213dc83) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 46e77fdb4155..5400feb35567 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1894,14 +1894,24 @@ are always available. They are listed here in alphabetical order. >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) [('a', 1), ('b', 2), ('c', 3)] - Unlike the default behavior, it checks that the lengths of iterables are - identical, raising a :exc:`ValueError` if they aren't: - - >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True)) + Unlike the default behavior, it raises a :exc:`ValueError` if one iterable + is exhausted before the others: + + >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP + ... print(item) + ... + (0, 'fee') + (1, 'fi') + (2, 'fo') Traceback (most recent call last): ... ValueError: zip() argument 2 is longer than argument 1 + .. + This doctest is disabled because doctest does not support capturing + output and exceptions in the same code unit. + https://github.com/python/cpython/issues/65382 + Without the ``strict=True`` argument, any bug that results in iterables of different lengths will be silenced, possibly manifesting as a hard-to-find bug in another part of the program. From webhook-mailer at python.org Thu Dec 29 10:14:19 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 29 Dec 2022 15:14:19 -0000 Subject: [Python-checkins] gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (#100601) Message-ID: <mailman.3128.1672326860.3313.python-checkins@python.org> https://github.com/python/cpython/commit/76856366d3ece34c3e738f7167329e97bbf52b34 commit: 76856366d3ece34c3e738f7167329e97bbf52b34 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-29T20:43:38+05:30 summary: gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (#100601) files: M Lib/test/test_coroutines.py diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 43a3ff0536fe..6ab19efcc588 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2214,6 +2214,7 @@ async def f(): gen = f() with self.assertWarns(RuntimeWarning): gen.cr_frame.clear() + gen.close() def test_stack_in_coroutine_throw(self): # Regression test for https://github.com/python/cpython/issues/93592 From webhook-mailer at python.org Thu Dec 29 10:17:26 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 29 Dec 2022 15:17:26 -0000 Subject: [Python-checkins] gh-100583: Improve the `pydoc` documentation (#100590) Message-ID: <mailman.3129.1672327047.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7223d50b9785bc7b0cd76dcc68d97dabcbade4b6 commit: 7223d50b9785bc7b0cd76dcc68d97dabcbade4b6 branch: main author: ram vikram singh <ramvikrams243 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-29T20:47:20+05:30 summary: gh-100583: Improve the `pydoc` documentation (#100590) files: M Doc/library/pydoc.rst diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 94daf4a58f9c..03e0915bf6d1 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -33,7 +33,7 @@ as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: - pydoc sys + python -m pydoc sys at a shell prompt will display documentation on the :mod:`sys` module, in a style similar to the manual pages shown by the Unix :program:`man` command. The @@ -65,18 +65,18 @@ manner similar to the Unix :program:`man` command. The synopsis line of a module is the first line of its documentation string. You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting web browsers. :program:`pydoc -p 1234` +that will serve documentation to visiting web browsers. :program:`python -m pydoc -p 1234` will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. -:program:`pydoc -n <hostname>` will start the server listening at the given +:program:`python -m pydoc -n <hostname>` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the server responds to. During development this is especially useful if you want to run pydoc from within a container. -:program:`pydoc -b` will start the server and additionally open a web +:program:`python -m pydoc -b` will start the server and additionally open a web browser to a module index page. Each served page has a navigation bar at the top where you can *Get* help on an individual item, *Search* all modules with a keyword in their synopsis line, and go to the *Module index*, *Topics* and From webhook-mailer at python.org Thu Dec 29 10:24:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 29 Dec 2022 15:24:17 -0000 Subject: [Python-checkins] gh-100583: Improve the `pydoc` documentation (GH-100590) Message-ID: <mailman.3130.1672327459.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e180dff4c7fff1d0b2d66e6be3e9c06835a130ef commit: e180dff4c7fff1d0b2d66e6be3e9c06835a130ef branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-29T07:24:11-08:00 summary: gh-100583: Improve the `pydoc` documentation (GH-100590) (cherry picked from commit 7223d50b9785bc7b0cd76dcc68d97dabcbade4b6) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/pydoc.rst diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 94daf4a58f9c..03e0915bf6d1 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -33,7 +33,7 @@ as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: - pydoc sys + python -m pydoc sys at a shell prompt will display documentation on the :mod:`sys` module, in a style similar to the manual pages shown by the Unix :program:`man` command. The @@ -65,18 +65,18 @@ manner similar to the Unix :program:`man` command. The synopsis line of a module is the first line of its documentation string. You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting web browsers. :program:`pydoc -p 1234` +that will serve documentation to visiting web browsers. :program:`python -m pydoc -p 1234` will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. -:program:`pydoc -n <hostname>` will start the server listening at the given +:program:`python -m pydoc -n <hostname>` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the server responds to. During development this is especially useful if you want to run pydoc from within a container. -:program:`pydoc -b` will start the server and additionally open a web +:program:`python -m pydoc -b` will start the server and additionally open a web browser to a module index page. Each served page has a navigation bar at the top where you can *Get* help on an individual item, *Search* all modules with a keyword in their synopsis line, and go to the *Module index*, *Topics* and From webhook-mailer at python.org Thu Dec 29 10:41:40 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 29 Dec 2022 15:41:40 -0000 Subject: [Python-checkins] gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (GH-100601) Message-ID: <mailman.3131.1672328501.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dd520e7bae18549971b900b6dadc69bfc35d4b3f commit: dd520e7bae18549971b900b6dadc69bfc35d4b3f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-29T07:41:34-08:00 summary: gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (GH-100601) (cherry picked from commit 76856366d3ece34c3e738f7167329e97bbf52b34) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_coroutines.py diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 10f1a9efbcbd..93ddbf6e8976 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2207,6 +2207,7 @@ async def f(): gen = f() with self.assertWarns(RuntimeWarning): gen.cr_frame.clear() + gen.close() def test_stack_in_coroutine_throw(self): # Regression test for https://github.com/python/cpython/issues/93592 From webhook-mailer at python.org Thu Dec 29 11:15:20 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 29 Dec 2022 16:15:20 -0000 Subject: [Python-checkins] [3.10] gh-100583: Improve the `pydoc` documentation (GH-100590) (#100607) Message-ID: <mailman.3132.1672330520.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b1cc33db0b413c42393ea3a9b9cdf2c585d804a commit: 2b1cc33db0b413c42393ea3a9b9cdf2c585d804a branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-29T21:45:13+05:30 summary: [3.10] gh-100583: Improve the `pydoc` documentation (GH-100590) (#100607) gh-100583: Improve the `pydoc` documentation (GH-100590) (cherry picked from commit 7223d50b9785bc7b0cd76dcc68d97dabcbade4b6) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/pydoc.rst diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 94daf4a58f9c..03e0915bf6d1 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -33,7 +33,7 @@ as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: - pydoc sys + python -m pydoc sys at a shell prompt will display documentation on the :mod:`sys` module, in a style similar to the manual pages shown by the Unix :program:`man` command. The @@ -65,18 +65,18 @@ manner similar to the Unix :program:`man` command. The synopsis line of a module is the first line of its documentation string. You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting web browsers. :program:`pydoc -p 1234` +that will serve documentation to visiting web browsers. :program:`python -m pydoc -p 1234` will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. -:program:`pydoc -n <hostname>` will start the server listening at the given +:program:`python -m pydoc -n <hostname>` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the server responds to. During development this is especially useful if you want to run pydoc from within a container. -:program:`pydoc -b` will start the server and additionally open a web +:program:`python -m pydoc -b` will start the server and additionally open a web browser to a module index page. Each served page has a navigation bar at the top where you can *Get* help on an individual item, *Search* all modules with a keyword in their synopsis line, and go to the *Module index*, *Topics* and From webhook-mailer at python.org Thu Dec 29 17:13:52 2022 From: webhook-mailer at python.org (zware) Date: Thu, 29 Dec 2022 22:13:52 -0000 Subject: [Python-checkins] gh-100540: Remove obsolete Modules/_ctypes/darwin/ dlfcn shim (GH-100541) Message-ID: <mailman.3133.1672352033.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2df82db48506e5a2044a28f147fdb42f662d37b9 commit: 2df82db48506e5a2044a28f147fdb42f662d37b9 branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-29T16:13:28-06:00 summary: gh-100540: Remove obsolete Modules/_ctypes/darwin/ dlfcn shim (GH-100541) As far as I can tell, this hasn't been actually used since Mac OS X 10.2. files: A Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst D Modules/_ctypes/ctypes_dlfcn.h D Modules/_ctypes/darwin/LICENSE D Modules/_ctypes/darwin/README D Modules/_ctypes/darwin/README.ctypes D Modules/_ctypes/darwin/dlfcn.h D Modules/_ctypes/darwin/dlfcn_simple.c M Makefile.pre.in M Modules/_ctypes/_ctypes.c M Modules/_ctypes/callproc.c M PCbuild/_ctypes.vcxproj M PCbuild/_ctypes.vcxproj.filters M Tools/c-analyzer/cpython/_parser.py M configure M configure.ac diff --git a/Makefile.pre.in b/Makefile.pre.in index dd6c3fbd1c64..1f8bd561f61d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2591,7 +2591,7 @@ MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h -MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h $(srcdir)/Modules/_ctypes/darwin/dlfcn.h +MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ diff --git a/Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst b/Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst new file mode 100644 index 000000000000..a42814e1861d --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst @@ -0,0 +1,2 @@ +Removed obsolete ``dlfcn.h`` shim from the ``_ctypes`` extension module, +which has not been necessary since Mac OS X 10.2. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index f69a37709963..4ce6433a2e45 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -120,7 +120,7 @@ bytes(cdata) #define IS_INTRESOURCE(x) (((size_t)(x) >> 16) == 0) #endif #else -#include "ctypes_dlfcn.h" +#include <dlfcn.h> #endif #include "ctypes.h" @@ -768,7 +768,7 @@ CDataType_in_dll(PyObject *type, PyObject *args) return NULL; } #else - address = (void *)ctypes_dlsym(handle, name); + address = (void *)dlsym(handle, name); if (!address) { #ifdef __CYGWIN__ /* dlerror() isn't very helpful on cygwin */ @@ -776,7 +776,7 @@ CDataType_in_dll(PyObject *type, PyObject *args) "symbol '%s' not found", name); #else - PyErr_SetString(PyExc_ValueError, ctypes_dlerror()); + PyErr_SetString(PyExc_ValueError, dlerror()); #endif return NULL; } @@ -3560,7 +3560,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else - address = (PPROC)ctypes_dlsym(handle, name); + address = (PPROC)dlsym(handle, name); if (!address) { #ifdef __CYGWIN__ /* dlerror() isn't very helpful on cygwin */ @@ -3568,7 +3568,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) "function '%s' not found", name); #else - PyErr_SetString(PyExc_AttributeError, ctypes_dlerror()); + PyErr_SetString(PyExc_AttributeError, dlerror()); #endif Py_DECREF(ftuple); return NULL; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 28b7cd406971..1958758dd0cf 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -67,7 +67,7 @@ #include <windows.h> #include <tchar.h> #else -#include "ctypes_dlfcn.h" +#include <dlfcn.h> #endif #ifdef __APPLE__ @@ -1537,10 +1537,10 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlopen", "O", name) < 0) { return NULL; } - handle = ctypes_dlopen(name_str, mode); + handle = dlopen(name_str, mode); Py_XDECREF(name2); if (!handle) { - const char *errmsg = ctypes_dlerror(); + const char *errmsg = dlerror(); if (!errmsg) errmsg = "dlopen() error"; PyErr_SetString(PyExc_OSError, @@ -1558,7 +1558,7 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) return NULL; if (dlclose(handle)) { PyErr_SetString(PyExc_OSError, - ctypes_dlerror()); + dlerror()); return NULL; } Py_RETURN_NONE; @@ -1576,10 +1576,10 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { return NULL; } - ptr = ctypes_dlsym((void*)handle, name); + ptr = dlsym((void*)handle, name); if (!ptr) { PyErr_SetString(PyExc_OSError, - ctypes_dlerror()); + dlerror()); return NULL; } return PyLong_FromVoidPtr(ptr); diff --git a/Modules/_ctypes/ctypes_dlfcn.h b/Modules/_ctypes/ctypes_dlfcn.h deleted file mode 100644 index 54cdde9a4fdb..000000000000 --- a/Modules/_ctypes/ctypes_dlfcn.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef _CTYPES_DLFCN_H_ -#define _CTYPES_DLFCN_H_ - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#ifndef MS_WIN32 - -#include <dlfcn.h> - -#ifndef CTYPES_DARWIN_DLFCN - -#define ctypes_dlsym dlsym -#define ctypes_dlerror dlerror -#define ctypes_dlopen dlopen -#define ctypes_dlclose dlclose -#define ctypes_dladdr dladdr - -#endif /* !CTYPES_DARWIN_DLFCN */ - -#endif /* !MS_WIN32 */ - -#ifdef __cplusplus -} -#endif /* __cplusplus */ -#endif /* _CTYPES_DLFCN_H_ */ diff --git a/Modules/_ctypes/darwin/LICENSE b/Modules/_ctypes/darwin/LICENSE deleted file mode 100644 index 786fb50258eb..000000000000 --- a/Modules/_ctypes/darwin/LICENSE +++ /dev/null @@ -1,31 +0,0 @@ -Copyright (c) 2002 Jorge Acereda <jacereda at users.sourceforge.net> & - Peter O'Gorman <ogorman at users.sourceforge.net> - -Portions may be copyright others, see the AUTHORS file included with this -distribution. - -Maintained by Peter O'Gorman <ogorman at users.sourceforge.net> - -Bug Reports and other queries should go to <ogorman at users.sourceforge.net> - - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - diff --git a/Modules/_ctypes/darwin/README b/Modules/_ctypes/darwin/README deleted file mode 100644 index 4d63f3dfa5eb..000000000000 --- a/Modules/_ctypes/darwin/README +++ /dev/null @@ -1,95 +0,0 @@ -dlcompat for Darwin -========================= - -This is dlcompat, a small library that emulates the dlopen() -interface on top of Darwin's dyld API. - -dlcompat allows loading a ".dylib" library (as long as the RTLD_LOCAL -flag isn't passed to dlopen()). It can be configured to yield a warning -when trying to close it (dynamic libraries cannot currently be unloaded). - -It automatically searches for modules in several directories when no -absolute path is specified and the module is not found in the current -directory. - -The paths searched are those specified in the environment variables -LD_LIBRARY_PATH and DYLD_LIBRARY_PATH plus /lib, /usr/local/lib and -/usr/lib or the path specified in the environment variable -DYLD_FALLBACK_LIBRARY_PATH. - -In the default install the behavior of dlsym is to automatically prepend -an underscore to passed in symbol names, this allows easier porting of -applications which were written specifically for ELF based lifeforms. - -Installation --------------- -Type: - ./configure - make - sudo make install - -This will compile the source file, generate both a static and shared -library called libdl and install it into /usr/local/lib. The header -file dlfcn.h will be installed in /usr/local/include. - -If you want to place the files somewhere else, run - - make clean - ./configure --prefix=<prefix> - make - sudo make install - -where <prefix> is the hierarchy you want to install into, e.g. /usr -for /usr/lib and /usr/include (_NOT_ recommended!). - -To enable debugging output (useful for me), run - - make clean - ./configure --enable-debug - make - sudo make install - -If you want old dlcompat style behavior of not prepending the underscore -on calls to dlsym then type: - - make clean - ./configure --enable-fink - make - sudo make install - -Usage -------- -Software that uses GNU autoconf will likely check for a library called -libdl, that's why I named it that way. For software that doesn't find -the library on its own, you must add a '-ldl' to the appropriate -Makefile (or environment) variable, usually LIBS. - -If you installed dlcompat into a directory other than /usr/local/lib, -you must tell the compiler where to find it. Add '-L<prefix>/lib' to -LDFLAGS (or CFLAGS) and '-I<prefix>/include' to CPPFLAGS (or CFLAGS). - -Notes ------ -If you are writing new software and plan to have Mac OX X compatibility you -should look at the dyld api's in /usr/include/mach-o/dyld.h, rather than -using dlcompat, using the native api's is the supported method of loading -dynamically on Mac OS X, if you want an small example, look at dlfcn_simple.c, -which should help get you started. - -Also note that the functions in dlcompat are not thread safe, and while it is not -POSIX spec compliant, it is about as close to compliance as it is going to get though. - -You can always get the latest version from opendarwin cvs: - - cvs -d :pserver:anonymous at anoncvs.opendarwin.org:/cvs/od login - cvs -z3 -d :pserver:anonymous at anoncvs.opendarwin.org:/cvs/od \ - co -d dlcompat proj/dlcompat - - -It is hoped that this library will be useful, and as bug free as possible, if you find -any bugs please let us know about them so they can be fixed. - -Please send bug reports to Peter O'Gorman <ogorman at users.sourceforge.net> - -Thanks. - diff --git a/Modules/_ctypes/darwin/README.ctypes b/Modules/_ctypes/darwin/README.ctypes deleted file mode 100644 index 8520b01f49da..000000000000 --- a/Modules/_ctypes/darwin/README.ctypes +++ /dev/null @@ -1,11 +0,0 @@ -The files in this directory are taken from -http://www.opendarwin.org/cgi-bin/cvsweb.cgi/~checkout~/proj/dlcompat/ - -The LICENSE in this directory applies to these files. - -Thomas Heller, Jan 2003 - -These files have been modified so they fall back to the system -dlfcn calls if available in libSystem. - -Bob Ippolito, Feb 2006 diff --git a/Modules/_ctypes/darwin/dlfcn.h b/Modules/_ctypes/darwin/dlfcn.h deleted file mode 100644 index a2afc3eeb847..000000000000 --- a/Modules/_ctypes/darwin/dlfcn.h +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright (c) 2002 Jorge Acereda <jacereda at users.sourceforge.net> & - Peter O'Gorman <ogorman at users.sourceforge.net> - -Portions may be copyright others, see the AUTHORS file included with this -distribution. - -Maintained by Peter O'Gorman <ogorman at users.sourceforge.net> - -Bug Reports and other queries should go to <ogorman at users.sourceforge.net> - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ -#ifndef _DLFCN_H_ -#define _DLFCN_H_ - -#include <AvailabilityMacros.h> - -#ifdef __cplusplus -extern "C" { -#endif - - -/* - * Structure filled in by dladdr(). - */ - -typedef struct dl_info { - const char *dli_fname; /* Pathname of shared object */ - void *dli_fbase; /* Base address of shared object */ - const char *dli_sname; /* Name of nearest symbol */ - void *dli_saddr; /* Address of nearest symbol */ -} Dl_info; - - -#if MAC_OS_X_VERSION_MIN_REQUIRED <= MAC_OS_X_VERSION_10_2 -#warning CTYPES_DARWIN_DLFCN -#define CTYPES_DARWIN_DLFCN -extern void * (*ctypes_dlopen)(const char *path, int mode); -extern void * (*ctypes_dlsym)(void * handle, const char *symbol); -extern const char * (*ctypes_dlerror)(void); -extern int (*ctypes_dlclose)(void * handle); -extern int (*ctypes_dladdr)(const void *, Dl_info *); -#else -extern void * dlopen(const char *path, int mode); -extern void * dlsym(void * handle, const char *symbol); -extern const char * dlerror(void); -extern int dlclose(void * handle); -extern int dladdr(const void *, Dl_info *); -#endif - -#define RTLD_LAZY 0x1 -#define RTLD_NOW 0x2 -#define RTLD_LOCAL 0x4 -#define RTLD_GLOBAL 0x8 -#define RTLD_NOLOAD 0x10 -#define RTLD_NODELETE 0x80 - -/* These are from the Mac OS X 10.4 headers */ -#define RTLD_NEXT ((void *) -1) /* Search subsequent objects. */ -#define RTLD_DEFAULT ((void *) -2) /* Use default search algorithm. */ - -#ifdef __cplusplus -} -#endif - -#endif /* _DLFCN_H_ */ diff --git a/Modules/_ctypes/darwin/dlfcn_simple.c b/Modules/_ctypes/darwin/dlfcn_simple.c deleted file mode 100644 index 2b293bb8695b..000000000000 --- a/Modules/_ctypes/darwin/dlfcn_simple.c +++ /dev/null @@ -1,272 +0,0 @@ -/* -Copyright (c) 2002 Peter O'Gorman <ogorman at users.sourceforge.net> - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - - -/* Just to prove that it isn't that hard to add Mac calls to your code :) - This works with pretty much everything, including kde3 xemacs and the gimp, - I'd guess that it'd work in at least 95% of cases, use this as your starting - point, rather than the mess that is dlfcn.c, assuming that your code does not - require ref counting or symbol lookups in dependent libraries -*/ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <stdarg.h> -#include <limits.h> -#include <mach-o/dyld.h> -#include <AvailabilityMacros.h> -#include "dlfcn.h" - -#ifdef CTYPES_DARWIN_DLFCN - -#define ERR_STR_LEN 256 - -#ifndef MAC_OS_X_VERSION_10_3 -#define MAC_OS_X_VERSION_10_3 1030 -#endif - -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3 -#define DARWIN_HAS_DLOPEN -extern void * dlopen(const char *path, int mode) __attribute__((weak_import)); -extern void * dlsym(void * handle, const char *symbol) __attribute__((weak_import)); -extern const char * dlerror(void) __attribute__((weak_import)); -extern int dlclose(void * handle) __attribute__((weak_import)); -extern int dladdr(const void *, Dl_info *) __attribute__((weak_import)); -#endif /* MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3 */ - -#ifndef DARWIN_HAS_DLOPEN -#define dlopen darwin_dlopen -#define dlsym darwin_dlsym -#define dlerror darwin_dlerror -#define dlclose darwin_dlclose -#define dladdr darwin_dladdr -#endif - -void * (*ctypes_dlopen)(const char *path, int mode); -void * (*ctypes_dlsym)(void * handle, const char *symbol); -const char * (*ctypes_dlerror)(void); -int (*ctypes_dlclose)(void * handle); -int (*ctypes_dladdr)(const void *, Dl_info *); - -#if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 -/* Mac OS X 10.3+ has dlopen, so strip all this dead code to avoid warnings */ - -static void *dlsymIntern(void *handle, const char *symbol); - -static const char *error(int setget, const char *str, ...); - -/* Set and get the error string for use by dlerror */ -static const char *error(int setget, const char *str, ...) -{ - static char errstr[ERR_STR_LEN]; - static int err_filled = 0; - const char *retval; - va_list arg; - if (setget == 0) - { - va_start(arg, str); - strncpy(errstr, "dlcompat: ", ERR_STR_LEN); - vsnprintf(errstr + 10, ERR_STR_LEN - 10, str, arg); - va_end(arg); - err_filled = 1; - retval = NULL; - } - else - { - if (!err_filled) - retval = NULL; - else - retval = errstr; - err_filled = 0; - } - return retval; -} - -/* darwin_dlopen */ -static void *darwin_dlopen(const char *path, int mode) -{ - void *module = 0; - NSObjectFileImage ofi = 0; - NSObjectFileImageReturnCode ofirc; - - /* If we got no path, the app wants the global namespace, use -1 as the marker - in this case */ - if (!path) - return (void *)-1; - - /* Create the object file image, works for things linked with the -bundle arg to ld */ - ofirc = NSCreateObjectFileImageFromFile(path, &ofi); - switch (ofirc) - { - case NSObjectFileImageSuccess: - /* It was okay, so use NSLinkModule to link in the image */ - module = NSLinkModule(ofi, path, - NSLINKMODULE_OPTION_RETURN_ON_ERROR - | (mode & RTLD_GLOBAL) ? 0 : NSLINKMODULE_OPTION_PRIVATE - | (mode & RTLD_LAZY) ? 0 : NSLINKMODULE_OPTION_BINDNOW); - NSDestroyObjectFileImage(ofi); - break; - case NSObjectFileImageInappropriateFile: - /* It may have been a dynamic library rather than a bundle, try to load it */ - module = (void *)NSAddImage(path, NSADDIMAGE_OPTION_RETURN_ON_ERROR); - break; - default: - /* God knows what we got */ - error(0, "Can not open \"%s\"", path); - return 0; - } - if (!module) - error(0, "Can not open \"%s\"", path); - return module; - -} - -/* dlsymIntern is used by dlsym to find the symbol */ -static void *dlsymIntern(void *handle, const char *symbol) -{ - NSSymbol nssym = 0; - /* If the handle is -1, if is the app global context */ - if (handle == (void *)-1) - { - /* Global context, use NSLookupAndBindSymbol */ - if (NSIsSymbolNameDefined(symbol)) - { - nssym = NSLookupAndBindSymbol(symbol); - } - - } - /* Now see if the handle is a struch mach_header* or not, use NSLookupSymbol in image - for libraries, and NSLookupSymbolInModule for bundles */ - else - { - /* Check for both possible magic numbers depending on x86/ppc byte order */ - if ((((struct mach_header *)handle)->magic == MH_MAGIC) || - (((struct mach_header *)handle)->magic == MH_CIGAM)) - { - if (NSIsSymbolNameDefinedInImage((struct mach_header *)handle, symbol)) - { - nssym = NSLookupSymbolInImage((struct mach_header *)handle, - symbol, - NSLOOKUPSYMBOLINIMAGE_OPTION_BIND - | NSLOOKUPSYMBOLINIMAGE_OPTION_RETURN_ON_ERROR); - } - - } - else - { - nssym = NSLookupSymbolInModule(handle, symbol); - } - } - if (!nssym) - { - error(0, "Symbol \"%s\" Not found", symbol); - return NULL; - } - return NSAddressOfSymbol(nssym); -} - -static const char *darwin_dlerror(void) -{ - return error(1, (char *)NULL); -} - -static int darwin_dlclose(void *handle) -{ - if ((((struct mach_header *)handle)->magic == MH_MAGIC) || - (((struct mach_header *)handle)->magic == MH_CIGAM)) - { - error(0, "Can't remove dynamic libraries on darwin"); - return 0; - } - if (!NSUnLinkModule(handle, 0)) - { - error(0, "unable to unlink module %s", NSNameOfModule(handle)); - return 1; - } - return 0; -} - - -/* dlsym, prepend the underscore and call dlsymIntern */ -static void *darwin_dlsym(void *handle, const char *symbol) -{ - static char undersym[257]; /* Saves calls to malloc(3) */ - int sym_len = strlen(symbol); - void *value = NULL; - char *malloc_sym = NULL; - - if (sym_len < 256) - { - snprintf(undersym, 256, "_%s", symbol); - value = dlsymIntern(handle, undersym); - } - else - { - malloc_sym = malloc(sym_len + 2); - if (malloc_sym) - { - sprintf(malloc_sym, "_%s", symbol); - value = dlsymIntern(handle, malloc_sym); - free(malloc_sym); - } - else - { - error(0, "Unable to allocate memory"); - } - } - return value; -} - -static int darwin_dladdr(const void *handle, Dl_info *info) { - return 0; -} -#endif /* MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 */ - -#if __GNUC__ < 4 -#pragma CALL_ON_LOAD ctypes_dlfcn_init -#else -static void __attribute__ ((constructor)) ctypes_dlfcn_init(void); -static -#endif -void ctypes_dlfcn_init(void) { - if (dlopen != NULL) { - ctypes_dlsym = dlsym; - ctypes_dlopen = dlopen; - ctypes_dlerror = dlerror; - ctypes_dlclose = dlclose; - ctypes_dladdr = dladdr; - } else { -#if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 - ctypes_dlsym = darwin_dlsym; - ctypes_dlopen = darwin_dlopen; - ctypes_dlerror = darwin_dlerror; - ctypes_dlclose = darwin_dlclose; - ctypes_dladdr = darwin_dladdr; -#endif /* MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 */ - } -} - -#endif /* CTYPES_DARWIN_DLFCN */ diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 6ac26f1916c9..253da31e9ce1 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -102,7 +102,6 @@ </ItemDefinitionGroup> <ItemGroup> <ClInclude Include="..\Modules\_ctypes\ctypes.h" /> - <ClInclude Include="..\Modules\_ctypes\ctypes_dlfcn.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_ctypes\_ctypes.c" /> diff --git a/PCbuild/_ctypes.vcxproj.filters b/PCbuild/_ctypes.vcxproj.filters index 118c4f0698cc..a38473e3e81d 100644 --- a/PCbuild/_ctypes.vcxproj.filters +++ b/PCbuild/_ctypes.vcxproj.filters @@ -15,9 +15,6 @@ <ClInclude Include="..\Modules\_ctypes\ctypes.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="..\Modules\_ctypes\ctypes_dlfcn.h"> - <Filter>Header Files</Filter> - </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_ctypes\_ctypes.c"> diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ac1721c061f0..ab1d6257f1b1 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -51,7 +51,6 @@ def clean_lines(text): # @begin=conf@ # OSX -#Modules/_ctypes/darwin/*.c Modules/_scproxy.c # SystemConfiguration/SystemConfiguration.h # Windows diff --git a/configure b/configure index 6afd1e9c367c..946218fd8d85 100755 --- a/configure +++ b/configure @@ -12802,8 +12802,7 @@ if test "x$have_libffi" = xyes; then : case $ac_sys_system in #( Darwin) : - as_fn_append LIBFFI_CFLAGS " -I\$(srcdir)/Modules/_ctypes/darwin" - ctypes_malloc_closure=yes + ctypes_malloc_closure=yes ;; #( sunos5) : as_fn_append LIBFFI_LIBS " -mimpure-text" diff --git a/configure.ac b/configure.ac index 48736649a98e..22028972cb3d 100644 --- a/configure.ac +++ b/configure.ac @@ -3741,7 +3741,6 @@ AS_VAR_IF([have_libffi], [yes], [ AS_CASE([$ac_sys_system], [Darwin], [ dnl when do we need USING_APPLE_OS_LIBFFI? - AS_VAR_APPEND([LIBFFI_CFLAGS], [" -I\$(srcdir)/Modules/_ctypes/darwin"]) ctypes_malloc_closure=yes ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] From webhook-mailer at python.org Thu Dec 29 17:41:45 2022 From: webhook-mailer at python.org (gpshead) Date: Thu, 29 Dec 2022 22:41:45 -0000 Subject: [Python-checkins] gh-100228: Warn from os.fork() if other threads exist. (#100229) Message-ID: <mailman.3134.1672353706.3313.python-checkins@python.org> https://github.com/python/cpython/commit/894f2c3c161933bd820ad322b3b678d89bc2377c commit: 894f2c3c161933bd820ad322b3b678d89bc2377c branch: main author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-29T14:41:39-08:00 summary: gh-100228: Warn from os.fork() if other threads exist. (#100229) Not comprehensive, best effort warning. There are cases when threads exist on some platforms that this code cannot detect. macOS when API permissions allow and Linux with a readable /proc procfs present are the currently supported cases where a warning should show up reliably. Starting with a DeprecationWarning for now, it is less disruptive than something like RuntimeWarning and most likely to only be seen in people's CI tests - a good place to start with this messaging. files: A Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst M Include/internal/pycore_global_objects_fini_generated.h M Include/internal/pycore_global_strings.h M Include/internal/pycore_runtime_init_generated.h M Include/internal/pycore_unicodeobject_generated.h M Lib/test/fork_wait.py M Lib/test/test_os.py M Lib/test/test_thread.py M Lib/test/test_threading.py M Lib/threading.py M Modules/_testcapimodule.c M Modules/posixmodule.c diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6aba2f19ebde..a5365d53150b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -733,6 +733,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__xor__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); @@ -753,6 +754,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index acb9a4fbb92d..3d9e61e50ecc 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -219,6 +219,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__xor__) STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) + STRUCT_FOR_ID(_active) STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) @@ -239,6 +240,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_is_text_encoding) STRUCT_FOR_ID(_length_) + STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) STRUCT_FOR_ID(_needs_com_addref_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6d1b8702c776..3534b94753e0 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -725,6 +725,7 @@ extern "C" { INIT_ID(__xor__), \ INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ + INIT_ID(_active), \ INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ @@ -745,6 +746,7 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_is_text_encoding), \ INIT_ID(_length_), \ + INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ INIT_ID(_needs_com_addref_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7f407c0141b8..02435071bb2c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -344,6 +344,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_abstract_); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_active); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_annotation); PyUnicode_InternInPlace(&string); string = &_Py_ID(_anonymous_); @@ -384,6 +386,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_length_); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_limbo); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_lock_unlock_module); PyUnicode_InternInPlace(&string); string = &_Py_ID(_loop); diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index ebd07e612e58..c26c7aaaeb43 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -13,6 +13,7 @@ import threading from test import support from test.support import threading_helper +import warnings LONGSLEEP = 2 @@ -63,19 +64,17 @@ def test_wait(self): prefork_lives = self.alive.copy() - if sys.platform in ['unixware7']: - cpid = os.fork1() - else: - cpid = os.fork() - - if cpid == 0: - # Child - time.sleep(LONGSLEEP) - n = 0 - for key in self.alive: - if self.alive[key] != prefork_lives[key]: - n += 1 - os._exit(n) - else: - # Parent - self.wait_impl(cpid, exitcode=0) + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (cpid := os.fork()) == 0: + # Child + time.sleep(LONGSLEEP) + n = 0 + for key in self.alive: + if self.alive[key] != prefork_lives[key]: + n += 1 + os._exit(n) + else: + # Parent + self.wait_impl(cpid, exitcode=0) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e6e25b507de0..58e04dd1348f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4577,6 +4577,34 @@ def test_fork(self): assert_python_ok("-c", code) assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + @unittest.skipUnless(sys.platform in ("linux", "darwin"), + "Only Linux and macOS detect this today.") + def test_fork_warns_when_non_python_thread_exists(self): + code = """if 1: + import os, threading, warnings + from _testcapi import _spawn_pthread_waiter, _end_spawned_pthread + _spawn_pthread_waiter() + try: + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + if os.fork() == 0: + assert not ws, f"unexpected warnings in child: {ws}" + os._exit(0) # child + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + # Waiting allows an error in the child to hit stderr. + exitcode = os.wait()[1] + assert exitcode == 0, f"child exited {exitcode}" + assert threading.active_count() == 1, threading.enumerate() + finally: + _end_spawned_pthread() + """ + _, out, err = assert_python_ok("-c", code, PYTHONOPTIMIZE='0') + self.assertEqual(err.decode("utf-8"), "") + self.assertEqual(out.decode("utf-8"), "") + # Only test if the C version is provided, otherwise TestPEP519 already tested # the pure Python implementation. diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 2ae5e9ccb440..8656fbdd83e9 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -5,6 +5,7 @@ from test.support import threading_helper import _thread as thread import time +import warnings import weakref from test import lock_tests @@ -238,11 +239,13 @@ def test_forkinthread(self): def fork_thread(read_fd, write_fd): nonlocal pid - # fork in a thread - pid = os.fork() - if pid: - # parent process - return + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + # fork in a thread (DANGER, undefined per POSIX) + if (pid := os.fork()): + # parent process + return # child process try: diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 13ba5068ae20..31bf46311a80 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -20,6 +20,7 @@ import signal import textwrap import traceback +import warnings from unittest import mock from test import lock_tests @@ -563,7 +564,7 @@ def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. code = """if 1: - import _thread, threading, os, time + import _thread, threading, os, time, warnings def background_thread(evt): # Creates and registers the _DummyThread instance @@ -575,11 +576,16 @@ def background_thread(evt): _thread.start_new_thread(background_thread, (evt,)) evt.wait() assert threading.active_count() == 2, threading.active_count() - if os.fork() == 0: - assert threading.active_count() == 1, threading.active_count() - os._exit(0) - else: - os.wait() + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + os.wait() """ _, out, err = assert_python_ok("-c", code) self.assertEqual(out, b'') @@ -598,13 +604,15 @@ def test_is_alive_after_fork(self): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - pid = os.fork() - if pid == 0: - os._exit(11 if t.is_alive() else 10) - else: - t.join() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (pid := os.fork()) == 0: + os._exit(11 if t.is_alive() else 10) + else: + t.join() - support.wait_process(pid, exitcode=10) + support.wait_process(pid, exitcode=10) def test_main_thread(self): main = threading.main_thread() @@ -645,21 +653,26 @@ def test_main_thread_after_fork(self): @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork_from_nonmain_thread(self): code = """if 1: - import os, threading, sys + import os, threading, sys, warnings from test import support def func(): - pid = os.fork() - if pid == 0: - main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) - # stdout is fully buffered because not a tty, - # we have to flush before exit. - sys.stdout.flush() - else: - support.wait_process(pid, exitcode=0) + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + support.wait_process(pid, exitcode=0) th = threading.Thread(target=func) th.start() @@ -667,7 +680,7 @@ def func(): """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') - self.assertEqual(err, b"") + self.assertEqual(err.decode('utf-8'), "") self.assertEqual(data, "Thread-1 (func)\nTrue\nTrue\n") def test_main_thread_during_shutdown(self): @@ -1173,15 +1186,18 @@ def do_fork_and_wait(): else: os._exit(50) - # start a bunch of threads that will fork() child processes - threads = [] - for i in range(16): - t = threading.Thread(target=do_fork_and_wait) - threads.append(t) - t.start() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() - for t in threads: - t.join() + for t in threads: + t.join() @support.requires_fork() def test_clear_threads_states_after_fork(self): @@ -1194,18 +1210,22 @@ def test_clear_threads_states_after_fork(self): threads.append(t) t.start() - pid = os.fork() - if pid == 0: - # check that threads states have been cleared - if len(sys._current_frames()) == 1: - os._exit(51) - else: - os._exit(52) - else: - support.wait_process(pid, exitcode=51) - - for t in threads: - t.join() + try: + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(51) + else: + os._exit(52) + else: + support.wait_process(pid, exitcode=51) + finally: + for t in threads: + t.join() class SubinterpThreadingTests(BaseTestCase): diff --git a/Lib/threading.py b/Lib/threading.py index 723bd58bf5a6..df273870fa42 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1490,6 +1490,8 @@ def active_count(): enumerate(). """ + # NOTE: if the logic in here ever changes, update Modules/posixmodule.c + # warn_about_fork_with_threads() to match. with _active_limbo_lock: return len(_active) + len(_limbo) diff --git a/Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst b/Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst new file mode 100644 index 000000000000..ca6e4a2a1f0d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst @@ -0,0 +1,5 @@ +A :exc:`DeprecationWarning` may be raised when :func:`os.fork()` or +:func:`os.forkpty()` is called from multi-threaded processes. Forking +with threads is unsafe and can cause deadlocks, crashes and subtle +problems. Lack of a warning does not indicate that the fork call was +actually safe, as Python may not be aware of all threads. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c32fdb5f5fbe..c777c3efc492 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -25,6 +25,9 @@ #include "structmember.h" // for offsetof(), T_OBJECT #include <float.h> // FLT_MAX #include <signal.h> +#ifndef MS_WINDOWS +#include <unistd.h> +#endif #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> // W_STOPCODE @@ -871,6 +874,46 @@ test_thread_state(PyObject *self, PyObject *args) Py_RETURN_NONE; } +#ifndef MS_WINDOWS +static PyThread_type_lock wait_done = NULL; + +static void wait_for_lock(void *unused) { + PyThread_acquire_lock(wait_done, 1); + PyThread_release_lock(wait_done); + PyThread_free_lock(wait_done); + wait_done = NULL; +} + +// These can be used to test things that care about the existence of another +// thread that the threading module doesn't know about. + +static PyObject * +spawn_pthread_waiter(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (wait_done) { + PyErr_SetString(PyExc_RuntimeError, "thread already running"); + return NULL; + } + wait_done = PyThread_allocate_lock(); + if (wait_done == NULL) + return PyErr_NoMemory(); + PyThread_acquire_lock(wait_done, 1); + PyThread_start_new_thread(wait_for_lock, NULL); + Py_RETURN_NONE; +} + +static PyObject * +end_spawned_pthread(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (!wait_done) { + PyErr_SetString(PyExc_RuntimeError, "call _spawn_pthread_waiter 1st"); + return NULL; + } + PyThread_release_lock(wait_done); + Py_RETURN_NONE; +} +#endif // not MS_WINDOWS + /* test Py_AddPendingCalls using threads */ static int _pending_callback(void *arg) { @@ -3207,6 +3250,10 @@ static PyMethodDef TestMethods[] = { {"test_get_type_name", test_get_type_name, METH_NOARGS}, {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, +#ifndef MS_WINDOWS + {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, + {"_end_spawned_pthread", end_spawned_pthread, METH_NOARGS}, +#endif {"_pending_threadfunc", pending_threadfunc, METH_VARARGS}, #ifdef HAVE_GETTIMEOFDAY {"profile_int", profile_int, METH_NOARGS}, diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 607d40b59d96..1d9a33ab7bc9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -72,6 +72,8 @@ */ #if defined(__APPLE__) +#include <mach/mach.h> + #if defined(__has_builtin) #if __has_builtin(__builtin_available) #define HAVE_BUILTIN_AVAILABLE 1 @@ -6745,6 +6747,104 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, } #endif /* HAVE_FORK */ +// Common code to raise a warning if we detect there is more than one thread +// running in the process. Best effort, silent if unable to count threads. +// Constraint: Quick. Never overcounts. Never leaves an error set. +// +// This code might do an import, thus acquiring the import lock, which +// PyOS_BeforeFork() also does. As this should only be called from +// the parent process, it is in the same thread so that works. +static void warn_about_fork_with_threads(const char* name) { + // TODO: Consider making an `os` module API to return the current number + // of threads in the process. That'd presumably use this platform code but + // raise an error rather than using the inaccurate fallback. + Py_ssize_t num_python_threads = 0; +#if defined(__APPLE__) && defined(HAVE_GETPID) + mach_port_t macos_self = mach_task_self(); + mach_port_t macos_task; + if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { + thread_array_t macos_threads; + mach_msg_type_number_t macos_n_threads; + if (task_threads(macos_task, &macos_threads, + &macos_n_threads) == KERN_SUCCESS) { + num_python_threads = macos_n_threads; + } + } +#elif defined(__linux__) + // Linux /proc/self/stat 20th field is the number of threads. + FILE* proc_stat = fopen("/proc/self/stat", "r"); + if (proc_stat) { + size_t n; + // Size chosen arbitrarily. ~60% more bytes than a 20th column index + // observed on the author's workstation. + char stat_line[160]; + n = fread(&stat_line, 1, 159, proc_stat); + stat_line[n] = '\0'; + fclose(proc_stat); + + char *saveptr = NULL; + char *field = strtok_r(stat_line, " ", &saveptr); + unsigned int idx; + for (idx = 19; idx && field; --idx) { + field = strtok_r(NULL, " ", &saveptr); + } + if (idx == 0 && field) { // found the 20th field + num_python_threads = atoi(field); // 0 on error + } + } +#endif + if (num_python_threads <= 0) { + // Fall back to just the number our threading module knows about. + // An incomplete view of the world, but better than nothing. + PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); + if (!threading) { + PyErr_Clear(); + return; + } + PyObject *threading_active = + PyObject_GetAttr(threading, &_Py_ID(_active)); + if (!threading_active) { + PyErr_Clear(); + Py_DECREF(threading); + return; + } + PyObject *threading_limbo = + PyObject_GetAttr(threading, &_Py_ID(_limbo)); + if (!threading_limbo) { + PyErr_Clear(); + Py_DECREF(threading); + Py_DECREF(threading_active); + return; + } + Py_DECREF(threading); + // Duplicating what threading.active_count() does but without holding + // threading._active_limbo_lock so our count could be inaccurate if + // these dicts are mid-update from another thread. Not a big deal. + // Worst case if someone replaced threading._active or threading._limbo + // with non-dicts, we get -1 from *Length() below and undercount. + // Nobody should, but we're best effort so we clear errors and move on. + num_python_threads = (PyMapping_Length(threading_active) + + PyMapping_Length(threading_limbo)); + PyErr_Clear(); + Py_DECREF(threading_active); + Py_DECREF(threading_limbo); + } + if (num_python_threads > 1) { + PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, +#ifdef HAVE_GETPID + "This process (pid=%d) is multi-threaded, " +#else + "This process is multi-threaded, " +#endif + "use of %s() may lead to deadlocks in the child.", +#ifdef HAVE_GETPID + getpid(), +#endif + name); + PyErr_Clear(); + } +} #ifdef HAVE_FORK1 /*[clinic input] @@ -6771,6 +6871,7 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + warn_about_fork_with_threads("fork1"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } @@ -6810,6 +6911,7 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + warn_about_fork_with_threads("fork"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } @@ -7479,6 +7581,7 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + warn_about_fork_with_threads("forkpty"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } From webhook-mailer at python.org Fri Dec 30 06:55:31 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 30 Dec 2022 11:55:31 -0000 Subject: [Python-checkins] gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (#99434) Message-ID: <mailman.3135.1672401333.3313.python-checkins@python.org> https://github.com/python/cpython/commit/79c10b7da84f52999dc483fc62c8e758ad3eff23 commit: 79c10b7da84f52999dc483fc62c8e758ad3eff23 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-30T17:25:01+05:30 summary: gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (#99434) files: A Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst M Lib/doctest.py M Lib/test/doctest_lineno.py diff --git a/Lib/doctest.py b/Lib/doctest.py index b2ef2ce63672..dafad505ef0e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -956,7 +956,8 @@ def _from_module(self, module, object): return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): diff --git a/Lib/test/doctest_lineno.py b/Lib/test/doctest_lineno.py index be198513a150..729a68aceaa9 100644 --- a/Lib/test/doctest_lineno.py +++ b/Lib/test/doctest_lineno.py @@ -48,3 +48,6 @@ def method_with_doctest(self): >>> MethodWrapper.method_with_doctest.__name__ 'method_with_doctest' """ + +# https://github.com/python/cpython/issues/99433 +str_wrapper = object().__str__ diff --git a/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst new file mode 100644 index 000000000000..8e13a9a66a7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` failure on :class:`types.MethodWrapperType` in modules. From webhook-mailer at python.org Fri Dec 30 07:19:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 12:19:00 -0000 Subject: [Python-checkins] gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (GH-99434) Message-ID: <mailman.3136.1672402742.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c88a83e7d81fbf394bbdebe0f453bb64bdf33ba6 commit: c88a83e7d81fbf394bbdebe0f453bb64bdf33ba6 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T04:18:54-08:00 summary: gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (GH-99434) (cherry picked from commit 79c10b7da84f52999dc483fc62c8e758ad3eff23) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: A Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst M Lib/doctest.py M Lib/test/doctest_lineno.py diff --git a/Lib/doctest.py b/Lib/doctest.py index b2ef2ce63672..dafad505ef0e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -956,7 +956,8 @@ def _from_module(self, module, object): return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): diff --git a/Lib/test/doctest_lineno.py b/Lib/test/doctest_lineno.py index be198513a150..729a68aceaa9 100644 --- a/Lib/test/doctest_lineno.py +++ b/Lib/test/doctest_lineno.py @@ -48,3 +48,6 @@ def method_with_doctest(self): >>> MethodWrapper.method_with_doctest.__name__ 'method_with_doctest' """ + +# https://github.com/python/cpython/issues/99433 +str_wrapper = object().__str__ diff --git a/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst new file mode 100644 index 000000000000..8e13a9a66a7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` failure on :class:`types.MethodWrapperType` in modules. From webhook-mailer at python.org Fri Dec 30 11:35:31 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Fri, 30 Dec 2022 16:35:31 -0000 Subject: [Python-checkins] gh-100616: Document 'attr' parameter for window.vline() in curses module (#24961) Message-ID: <mailman.3137.1672418133.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f4fcfdf8c593611f98b9358cc0c5604c15306465 commit: f4fcfdf8c593611f98b9358cc0c5604c15306465 branch: main author: mathieui <mathieui at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-30T08:35:04-08:00 summary: gh-100616: Document 'attr' parameter for window.vline() in curses module (#24961) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst M Doc/library/curses.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index f9f94b22f69e..f50b51c3780e 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1297,11 +1297,11 @@ the following methods and attributes: :meth:`refresh`. -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch*. + character *ch* with attributes *attr*. Constants diff --git a/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst new file mode 100644 index 000000000000..97a7022c94b4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst @@ -0,0 +1,2 @@ +Document existing ``attr`` parameter to :func:`curses.window.vline` function +in :mod:`curses`. From webhook-mailer at python.org Fri Dec 30 12:06:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 17:06:00 -0000 Subject: [Python-checkins] gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) Message-ID: <mailman.3138.1672419961.3313.python-checkins@python.org> https://github.com/python/cpython/commit/297465a4db0aff45e57cf5bfef4f3ab4d9538de0 commit: 297465a4db0aff45e57cf5bfef4f3ab4d9538de0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T09:05:49-08:00 summary: gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) (cherry picked from commit f4fcfdf8c593611f98b9358cc0c5604c15306465) Co-authored-by: mathieui <mathieui at users.noreply.github.com> Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst M Doc/library/curses.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 9b2c3fbd8e4e..56311033d631 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1300,11 +1300,11 @@ the following methods and attributes: :meth:`refresh`. -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch*. + character *ch* with attributes *attr*. Constants diff --git a/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst new file mode 100644 index 000000000000..97a7022c94b4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst @@ -0,0 +1,2 @@ +Document existing ``attr`` parameter to :func:`curses.window.vline` function +in :mod:`curses`. From webhook-mailer at python.org Fri Dec 30 12:06:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 17:06:00 -0000 Subject: [Python-checkins] gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) Message-ID: <mailman.3139.1672419961.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3ab7b61b416b3ce4a01b0566a8d2bec3e3d0e3d4 commit: 3ab7b61b416b3ce4a01b0566a8d2bec3e3d0e3d4 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T09:05:40-08:00 summary: gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) (cherry picked from commit f4fcfdf8c593611f98b9358cc0c5604c15306465) Co-authored-by: mathieui <mathieui at users.noreply.github.com> Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst M Doc/library/curses.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 55ab06018d70..5d74b73e2e16 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1314,11 +1314,11 @@ the following methods and attributes: :meth:`refresh`. -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch*. + character *ch* with attributes *attr*. Constants diff --git a/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst new file mode 100644 index 000000000000..97a7022c94b4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst @@ -0,0 +1,2 @@ +Document existing ``attr`` parameter to :func:`curses.window.vline` function +in :mod:`curses`. From webhook-mailer at python.org Fri Dec 30 16:21:38 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 21:21:38 -0000 Subject: [Python-checkins] gh-95778: add doc missing in some places (GH-100627) Message-ID: <mailman.3140.1672435299.3313.python-checkins@python.org> https://github.com/python/cpython/commit/46521826cb1883e29e4640f94089dd92c57efc5b commit: 46521826cb1883e29e4640f94089dd92c57efc5b branch: main author: ?ric <earaujo at caravan.coop> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T13:21:15-08:00 summary: gh-95778: add doc missing in some places (GH-100627) files: M Misc/python.man M Python/initconfig.c diff --git a/Misc/python.man b/Misc/python.man index 1705eeb0c9c1..bf7cf767d164 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -358,6 +358,10 @@ Set implementation-specific option. The following options are available: -X frozen_modules=[on|off]: whether or not frozen modules should be used. The default is "on" (or "off" if you are running a local build). + -X int_max_str_digits=number: limit the size of int<->str conversions. + This helps avoid denial of service attacks when parsing untrusted data. + The default is sys.int_info.default_max_str_digits. 0 disables. + .TP .B \-x Skip the first line of the source. This is intended for a DOS @@ -531,6 +535,11 @@ values. The integer must be a decimal number in the range [0,4294967295]. Specifying the value 0 will disable hash randomization. +.IP PYTHONINTMAXSTRDIGITS +Limit the maximum digit characters in an int value +when converting from a string and when converting an int back to a str. +A value of 0 disables the limit. Conversions to or from bases 2, 4, 8, +16, and 32 are never limited. .IP PYTHONMALLOC Set the Python memory allocators and/or install debug hooks. The available memory allocators are diff --git a/Python/initconfig.c b/Python/initconfig.c index d05099cd9977..d7b2dc4a2974 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -180,6 +180,8 @@ static const char usage_envvars[] = "PYTHONDEBUG : enable parser debug mode (-d)\n" "PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" "PYTHONINSPECT : inspect interactively after running script (-i)\n" +"PYTHONINTMAXSTRDIGITS : limit max digit characters in an int value\n" +" (-X int_max_str_digits=number)\n" "PYTHONNOUSERSITE : disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" "PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" From webhook-mailer at python.org Fri Dec 30 16:51:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 21:51:11 -0000 Subject: [Python-checkins] gh-95778: add doc missing in some places (GH-100627) Message-ID: <mailman.3141.1672437072.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b1e314ab9f8c3a2b53c7179674811f9c79328ce7 commit: b1e314ab9f8c3a2b53c7179674811f9c79328ce7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T13:51:06-08:00 summary: gh-95778: add doc missing in some places (GH-100627) (cherry picked from commit 46521826cb1883e29e4640f94089dd92c57efc5b) Co-authored-by: ?ric <earaujo at caravan.coop> files: M Misc/python.man M Python/initconfig.c diff --git a/Misc/python.man b/Misc/python.man index c979f5dff3ca..07a08315d8f3 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -358,6 +358,10 @@ Set implementation-specific option. The following options are available: -X frozen_modules=[on|off]: whether or not frozen modules should be used. The default is "on" (or "off" if you are running a local build). + -X int_max_str_digits=number: limit the size of int<->str conversions. + This helps avoid denial of service attacks when parsing untrusted data. + The default is sys.int_info.default_max_str_digits. 0 disables. + .TP .B \-x Skip the first line of the source. This is intended for a DOS @@ -531,6 +535,11 @@ values. The integer must be a decimal number in the range [0,4294967295]. Specifying the value 0 will disable hash randomization. +.IP PYTHONINTMAXSTRDIGITS +Limit the maximum digit characters in an int value +when converting from a string and when converting an int back to a str. +A value of 0 disables the limit. Conversions to or from bases 2, 4, 8, +16, and 32 are never limited. .IP PYTHONMALLOC Set the Python memory allocators and/or install debug hooks. The available memory allocators are diff --git a/Python/initconfig.c b/Python/initconfig.c index 0ccca265314d..d81cbaff7ec9 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -168,6 +168,8 @@ static const char usage_envvars[] = "PYTHONDEBUG : enable parser debug mode (-d)\n" "PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" "PYTHONINSPECT : inspect interactively after running script (-i)\n" +"PYTHONINTMAXSTRDIGITS : limit max digit characters in an int value\n" +" (-X int_max_str_digits=number)\n" "PYTHONNOUSERSITE : disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" "PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" From webhook-mailer at python.org Fri Dec 30 21:36:43 2022 From: webhook-mailer at python.org (rhettinger) Date: Sat, 31 Dec 2022 02:36:43 -0000 Subject: [Python-checkins] Improve comments in itertools uniquification recipes (GH-100631) Message-ID: <mailman.3142.1672454204.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4ebaae8aedd91d2cffa5e89123f3539ddcbdfd5b commit: 4ebaae8aedd91d2cffa5e89123f3539ddcbdfd5b branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-30T20:36:38-06:00 summary: Improve comments in itertools uniquification recipes (GH-100631) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5d6efabcef60..4ad679dfccdb 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1019,15 +1019,14 @@ which incur interpreter overhead. def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D + # unique_everseen('ABBcCAD', str.lower) --> A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): seen.add(element) yield element - # Note: The steps shown above are intended to demonstrate - # filterfalse(). For order preserving deduplication, - # a better solution is: + # For order preserving deduplication, + # a faster but non-lazy solution is: # yield from dict.fromkeys(iterable) else: for element in iterable: @@ -1035,11 +1034,15 @@ which incur interpreter overhead. if k not in seen: seen.add(k) yield element + # For use cases that allow the last matching element to be returned, + # a faster but non-lazy solution is: + # t1, t2 = tee(iterable) + # yield from dict(zip(map(key, t1), t2)).values() def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBCcAD', str.lower) --> A B C A D + # unique_justseen('ABBcCAD', str.lower) --> A B c A D return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def iter_except(func, exception, first=None): @@ -1371,15 +1374,17 @@ which incur interpreter overhead. >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'D'] + >>> list(unique_everseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'A', 'D'] + >>> list(unique_justseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) From webhook-mailer at python.org Fri Dec 30 23:23:44 2022 From: webhook-mailer at python.org (rhettinger) Date: Sat, 31 Dec 2022 04:23:44 -0000 Subject: [Python-checkins] [3.11] Improve comments in itertools uniquification recipes (GH-100631) (GH-100632) Message-ID: <mailman.3143.1672460626.3313.python-checkins@python.org> https://github.com/python/cpython/commit/18006309eac07e5f42232fd4ba016640aeac5b1a commit: 18006309eac07e5f42232fd4ba016640aeac5b1a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-30T22:23:39-06:00 summary: [3.11] Improve comments in itertools uniquification recipes (GH-100631) (GH-100632) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 93bc1f792f54..fbbc96c60aba 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -989,15 +989,14 @@ which incur interpreter overhead. def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D + # unique_everseen('ABBcCAD', str.lower) --> A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): seen.add(element) yield element - # Note: The steps shown above are intended to demonstrate - # filterfalse(). For order preserving deduplication, - # a better solution is: + # For order preserving deduplication, + # a faster but non-lazy solution is: # yield from dict.fromkeys(iterable) else: for element in iterable: @@ -1005,11 +1004,15 @@ which incur interpreter overhead. if k not in seen: seen.add(k) yield element + # For use cases that allow the last matching element to be returned, + # a faster but non-lazy solution is: + # t1, t2 = tee(iterable) + # yield from dict(zip(map(key, t1), t2)).values() def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBCcAD', str.lower) --> A B C A D + # unique_justseen('ABBcCAD', str.lower) --> A B c A D return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def iter_except(func, exception, first=None): @@ -1365,15 +1368,17 @@ which incur interpreter overhead. >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'D'] + >>> list(unique_everseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'A', 'D'] + >>> list(unique_justseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) From webhook-mailer at python.org Sat Dec 31 01:24:13 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 31 Dec 2022 06:24:13 -0000 Subject: [Python-checkins] GH-85979: Clarify specification of `object.__await__` (#22320) Message-ID: <mailman.3144.1672467854.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f59c7f8edd5ba5f6c1954383542a2292bcf51d91 commit: f59c7f8edd5ba5f6c1954383542a2292bcf51d91 branch: main author: Paolo Lammens <lammenspaolo at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-31T11:54:04+05:30 summary: GH-85979: Clarify specification of `object.__await__` (#22320) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index fd682fcff020..1d2ddf3507ae 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2950,6 +2950,14 @@ are awaitable. :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + .. note:: + + The language doesn't place any restriction on the type or value of the + objects yielded by the iterator returned by ``__await__``, as this is + specific to the implementation of the asynchronous execution framework + (e.g. :mod:`asyncio`) that will be managing the :term:`awaitable` object. + + .. versionadded:: 3.5 .. seealso:: :pep:`492` for additional information about awaitable objects. From webhook-mailer at python.org Sat Dec 31 01:31:48 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 06:31:48 -0000 Subject: [Python-checkins] GH-85979: Clarify specification of `object.__await__` (GH-22320) Message-ID: <mailman.3145.1672468310.3313.python-checkins@python.org> https://github.com/python/cpython/commit/504aa92b70d3902f373dc5a6ca0b3ddc1ad233d5 commit: 504aa92b70d3902f373dc5a6ca0b3ddc1ad233d5 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T22:31:41-08:00 summary: GH-85979: Clarify specification of `object.__await__` (GH-22320) (cherry picked from commit f59c7f8edd5ba5f6c1954383542a2292bcf51d91) Co-authored-by: Paolo Lammens <lammenspaolo at gmail.com> Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index eafb6ff15998..6aed328a9fdb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2879,6 +2879,14 @@ are awaitable. :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + .. note:: + + The language doesn't place any restriction on the type or value of the + objects yielded by the iterator returned by ``__await__``, as this is + specific to the implementation of the asynchronous execution framework + (e.g. :mod:`asyncio`) that will be managing the :term:`awaitable` object. + + .. versionadded:: 3.5 .. seealso:: :pep:`492` for additional information about awaitable objects. From webhook-mailer at python.org Sat Dec 31 01:34:18 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 06:34:18 -0000 Subject: [Python-checkins] GH-85979: Clarify specification of `object.__await__` (GH-22320) Message-ID: <mailman.3146.1672468459.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f9ddbc0de5ca0689f87f18819c0bf4b7f1271726 commit: f9ddbc0de5ca0689f87f18819c0bf4b7f1271726 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T22:34:12-08:00 summary: GH-85979: Clarify specification of `object.__await__` (GH-22320) (cherry picked from commit f59c7f8edd5ba5f6c1954383542a2292bcf51d91) Co-authored-by: Paolo Lammens <lammenspaolo at gmail.com> Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 92cd2f8f8074..afd4a5477ffb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2950,6 +2950,14 @@ are awaitable. :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + .. note:: + + The language doesn't place any restriction on the type or value of the + objects yielded by the iterator returned by ``__await__``, as this is + specific to the implementation of the asynchronous execution framework + (e.g. :mod:`asyncio`) that will be managing the :term:`awaitable` object. + + .. versionadded:: 3.5 .. seealso:: :pep:`492` for additional information about awaitable objects. From webhook-mailer at python.org Sat Dec 31 04:15:35 2022 From: webhook-mailer at python.org (methane) Date: Sat, 31 Dec 2022 09:15:35 -0000 Subject: [Python-checkins] gh-94808: Improve coverage of dictresize (GH-100619) Message-ID: <mailman.3147.1672478136.3313.python-checkins@python.org> https://github.com/python/cpython/commit/636e9dd23f88c701eecf91156835fe0fc8b1feb6 commit: 636e9dd23f88c701eecf91156835fe0fc8b1feb6 branch: main author: tqxia <44689929+tqxia at users.noreply.github.com> committer: methane <songofacandy at gmail.com> date: 2022-12-31T18:15:30+09:00 summary: gh-94808: Improve coverage of dictresize (GH-100619) files: M Lib/test/test_dict.py diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 5b8baaf9e6e2..79638340059f 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1094,6 +1094,21 @@ def __init__(self, order): d.update(o.__dict__) self.assertEqual(list(d), ["c", "b", "a"]) + @support.cpython_only + def test_splittable_to_generic_combinedtable(self): + """split table must be correctly resized and converted to generic combined table""" + class C: + pass + + a = C() + a.x = 1 + d = a.__dict__ + before_resize = sys.getsizeof(d) + d[2] = 2 # split table is resized to a generic combined table + + self.assertGreater(sys.getsizeof(d), before_resize) + self.assertEqual(list(d), ['x', 2]) + def test_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): data = {1:"a", 2:"b", 3:"c"} From webhook-mailer at python.org Sat Dec 31 05:23:38 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 31 Dec 2022 10:23:38 -0000 Subject: [Python-checkins] gh-100633 Tutorial: Fix dataclasses import (#100638) Message-ID: <mailman.3148.1672482219.3313.python-checkins@python.org> https://github.com/python/cpython/commit/98308dbeb110198ebe28bdb7720d3671b3e7f57b commit: 98308dbeb110198ebe28bdb7720d3671b3e7f57b branch: main author: Owain Davies <116417456+OTheDev at users.noreply.github.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-31T10:23:18Z summary: gh-100633 Tutorial: Fix dataclasses import (#100638) import dataclass not dataclasses from dataclasses files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index a206ba371976..116801177a3a 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -741,7 +741,7 @@ Sometimes it is useful to have a data type similar to the Pascal "record" or C "struct", bundling together a few named data items. The idiomatic approach is to use :mod:`dataclasses` for this purpose:: - from dataclasses import dataclasses + from dataclasses import dataclass @dataclass class Employee: From webhook-mailer at python.org Sat Dec 31 07:28:48 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 12:28:48 -0000 Subject: [Python-checkins] gh-100633 Tutorial: Fix dataclasses import (GH-100638) Message-ID: <mailman.3149.1672489728.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f7ad4ffa563fbe14d6b0020c45720bebde73de62 commit: f7ad4ffa563fbe14d6b0020c45720bebde73de62 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T04:28:36-08:00 summary: gh-100633 Tutorial: Fix dataclasses import (GH-100638) import dataclass not dataclasses from dataclasses (cherry picked from commit 98308dbeb110198ebe28bdb7720d3671b3e7f57b) Co-authored-by: Owain Davies <116417456+OTheDev at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 5abb767cb0a8..30450c70f3e8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -740,7 +740,7 @@ Sometimes it is useful to have a data type similar to the Pascal "record" or C "struct", bundling together a few named data items. The idiomatic approach is to use :mod:`dataclasses` for this purpose:: - from dataclasses import dataclasses + from dataclasses import dataclass @dataclass class Employee: From webhook-mailer at python.org Sat Dec 31 07:29:05 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 12:29:05 -0000 Subject: [Python-checkins] gh-100633 Tutorial: Fix dataclasses import (GH-100638) Message-ID: <mailman.3150.1672489746.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f80ba44f7d01b153b12eb7573568f9b1f33654ee commit: f80ba44f7d01b153b12eb7573568f9b1f33654ee branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T04:28:59-08:00 summary: gh-100633 Tutorial: Fix dataclasses import (GH-100638) import dataclass not dataclasses from dataclasses (cherry picked from commit 98308dbeb110198ebe28bdb7720d3671b3e7f57b) Co-authored-by: Owain Davies <116417456+OTheDev at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 5abb767cb0a8..30450c70f3e8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -740,7 +740,7 @@ Sometimes it is useful to have a data type similar to the Pascal "record" or C "struct", bundling together a few named data items. The idiomatic approach is to use :mod:`dataclasses` for this purpose:: - from dataclasses import dataclasses + from dataclasses import dataclass @dataclass class Employee: From webhook-mailer at python.org Sat Dec 31 11:58:04 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 31 Dec 2022 16:58:04 -0000 Subject: [Python-checkins] GH-87002: fix caching documentation in `struct` module (#24164) Message-ID: <mailman.3151.1672505885.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ac7a0a9f9e9a749ebcdd9d1ac21f73449d279958 commit: ac7a0a9f9e9a749ebcdd9d1ac21f73449d279958 branch: main author: Sandeep Subramanian <sandeepsubramanian94 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-31T22:27:58+05:30 summary: GH-87002: fix caching documentation in `struct` module (#24164) files: M Doc/library/struct.rst diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 50d70731f775..830ba69e294d 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -548,9 +548,9 @@ The :mod:`struct` module also defines the following type: .. note:: The compiled versions of the most recent format strings passed to - :class:`Struct` and the module-level functions are cached, so programs - that use only a few format strings needn't worry about reusing a single - :class:`Struct` instance. + the module-level functions are cached, so programs that use only a few + format strings needn't worry about reusing a single :class:`Struct` + instance. Compiled Struct objects support the following methods and attributes: From webhook-mailer at python.org Sat Dec 31 12:10:37 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 31 Dec 2022 17:10:37 -0000 Subject: [Python-checkins] Fix `pydtrace.d` path comment in `Include/pydtrace.h` (#28539) Message-ID: <mailman.3152.1672506638.3313.python-checkins@python.org> https://github.com/python/cpython/commit/baf234971f20e5e6c7438f46889e9c8f172cc25e commit: baf234971f20e5e6c7438f46889e9c8f172cc25e branch: main author: Mark Hansen <mark at markhansen.co.nz> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-31T22:40:25+05:30 summary: Fix `pydtrace.d` path comment in `Include/pydtrace.h` (#28539) files: M Include/pydtrace.h diff --git a/Include/pydtrace.h b/Include/pydtrace.h index 75f8e7f70979..e197d3669453 100644 --- a/Include/pydtrace.h +++ b/Include/pydtrace.h @@ -12,7 +12,7 @@ extern "C" { /* pydtrace_probes.h, on systems with DTrace, is auto-generated to include `PyDTrace_{PROBE}` and `PyDTrace_{PROBE}_ENABLED()` macros for every probe - defined in pydtrace_provider.d. + defined in pydtrace.d. Calling these functions must be guarded by a `PyDTrace_{PROBE}_ENABLED()` check to minimize performance impact when probing is off. For example: From webhook-mailer at python.org Sat Dec 31 19:01:49 2022 From: webhook-mailer at python.org (terryjreedy) Date: Sun, 01 Jan 2023 00:01:49 -0000 Subject: [Python-checkins] IDLE - fix module browser test (#100647) Message-ID: <mailman.3153.1672531311.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1f6c87ca7b9351b2e5c5363504796fce0554c9b8 commit: 1f6c87ca7b9351b2e5c5363504796fce0554c9b8 branch: main author: Terry Jan Reedy <tjreedy at udel.edu> committer: terryjreedy <tjreedy at udel.edu> date: 2022-12-31T19:01:44-05:00 summary: IDLE - fix module browser test (#100647) files: M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 343d50a6e37b..6cfea3888cd6 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -170,8 +170,7 @@ def test_ondoubleclick(self, fopen): with mock.patch('os.path.exists', return_value=True): mbt.OnDoubleClick() - fopen.assert_called() - fopen.called_with(fname) + fopen.assert_called_once_with(fname) class ChildBrowserTreeItemTest(unittest.TestCase): From webhook-mailer at python.org Sat Dec 31 19:20:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 01 Jan 2023 00:20:59 -0000 Subject: [Python-checkins] IDLE - fix module browser test (GH-100647) Message-ID: <mailman.3154.1672532459.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0d57f10e40a4c4053169fe743882118e2bf3dee8 commit: 0d57f10e40a4c4053169fe743882118e2bf3dee8 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T16:20:53-08:00 summary: IDLE - fix module browser test (GH-100647) (cherry picked from commit 1f6c87ca7b9351b2e5c5363504796fce0554c9b8) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 343d50a6e37b..6cfea3888cd6 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -170,8 +170,7 @@ def test_ondoubleclick(self, fopen): with mock.patch('os.path.exists', return_value=True): mbt.OnDoubleClick() - fopen.assert_called() - fopen.called_with(fname) + fopen.assert_called_once_with(fname) class ChildBrowserTreeItemTest(unittest.TestCase): From webhook-mailer at python.org Sat Dec 31 19:31:58 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 01 Jan 2023 00:31:58 -0000 Subject: [Python-checkins] IDLE - fix module browser test (GH-100647) Message-ID: <mailman.3155.1672533120.3313.python-checkins@python.org> https://github.com/python/cpython/commit/83f85539a1559de285d2767c791235876ccda2d3 commit: 83f85539a1559de285d2767c791235876ccda2d3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T16:31:53-08:00 summary: IDLE - fix module browser test (GH-100647) (cherry picked from commit 1f6c87ca7b9351b2e5c5363504796fce0554c9b8) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 343d50a6e37b..6cfea3888cd6 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -170,8 +170,7 @@ def test_ondoubleclick(self, fopen): with mock.patch('os.path.exists', return_value=True): mbt.OnDoubleClick() - fopen.assert_called() - fopen.called_with(fname) + fopen.assert_called_once_with(fname) class ChildBrowserTreeItemTest(unittest.TestCase): From webhook-mailer at python.org Sat Dec 31 19:59:52 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sun, 01 Jan 2023 00:59:52 -0000 Subject: [Python-checkins] gh-100546: Remove incorrect positional-only marker from eval (#100547) Message-ID: <mailman.3156.1672534793.3313.python-checkins@python.org> https://github.com/python/cpython/commit/71159a8e078bda0c9a39c6cd0980b7ba238dc582 commit: 71159a8e078bda0c9a39c6cd0980b7ba238dc582 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-31T17:59:31-07:00 summary: gh-100546: Remove incorrect positional-only marker from eval (#100547) All the arguments are positional-only. The current status after #99476 seems to be to not use positional-only markers in documentation, hence I've simply removed it. files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index f00a072e6f41..77ebdd0367c3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -513,7 +513,7 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, /, globals=None, locals=None) +.. function:: eval(expression, globals=None, locals=None) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping From webhook-mailer at python.org Sat Dec 31 20:06:37 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 01 Jan 2023 01:06:37 -0000 Subject: [Python-checkins] gh-100546: Remove incorrect positional-only marker from eval (GH-100547) Message-ID: <mailman.3157.1672535198.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7ed71278768f2646af0a74caef6648c385d9f334 commit: 7ed71278768f2646af0a74caef6648c385d9f334 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T17:06:31-08:00 summary: gh-100546: Remove incorrect positional-only marker from eval (GH-100547) All the arguments are positional-only. The current status after GH-99476 seems to be to not use positional-only markers in documentation, hence I've simply removed it. (cherry picked from commit 71159a8e078bda0c9a39c6cd0980b7ba238dc582) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c71ebd235cf3..e9e3177a99c7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -512,7 +512,7 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, /, globals=None, locals=None) +.. function:: eval(expression, globals=None, locals=None) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping

From webhook-mailer at python.org Thu Dec 1 02:19:51 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 01 Dec 2022 07:19:51 -0000 Subject: [Python-checkins] Doc: Add summary line to isolation_level & autocommit sqlite3.connect params (#99917) Message-ID: https://github.com/python/cpython/commit/d460c8ec52716a37080d31fdc0f673edcc98bee8 commit: d460c8ec52716a37080d31fdc0f673edcc98bee8 branch: main author: C.A.M. Gerlach committer: erlend-aasland date: 2022-12-01T08:19:41+01:00 summary: Doc: Add summary line to isolation_level & autocommit sqlite3.connect params (#99917) Add summary lines to isolation_level and autocommit connect() params Co-authored-by: G?ry Ogam files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index a9ee1e207d0c..960f2966afe1 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -292,6 +292,7 @@ Module functions By default (``0``), type detection is disabled. :param isolation_level: + Control legacy transaction handling behaviour. See :attr:`Connection.isolation_level` and :ref:`sqlite3-transaction-control-isolation-level` for more information. Can be ``"DEFERRED"`` (default), ``"EXCLUSIVE"`` or ``"IMMEDIATE"``; @@ -325,6 +326,7 @@ Module functions enabling various :ref:`sqlite3-uri-tricks`. :param autocommit: + Control :pep:`249` transaction handling behaviour. See :attr:`Connection.autocommit` and :ref:`sqlite3-transaction-control-autocommit` for more information. *autocommit* currently defaults to From webhook-mailer at python.org Thu Dec 1 03:57:09 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 01 Dec 2022 08:57:09 -0000 Subject: [Python-checkins] [3.11] gh-99891: Fix infinite recursion in the tokenizer when showing warnings (GH-99893) (GH-99896) Message-ID: https://github.com/python/cpython/commit/6282ef6c3f07b1effbf51d5806f92791751d362e commit: 6282ef6c3f07b1effbf51d5806f92791751d362e branch: 3.11 author: Pablo Galindo Salgado committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-01T00:57:04-08:00 summary: [3.11] gh-99891: Fix infinite recursion in the tokenizer when showing warnings (GH-99893) (GH-99896) Automerge-Triggered-By: GH:pablogsal. (cherry picked from commit 417206a05c4545bde96c2bbbea92b53e6cac0d48) Co-authored-by: Pablo Galindo Salgado files: A Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst M Lib/test/test_source_encoding.py M Parser/tokenizer.c M Parser/tokenizer.h diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index e357264eb1d1..5fe0f3124444 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -161,6 +161,18 @@ def test_file_parse_error_multiline(self): finally: os.unlink(TESTFN) + def test_tokenizer_fstring_warning_in_first_line(self): + source = "0b1and 2" + with open(TESTFN, "w") as fd: + fd.write("{}".format(source)) + try: + retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN) + self.assertIn(b"SyntaxWarning: invalid binary litera", stderr) + self.assertEqual(stderr.count(source.encode()), 1) + finally: + os.unlink(TESTFN) + + class AbstractSourceEncodingTest: def test_default_coding(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst new file mode 100644 index 000000000000..20cd361affea --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst @@ -0,0 +1,3 @@ +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index eda38a09a995..ca11c7bebb4e 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -88,6 +88,7 @@ tok_new(void) tok->async_def_nl = 0; tok->interactive_underflow = IUNDERFLOW_NORMAL; tok->str = NULL; + tok->report_warnings = 1; return tok; } @@ -1186,6 +1187,10 @@ indenterror(struct tok_state *tok) static int parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { + if (!tok->report_warnings) { + return 0; + } + PyObject *errmsg; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES @@ -2194,6 +2199,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) return encoding; } } + // We don't want to report warnings here because it could cause infinite recursion + // if fetching the encoding shows a warning. + tok->report_warnings = 0; while (tok->lineno < 2 && tok->done == E_OK) { _PyTokenizer_Get(tok, &p_start, &p_end); } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0cb665104b2b..d9a5f457d9c5 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -84,6 +84,7 @@ struct tok_state { NEWLINE token after it. */ /* How to proceed when asked for a new token in interactive mode */ enum interactive_underflow_t interactive_underflow; + int report_warnings; }; extern struct tok_state *_PyTokenizer_FromString(const char *, int); From webhook-mailer at python.org Thu Dec 1 05:27:04 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 01 Dec 2022 10:27:04 -0000 Subject: [Python-checkins] =?utf-8?q?=5B3=2E11=5D_gh-99086=3A_Fix_-Wstric?= =?utf-8?q?t-prototypes=2C_-Wimplicit-function-declara=E2=80=A6_=28GH-9991?= =?utf-8?q?9=29?= Message-ID: https://github.com/python/cpython/commit/88b754b605adbb87325cd797c65b4eae2cf25573 commit: 88b754b605adbb87325cd797c65b4eae2cf25573 branch: 3.11 author: Sam James committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-01T02:26:56-08:00 summary: [3.11] gh-99086: Fix -Wstrict-prototypes, -Wimplicit-function-declara? (GH-99919) ?tion warnings in configure.ac (GH-99406) Follow up to 12078e78f6e4a21f344e4eaff529e1ff3b97734f. (cherry picked from commit e35ca417fe81a64985c2b29e863ce418ae75b96e) files: A Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst M configure M configure.ac diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst new file mode 100644 index 000000000000..2dace165ca1a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst @@ -0,0 +1 @@ +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/configure b/configure index cb8eb0d3fb20..9567ac3a62c8 100755 --- a/configure +++ b/configure @@ -6681,7 +6681,7 @@ if test "x$enable_profiling" = xyes; then CC="$CC -pg" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() { return 0; } +int main(void) { return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : @@ -8925,7 +8925,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -8980,7 +8980,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -9029,7 +9029,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -9078,7 +9078,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -11834,7 +11834,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -14144,6 +14144,7 @@ $as_echo_n "checking for pthread_create in -lpthread... " >&6; } /* end confdefs.h. */ #include +#include #include void * start_routine (void *arg) { exit (0); } @@ -14454,7 +14455,7 @@ else void *foo(void *parm) { return NULL; } - int main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -16058,7 +16059,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -16107,7 +16108,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -18801,7 +18802,7 @@ else #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -20044,7 +20045,7 @@ else #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -20823,7 +20824,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -21269,7 +21270,7 @@ else #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -21311,7 +21312,7 @@ else #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -21368,7 +21369,7 @@ else extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -22248,9 +22249,10 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -22449,7 +22451,7 @@ else #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -22504,7 +22506,7 @@ else ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -22547,7 +22549,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -22588,7 +22590,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -22664,7 +22666,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -22702,11 +22704,12 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -22749,9 +22752,10 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; diff --git a/configure.ac b/configure.ac index 948fd36aa805..90008bcae17f 100644 --- a/configure.ac +++ b/configure.ac @@ -1435,7 +1435,7 @@ AC_ARG_ENABLE(profiling, if test "x$enable_profiling" = xyes; then ac_save_cc="$CC" CC="$CC -pg" - AC_LINK_IFELSE([AC_LANG_SOURCE([[int main() { return 0; }]])], + AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void) { return 0; }]])], [], [enable_profiling=no]) CC="$ac_save_cc" @@ -2511,7 +2511,7 @@ AC_CACHE_CHECK([whether pthreads are available without options], void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2544,7 +2544,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2571,7 +2571,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2598,7 +2598,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -3489,7 +3489,7 @@ esac # check for systems that require aligned memory access AC_CACHE_CHECK([aligned memory access is required], [ac_cv_aligned_required], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -4103,6 +4103,7 @@ yes AC_MSG_CHECKING([for pthread_create in -lpthread]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include +#include #include void * start_routine (void *arg) { exit (0); }]], [[ @@ -4172,7 +4173,7 @@ if test "$posix_threads" = "yes"; then void *foo(void *parm) { return NULL; } - int main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -4729,7 +4730,7 @@ AC_CACHE_CHECK([for chflags], [ac_cv_have_chflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -4751,7 +4752,7 @@ AC_CACHE_CHECK([for lchflags], [ac_cv_have_lchflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -5024,7 +5025,7 @@ AS_VAR_IF([ac_cv_func_getaddrinfo], [yes], [ #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -5461,7 +5462,7 @@ CC="$CC $BASECFLAGS" AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -5768,7 +5769,7 @@ fi], # or fills with zeros (like the Cray J90, according to Tim Peters). AC_CACHE_CHECK([whether right shift extends the sign bit], [ac_cv_rshift_extends_sign], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -5923,7 +5924,7 @@ AC_CACHE_CHECK([for broken nice()], [ac_cv_broken_nice], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -5945,7 +5946,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -5981,7 +5982,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -6254,9 +6255,10 @@ AC_CHECK_TYPE(socklen_t,, AC_CACHE_CHECK([for broken mbstowcs], [ac_cv_broken_mbstowcs], AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -6382,7 +6384,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -6423,7 +6425,7 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -6451,7 +6453,7 @@ AC_LINK_IFELSE( #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -6472,7 +6474,7 @@ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -6513,7 +6515,7 @@ AC_LINK_IFELSE( AC_LANG_SOURCE([[ #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -6531,11 +6533,12 @@ AC_CACHE_CHECK([for the Linux getrandom() syscall], [ac_cv_getrandom_syscall], [ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -6558,9 +6561,10 @@ AC_CACHE_CHECK([for the getrandom() function], [ac_cv_func_getrandom], [ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; From webhook-mailer at python.org Thu Dec 1 06:02:55 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 01 Dec 2022 11:02:55 -0000 Subject: [Python-checkins] =?utf-8?q?=5B3=2E10=5D_GH-99086=3A_Fix_-Wstric?= =?utf-8?q?t-prototypes=2C_-Wimplicit-function-declara=E2=80=A6_=28=239992?= =?utf-8?q?0=29?= Message-ID: https://github.com/python/cpython/commit/807b103bddc4d466f3cfe78aed74c891617c9631 commit: 807b103bddc4d466f3cfe78aed74c891617c9631 branch: 3.10 author: Sam James committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-01T16:32:49+05:30 summary: [3.10] GH-99086: Fix -Wstrict-prototypes, -Wimplicit-function-declara? (#99920) [3.10] gh-99086: Fix -Wstrict-prototypes, -Wimplicit-function-declaration warnings in configure.ac (GH-99406) Follow up to 12078e78f6e4a21f344e4eaff529e1ff3b97734f. (cherry picked from commit e35ca417fe81a64985c2b29e863ce418ae75b96e) Co-authored-by: Sam James files: A Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst M configure M configure.ac diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst new file mode 100644 index 000000000000..2dace165ca1a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst @@ -0,0 +1 @@ +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/configure b/configure index abd9982ba44d..4b71c4e00f89 100755 --- a/configure +++ b/configure @@ -5929,7 +5929,7 @@ if test "x$enable_profiling" = xyes; then CC="$CC -pg" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() { return 0; } +int main(void) { return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : @@ -7752,7 +7752,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -7808,7 +7808,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -7858,7 +7858,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -7908,7 +7908,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -10447,7 +10447,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -11024,6 +11024,7 @@ $as_echo_n "checking for pthread_create in -lpthread... " >&6; } /* end confdefs.h. */ #include +#include #include void * start_routine (void *arg) { exit (0); } @@ -11325,7 +11326,7 @@ else void *foo(void *parm) { return NULL; } - int main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -12687,7 +12688,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -12736,7 +12737,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -13653,7 +13654,7 @@ else #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -14880,7 +14881,7 @@ else #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -15730,7 +15731,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -16178,7 +16179,7 @@ else #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -16221,7 +16222,7 @@ else #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -16279,7 +16280,7 @@ else extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -17040,9 +17041,10 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -17219,7 +17221,7 @@ else #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -17274,7 +17276,7 @@ else ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -17313,7 +17315,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -17347,7 +17349,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -17406,7 +17408,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -17436,11 +17438,12 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -17475,9 +17478,10 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; diff --git a/configure.ac b/configure.ac index 540171f86171..ac3be3850a9b 100644 --- a/configure.ac +++ b/configure.ac @@ -1099,7 +1099,7 @@ AC_ARG_ENABLE(profiling, if test "x$enable_profiling" = xyes; then ac_save_cc="$CC" CC="$CC -pg" - AC_LINK_IFELSE([AC_LANG_SOURCE([[int main() { return 0; }]])], + AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void) { return 0; }]])], [], [enable_profiling=no]) CC="$ac_save_cc" @@ -2057,7 +2057,7 @@ AC_CACHE_VAL(ac_cv_pthread_is_default, void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2092,7 +2092,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2121,7 +2121,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2150,7 +2150,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2991,7 +2991,7 @@ esac AC_MSG_CHECKING(aligned memory access is required) AC_CACHE_VAL(ac_cv_aligned_required, [AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -3279,6 +3279,7 @@ yes AC_MSG_CHECKING([for pthread_create in -lpthread]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include +#include #include void * start_routine (void *arg) { exit (0); }]], [[ @@ -3344,7 +3345,7 @@ if test "$posix_threads" = "yes"; then void *foo(void *parm) { return NULL; } - int main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -3912,7 +3913,7 @@ AC_CACHE_CHECK([for chflags], [ac_cv_have_chflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -3934,7 +3935,7 @@ AC_CACHE_CHECK([for lchflags], [ac_cv_have_lchflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -4135,7 +4136,7 @@ then #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -4593,7 +4594,7 @@ CC="$CC $BASECFLAGS" AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -4910,7 +4911,7 @@ fi], AC_MSG_CHECKING(whether right shift extends the sign bit) AC_CACHE_VAL(ac_cv_rshift_extends_sign, [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -5069,7 +5070,7 @@ AC_CACHE_VAL(ac_cv_broken_nice, [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -5093,7 +5094,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -5131,7 +5132,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -5487,9 +5488,10 @@ AC_CHECK_TYPE(socklen_t,, AC_MSG_CHECKING(for broken mbstowcs) AC_CACHE_VAL(ac_cv_broken_mbstowcs, AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -5600,7 +5602,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -5641,7 +5643,7 @@ if test "$have_gcc_asm_for_x87" = yes; then ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -5669,7 +5671,7 @@ AC_LINK_IFELSE( #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -5691,7 +5693,7 @@ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -5727,7 +5729,7 @@ AC_LINK_IFELSE( AC_LANG_SOURCE([[ #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -5745,11 +5747,12 @@ AC_MSG_CHECKING(for the Linux getrandom() syscall) AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -5772,9 +5775,10 @@ AC_MSG_CHECKING(for the getrandom() function) AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; From webhook-mailer at python.org Thu Dec 1 07:55:11 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Thu, 01 Dec 2022 12:55:11 -0000 Subject: [Python-checkins] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) Message-ID: https://github.com/python/cpython/commit/f08e52ccb027f6f703302b8c1a82db9fd3934270 commit: f08e52ccb027f6f703302b8c1a82db9fd3934270 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2022-12-01T14:54:51+02:00 summary: gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) Previously *consumed was not set in this case. files: A Lib/test/test_capi/test_codecs.py A Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst M Modules/_testcapi/unicode.c M Objects/unicodeobject.c diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py new file mode 100644 index 000000000000..e46726192aa0 --- /dev/null +++ b/Lib/test/test_capi/test_codecs.py @@ -0,0 +1,54 @@ +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class CAPITest(unittest.TestCase): + + def test_decodeutf8(self): + """Test PyUnicode_DecodeUTF8()""" + decodeutf8 = _testcapi.unicode_decodeutf8 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8(b), s) + self.assertEqual(decodeutf8(b, 'strict'), s) + + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') + self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') + self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and + # negative size. + + def test_decodeutf8stateful(self): + """Test PyUnicode_DecodeUTF8Stateful()""" + decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8stateful(b), (s, len(b))) + self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) + + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) + + self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of + # "consumed". + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst new file mode 100644 index 000000000000..40e3c8db5403 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst @@ -0,0 +1,2 @@ +Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: +``*consumed`` was not set. diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 4c5049dd406a..2d23993ce420 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -239,6 +239,40 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, utf8_len); } +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = 123456789; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + /* Test PyUnicode_Concat() */ static PyObject * unicode_concat(PyObject *self, PyObject *args) @@ -1025,6 +1059,8 @@ static PyMethodDef TestMethods[] = { {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, {"unicode_concat", unicode_concat, METH_VARARGS}, {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, {"unicode_split", unicode_split, METH_VARARGS}, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 55f029dd504c..19bde13a6f23 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4530,6 +4530,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, } s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); if (s == end) { + if (consumed) { + *consumed = size; + } return u; } From webhook-mailer at python.org Thu Dec 1 08:06:03 2022 From: webhook-mailer at python.org (vstinner) Date: Thu, 01 Dec 2022 13:06:03 -0000 Subject: [Python-checkins] gh-99894: Ensure the local names don't collide with the test file in traceback suggestion error checking (#99895) Message-ID: https://github.com/python/cpython/commit/0563be23a557917228a8b48cbb31bda285a3a815 commit: 0563be23a557917228a8b48cbb31bda285a3a815 branch: main author: Pablo Galindo Salgado committer: vstinner date: 2022-12-01T14:05:56+01:00 summary: gh-99894: Ensure the local names don't collide with the test file in traceback suggestion error checking (#99895) Co-authored-by: Victor Stinner files: M Lib/test/test_traceback.py diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index c17bbb48b65b..95b1bae4f608 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -2978,9 +2978,9 @@ class MyClass: for name in ("b", "v", "m", "py"): with self.subTest(name=name): actual = self.get_suggestion(MyClass, name) - self.assertNotIn("you mean", actual) - self.assertNotIn("vvv", actual) - self.assertNotIn("mom", actual) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv", actual) + self.assertNotIn("'mom'", actual) self.assertNotIn("'id'", actual) self.assertNotIn("'w'", actual) self.assertNotIn("'pytho'", actual) @@ -3168,9 +3168,9 @@ def test_import_from_error_bad_suggestions_do_not_trigger_for_small_names(self): for name in ("b", "v", "m", "py"): with self.subTest(name=name): actual = self.get_import_from_suggestion(code, name) - self.assertNotIn("you mean", actual) - self.assertNotIn("vvv", actual) - self.assertNotIn("mom", actual) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv'", actual) + self.assertNotIn("'mom'", actual) self.assertNotIn("'id'", actual) self.assertNotIn("'w'", actual) self.assertNotIn("'pytho'", actual) From webhook-mailer at python.org Thu Dec 1 08:08:04 2022 From: webhook-mailer at python.org (vstinner) Date: Thu, 01 Dec 2022 13:08:04 -0000 Subject: [Python-checkins] gh-99845: _PyObject_DictPointer(): fix dictoffset cast (#99922) Message-ID: https://github.com/python/cpython/commit/9707bf228e008485a3fbb63aa7ee28cf88014f91 commit: 9707bf228e008485a3fbb63aa7ee28cf88014f91 branch: 3.11 author: Victor Stinner committer: vstinner date: 2022-12-01T14:07:58+01:00 summary: gh-99845: _PyObject_DictPointer(): fix dictoffset cast (#99922) Cast size_t to Py_ssize_t, rather than casting it to long. On 64-bit Windows, long is 32-bit whereas Py_ssize_t is 64-bit. files: M Objects/object.c diff --git a/Objects/object.c b/Objects/object.c index 6a22f27de0b1..c4f2786c50a0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1076,8 +1076,9 @@ _PyObject_DictPointer(PyObject *obj) tsize = -tsize; } size_t size = _PyObject_VAR_SIZE(tp, tsize); + assert(size <= (size_t)PY_SSIZE_T_MAX); + dictoffset += (Py_ssize_t)size; - dictoffset += (long)size; _PyObject_ASSERT(obj, dictoffset > 0); _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0); } From webhook-mailer at python.org Thu Dec 1 08:32:21 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 01 Dec 2022 13:32:21 -0000 Subject: [Python-checkins] gh-99845: _PyObject_DictPointer(): fix dictoffset cast (GH-99922) Message-ID: https://github.com/python/cpython/commit/64dae2efd5a083d342d744d40ca8d6ebb28bc771 commit: 64dae2efd5a083d342d744d40ca8d6ebb28bc771 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-01T05:32:15-08:00 summary: gh-99845: _PyObject_DictPointer(): fix dictoffset cast (GH-99922) Cast size_t to Py_ssize_t, rather than casting it to long. On 64-bit Windows, long is 32-bit whereas Py_ssize_t is 64-bit. (cherry picked from commit 9707bf228e008485a3fbb63aa7ee28cf88014f91) Co-authored-by: Victor Stinner files: M Objects/object.c diff --git a/Objects/object.c b/Objects/object.c index 6d80d6df7410..0bef2e9dfb5e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1091,8 +1091,9 @@ _PyObject_GetDictPtr(PyObject *obj) tsize = -tsize; } size_t size = _PyObject_VAR_SIZE(tp, tsize); + assert(size <= (size_t)PY_SSIZE_T_MAX); + dictoffset += (Py_ssize_t)size; - dictoffset += (long)size; _PyObject_ASSERT(obj, dictoffset > 0); _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0); } From webhook-mailer at python.org Fri Dec 2 05:14:10 2022 From: webhook-mailer at python.org (zooba) Date: Fri, 02 Dec 2022 10:14:10 -0000 Subject: [Python-checkins] bpo-40882: Fix a memory leak in SharedMemory on Windows (GH-20684) Message-ID: https://github.com/python/cpython/commit/b027dd78bbdb66f2995bb898af304e66e5508bf6 commit: b027dd78bbdb66f2995bb898af304e66e5508bf6 branch: 3.10 author: Luke Garland committer: zooba date: 2022-12-02T10:13:33Z summary: bpo-40882: Fix a memory leak in SharedMemory on Windows (GH-20684) In multiprocessing.shared_memory.SharedMemory(), the temporary view returned by MapViewOfFile() should be unmapped when it is no longer needed. (cherry picked from commit 85c128e34daec7625b74746e127afa25888ccde1) Co-authored-by: Zackery Spytz files: A Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst M Lib/multiprocessing/shared_memory.py M Modules/_winapi.c M Modules/clinic/_winapi.c.h diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 881f2001dd59..9a1e5aa17b87 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -173,7 +173,10 @@ def __init__(self, name=None, create=False, size=0): ) finally: _winapi.CloseHandle(h_map) - size = _winapi.VirtualQuerySize(p_buf) + try: + size = _winapi.VirtualQuerySize(p_buf) + finally: + _winapi.UnmapViewOfFile(p_buf) self._mmap = mmap.mmap(-1, size, tagname=name) self._size = size diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst new file mode 100644 index 000000000000..2670aeef9a25 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst @@ -0,0 +1,2 @@ +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 9b30a9003261..f6bb07fd8b06 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1402,6 +1402,30 @@ _winapi_MapViewOfFile_impl(PyObject *module, HANDLE file_map, return address; } +/*[clinic input] +_winapi.UnmapViewOfFile + + address: LPCVOID + / +[clinic start generated code]*/ + +static PyObject * +_winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address) +/*[clinic end generated code: output=4f7e18ac75d19744 input=8c4b6119ad9288a3]*/ +{ + BOOL success; + + Py_BEGIN_ALLOW_THREADS + success = UnmapViewOfFile(address); + Py_END_ALLOW_THREADS + + if (!success) { + return PyErr_SetFromWindowsErr(0); + } + + Py_RETURN_NONE; +} + /*[clinic input] _winapi.OpenFileMapping -> HANDLE @@ -2095,6 +2119,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_READFILE_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF + _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF _WINAPI_WAITNAMEDPIPE_METHODDEF _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index f568357142d3..e3ed148ad70b 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -731,6 +731,32 @@ _winapi_MapViewOfFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_winapi_UnmapViewOfFile__doc__, +"UnmapViewOfFile($module, address, /)\n" +"--\n" +"\n"); + +#define _WINAPI_UNMAPVIEWOFFILE_METHODDEF \ + {"UnmapViewOfFile", (PyCFunction)_winapi_UnmapViewOfFile, METH_O, _winapi_UnmapViewOfFile__doc__}, + +static PyObject * +_winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address); + +static PyObject * +_winapi_UnmapViewOfFile(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + LPCVOID address; + + if (!PyArg_Parse(arg, "" F_POINTER ":UnmapViewOfFile", &address)) { + goto exit; + } + return_value = _winapi_UnmapViewOfFile_impl(module, address); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_OpenFileMapping__doc__, "OpenFileMapping($module, desired_access, inherit_handle, name, /)\n" "--\n" @@ -1216,4 +1242,4 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=d76d0a5901db2e2a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=acabf8f2b5cc44a1 input=a9049054013a1b77]*/ From webhook-mailer at python.org Fri Dec 2 12:28:58 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 02 Dec 2022 17:28:58 -0000 Subject: [Python-checkins] GH-91054: Add code object watchers API (GH-99859) Message-ID: https://github.com/python/cpython/commit/3c137dc613c860f605d3520d7fd722cd8ed79da6 commit: 3c137dc613c860f605d3520d7fd722cd8ed79da6 branch: main author: Itamar Ostricher committer: markshannon date: 2022-12-02T17:28:27Z summary: GH-91054: Add code object watchers API (GH-99859) * Add API to allow extensions to set callback function on creation and destruction of PyCodeObject Co-authored-by: Ye11ow-Flash files: A Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst M Doc/c-api/code.rst M Doc/whatsnew/3.12.rst M Include/cpython/code.h M Include/internal/pycore_code.h M Include/internal/pycore_interp.h M Lib/test/test_capi/test_watchers.py M Misc/ACKS M Modules/_testcapi/watchers.c M Objects/codeobject.c M Python/pystate.c diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 9054e7ee3181..a6eb86f1a0b5 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -115,3 +115,51 @@ bound into a function. the free variables. On error, ``NULL`` is returned and an exception is raised. .. versionadded:: 3.11 + +.. c:function:: int PyCode_AddWatcher(PyCode_WatchCallback callback) + + Register *callback* as a code object watcher for the current interpreter. + Return an ID which may be passed to :c:func:`PyCode_ClearWatcher`. + In case of error (e.g. no more watcher IDs available), + return ``-1`` and set an exception. + + .. versionadded:: 3.12 + +.. c:function:: int PyCode_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* previously returned from + :c:func:`PyCode_AddWatcher` for the current interpreter. + Return ``0`` on success, or ``-1`` and set an exception on error + (e.g. if the given *watcher_id* was never registered.) + + .. versionadded:: 3.12 + +.. c:type:: PyCodeEvent + + Enumeration of possible code object watcher events: + - ``PY_CODE_EVENT_CREATE`` + - ``PY_CODE_EVENT_DESTROY`` + + .. versionadded:: 3.12 + +.. c:type:: int (*PyCode_WatchCallback)(PyCodeEvent event, PyCodeObject* co) + + Type of a code object watcher callback function. + + If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked + after `co` has been fully initialized. Otherwise, the callback is invoked + before the destruction of *co* takes place, so the prior state of *co* + can be inspected. + + Users of this API should not rely on internal runtime implementation + details. Such details may include, but are not limited to, the exact + order and timing of creation and destruction of code objects. While + changes in these details may result in differences observable by watchers + (including whether a callback is invoked or not), it does not change + the semantics of the Python code being executed. + + If the callback returns with an exception set, it must return ``-1``; this + exception will be printed as an unraisable exception using + :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + + .. versionadded:: 3.12 diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c0f98b59ccaf..3f1ec0f9a344 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -773,6 +773,10 @@ New Features callbacks to receive notification on changes to a type. (Contributed by Carl Meyer in :gh:`91051`.) +* Added :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` + APIs to register callbacks to receive notification on creation and + destruction of code objects. + (Contributed by Itamar Ostricher in :gh:`91054`.) * Add :c:func:`PyFrame_GetVar` and :c:func:`PyFrame_GetVarString` functions to get a frame variable by its name. diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fd57e0035bc0..f11d099e0379 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -181,6 +181,41 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int); PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *); +typedef enum PyCodeEvent { + PY_CODE_EVENT_CREATE, + PY_CODE_EVENT_DESTROY +} PyCodeEvent; + + +/* + * A callback that is invoked for different events in a code object's lifecycle. + * + * The callback is invoked with a borrowed reference to co, after it is + * created and before it is destroyed. + * + * If the callback returns with an exception set, it must return -1. Otherwise + * it should return 0. + */ +typedef int (*PyCode_WatchCallback)( + PyCodeEvent event, + PyCodeObject* co); + +/* + * Register a per-interpreter callback that will be invoked for code object + * lifecycle events. + * + * Returns a handle that may be passed to PyCode_ClearWatcher on success, + * or -1 and sets an error if no more handles are available. + */ +PyAPI_FUNC(int) PyCode_AddWatcher(PyCode_WatchCallback callback); + +/* + * Clear the watcher associated with the watcher_id handle. + * + * Returns 0 on success or -1 if no watcher exists for the provided id. + */ +PyAPI_FUNC(int) PyCode_ClearWatcher(int watcher_id); + /* for internal use only */ struct _opaque { int computed_line; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 80c1bfb6c9af..357fc85a95cf 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -4,6 +4,8 @@ extern "C" { #endif +#define CODE_MAX_WATCHERS 8 + /* PEP 659 * Specialization and quickening structs and helper functions */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 532b28499080..c9597cfa7a4d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -191,6 +191,9 @@ struct _is { PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; + PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS]; + // One bit is set for each non-NULL entry in code_watchers + uint8_t active_code_watchers; struct _Py_unicode_state unicode; struct _Py_float_state float_state; diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 5e4f42a86006..ebe7d2783189 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -336,6 +336,74 @@ def test_no_more_ids_available(self): self.add_watcher() +class TestCodeObjectWatchers(unittest.TestCase): + @contextmanager + def code_watcher(self, which_watcher): + wid = _testcapi.add_code_watcher(which_watcher) + try: + yield wid + finally: + _testcapi.clear_code_watcher(wid) + + def assert_event_counts(self, exp_created_0, exp_destroyed_0, + exp_created_1, exp_destroyed_1): + self.assertEqual( + exp_created_0, _testcapi.get_code_watcher_num_created_events(0)) + self.assertEqual( + exp_destroyed_0, _testcapi.get_code_watcher_num_destroyed_events(0)) + self.assertEqual( + exp_created_1, _testcapi.get_code_watcher_num_created_events(1)) + self.assertEqual( + exp_destroyed_1, _testcapi.get_code_watcher_num_destroyed_events(1)) + + def test_code_object_events_dispatched(self): + # verify that all counts are zero before any watchers are registered + self.assert_event_counts(0, 0, 0, 0) + + # verify that all counts remain zero when a code object is + # created and destroyed with no watchers registered + co1 = _testcapi.code_newempty("test_watchers", "dummy1", 0) + self.assert_event_counts(0, 0, 0, 0) + del co1 + self.assert_event_counts(0, 0, 0, 0) + + # verify counts are as expected when first watcher is registered + with self.code_watcher(0): + self.assert_event_counts(0, 0, 0, 0) + co2 = _testcapi.code_newempty("test_watchers", "dummy2", 0) + self.assert_event_counts(1, 0, 0, 0) + del co2 + self.assert_event_counts(1, 1, 0, 0) + + # again with second watcher registered + with self.code_watcher(1): + self.assert_event_counts(1, 1, 0, 0) + co3 = _testcapi.code_newempty("test_watchers", "dummy3", 0) + self.assert_event_counts(2, 1, 1, 0) + del co3 + self.assert_event_counts(2, 2, 1, 1) + + # verify counts remain as they were after both watchers are cleared + co4 = _testcapi.code_newempty("test_watchers", "dummy4", 0) + self.assert_event_counts(2, 2, 1, 1) + del co4 + self.assert_event_counts(2, 2, 1, 1) + + def test_clear_out_of_range_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"Invalid code watcher ID -1"): + _testcapi.clear_code_watcher(-1) + with self.assertRaisesRegex(ValueError, r"Invalid code watcher ID 8"): + _testcapi.clear_code_watcher(8) # CODE_MAX_WATCHERS = 8 + + def test_clear_unassigned_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"No code watcher set for ID 1"): + _testcapi.clear_code_watcher(1) + + def test_allocate_too_many_watchers(self): + with self.assertRaisesRegex(RuntimeError, r"no more code watcher IDs available"): + _testcapi.allocate_too_many_code_watchers() + + class TestFuncWatchers(unittest.TestCase): @contextmanager def add_watcher(self, func): diff --git a/Misc/ACKS b/Misc/ACKS index 5d97067b85d3..d50cb3c2d1ee 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1320,6 +1320,7 @@ Michele Orr? Tom?? Orsava Oleg Oshmyan Denis Osipov +Itamar Ostricher Denis S. Otkidach Peter Otten Michael Otteneder @@ -1627,6 +1628,7 @@ Silas Sewell Ian Seyer Dmitry Shachnev Anish Shah +Jaineel Shah Daniel Shahaf Hui Shang Geoff Shannon diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst new file mode 100644 index 000000000000..c46459c15b9e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to +register callbacks to receive notification on creation and destruction of +code objects. diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 608cd780d12a..f0e51fd462e7 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -2,6 +2,7 @@ #define Py_BUILD_CORE #include "pycore_function.h" // FUNC_MAX_WATCHERS +#include "pycore_code.h" // CODE_MAX_WATCHERS // Test dict watching static PyObject *g_dict_watch_events; @@ -277,6 +278,126 @@ unwatch_type(PyObject *self, PyObject *args) Py_RETURN_NONE; } + +// Test code object watching + +#define NUM_CODE_WATCHERS 2 +static int num_code_object_created_events[NUM_CODE_WATCHERS] = {0, 0}; +static int num_code_object_destroyed_events[NUM_CODE_WATCHERS] = {0, 0}; + +static int +handle_code_object_event(int which_watcher, PyCodeEvent event, PyCodeObject *co) { + if (event == PY_CODE_EVENT_CREATE) { + num_code_object_created_events[which_watcher]++; + } + else if (event == PY_CODE_EVENT_DESTROY) { + num_code_object_destroyed_events[which_watcher]++; + } + else { + return -1; + } + return 0; +} + +static int +first_code_object_callback(PyCodeEvent event, PyCodeObject *co) +{ + return handle_code_object_event(0, event, co); +} + +static int +second_code_object_callback(PyCodeEvent event, PyCodeObject *co) +{ + return handle_code_object_event(1, event, co); +} + +static int +noop_code_event_handler(PyCodeEvent event, PyCodeObject *co) +{ + return 0; +} + +static PyObject * +add_code_watcher(PyObject *self, PyObject *which_watcher) +{ + int watcher_id; + assert(PyLong_Check(which_watcher)); + long which_l = PyLong_AsLong(which_watcher); + if (which_l == 0) { + watcher_id = PyCode_AddWatcher(first_code_object_callback); + } + else if (which_l == 1) { + watcher_id = PyCode_AddWatcher(second_code_object_callback); + } + else { + return NULL; + } + if (watcher_id < 0) { + return NULL; + } + return PyLong_FromLong(watcher_id); +} + +static PyObject * +clear_code_watcher(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + if (PyCode_ClearWatcher(watcher_id_l) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +get_code_watcher_num_created_events(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + assert(watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS); + return PyLong_FromLong(num_code_object_created_events[watcher_id_l]); +} + +static PyObject * +get_code_watcher_num_destroyed_events(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + assert(watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS); + return PyLong_FromLong(num_code_object_destroyed_events[watcher_id_l]); +} + +static PyObject * +allocate_too_many_code_watchers(PyObject *self, PyObject *args) +{ + int watcher_ids[CODE_MAX_WATCHERS + 1]; + int num_watchers = 0; + for (unsigned long i = 0; i < sizeof(watcher_ids) / sizeof(int); i++) { + int watcher_id = PyCode_AddWatcher(noop_code_event_handler); + if (watcher_id == -1) { + break; + } + watcher_ids[i] = watcher_id; + num_watchers++; + } + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + for (int i = 0; i < num_watchers; i++) { + if (PyCode_ClearWatcher(watcher_ids[i]) < 0) { + PyErr_WriteUnraisable(Py_None); + break; + } + } + if (type) { + PyErr_Restore(type, value, traceback); + return NULL; + } + else if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + // Test function watchers #define NUM_FUNC_WATCHERS 2 @@ -509,6 +630,16 @@ static PyMethodDef test_methods[] = { {"unwatch_type", unwatch_type, METH_VARARGS, NULL}, {"get_type_modified_events", get_type_modified_events, METH_NOARGS, NULL}, + // Code object watchers. + {"add_code_watcher", add_code_watcher, METH_O, NULL}, + {"clear_code_watcher", clear_code_watcher, METH_O, NULL}, + {"get_code_watcher_num_created_events", + get_code_watcher_num_created_events, METH_O, NULL}, + {"get_code_watcher_num_destroyed_events", + get_code_watcher_num_destroyed_events, METH_O, NULL}, + {"allocate_too_many_code_watchers", + (PyCFunction) allocate_too_many_code_watchers, METH_NOARGS, NULL}, + // Function watchers. {"add_func_watcher", add_func_watcher, METH_O, NULL}, {"clear_func_watcher", clear_func_watcher, METH_O, NULL}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index f5d90cf65fce..0c197d767b0a 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -12,6 +12,66 @@ #include "clinic/codeobject.c.h" +static void +notify_code_watchers(PyCodeEvent event, PyCodeObject *co) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->active_code_watchers) { + assert(interp->_initialized); + for (int i = 0; i < CODE_MAX_WATCHERS; i++) { + PyCode_WatchCallback cb = interp->code_watchers[i]; + if ((cb != NULL) && (cb(event, co) < 0)) { + PyErr_WriteUnraisable((PyObject *) co); + } + } + } +} + +int +PyCode_AddWatcher(PyCode_WatchCallback callback) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + + for (int i = 0; i < CODE_MAX_WATCHERS; i++) { + if (!interp->code_watchers[i]) { + interp->code_watchers[i] = callback; + interp->active_code_watchers |= (1 << i); + return i; + } + } + + PyErr_SetString(PyExc_RuntimeError, "no more code watcher IDs available"); + return -1; +} + +static inline int +validate_watcher_id(PyInterpreterState *interp, int watcher_id) +{ + if (watcher_id < 0 || watcher_id >= CODE_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid code watcher ID %d", watcher_id); + return -1; + } + if (!interp->code_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No code watcher set for ID %d", watcher_id); + return -1; + } + return 0; +} + +int +PyCode_ClearWatcher(int watcher_id) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + if (validate_watcher_id(interp, watcher_id) < 0) { + return -1; + } + interp->code_watchers[watcher_id] = NULL; + interp->active_code_watchers &= ~(1 << watcher_id); + return 0; +} + /****************** * generic helpers ******************/ @@ -355,6 +415,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) } co->_co_firsttraceable = entry_point; _PyCode_Quicken(co); + notify_code_watchers(PY_CODE_EVENT_CREATE, co); } static int @@ -1615,6 +1676,8 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, static void code_dealloc(PyCodeObject *co) { + notify_code_watchers(PY_CODE_EVENT_DESTROY, co); + if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyCodeObjectExtra *co_extra = co->co_extra; diff --git a/Python/pystate.c b/Python/pystate.c index 19fd9a6ae449..793ba917c41f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -466,6 +466,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } interp->active_func_watchers = 0; + for (int i=0; i < CODE_MAX_WATCHERS; i++) { + interp->code_watchers[i] = NULL; + } + interp->active_code_watchers = 0; + // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. From webhook-mailer at python.org Fri Dec 2 12:39:26 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 02 Dec 2022 17:39:26 -0000 Subject: [Python-checkins] gh-99741: Fix the Cross-Interpreter Data API (gh-99939) Message-ID: https://github.com/python/cpython/commit/b4f35055496d918b42ff305b5d09ebd333204a69 commit: b4f35055496d918b42ff305b5d09ebd333204a69 branch: main author: Eric Snow committer: ericsnowcurrently date: 2022-12-02T10:39:17-07:00 summary: gh-99741: Fix the Cross-Interpreter Data API (gh-99939) There were some minor issues that showed up while I was working on porting _xxsubinterpreters to multi-phase init. This fixes them. https://github.com/python/cpython/issues/99741 files: M Include/cpython/pystate.h M Include/internal/pycore_interp.h M Python/pystate.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index c51542bcc895..7468a1c4f26f 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -403,4 +403,5 @@ PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index c9597cfa7a4d..b5c46773c90f 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -249,9 +249,10 @@ extern void _PyInterpreterState_Clear(PyThreadState *tstate); struct _xidregitem; struct _xidregitem { - PyTypeObject *cls; - crossinterpdatafunc getdata; + struct _xidregitem *prev; struct _xidregitem *next; + PyObject *cls; // weakref to a PyTypeObject + crossinterpdatafunc getdata; }; PyAPI_FUNC(PyInterpreterState*) _PyInterpreterState_LookUpID(int64_t); diff --git a/Python/pystate.c b/Python/pystate.c index 793ba917c41f..2554cc286dfa 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1878,8 +1878,9 @@ _release_xidata(void *arg) _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg; if (data->free != NULL) { data->free(data->data); + data->data = NULL; } - Py_XDECREF(data->obj); + Py_CLEAR(data->obj); } static void @@ -1899,6 +1900,8 @@ _call_in_interpreter(struct _gilstate_runtime_state *gilstate, save_tstate = _PyThreadState_Swap(gilstate, tstate); } + // XXX Once the GIL is per-interpreter, this should be called with the + // calling interpreter's GIL released and the target interpreter's held. func(arg); // Switch back. @@ -1943,21 +1946,73 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) crossinterpdatafunc. It would be simpler and more efficient. */ static int -_register_xidata(struct _xidregistry *xidregistry, PyTypeObject *cls, +_xidregistry_add_type(struct _xidregistry *xidregistry, PyTypeObject *cls, crossinterpdatafunc getdata) { // Note that we effectively replace already registered classes // rather than failing. struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); - if (newhead == NULL) + if (newhead == NULL) { return -1; - newhead->cls = cls; + } + // XXX Assign a callback to clear the entry from the registry? + newhead->cls = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->cls == NULL) { + PyMem_RawFree(newhead); + return -1; + } newhead->getdata = getdata; + newhead->prev = NULL; newhead->next = xidregistry->head; + if (newhead->next != NULL) { + newhead->next->prev = newhead; + } xidregistry->head = newhead; return 0; } +static struct _xidregitem * +_xidregistry_remove_entry(struct _xidregistry *xidregistry, + struct _xidregitem *entry) +{ + struct _xidregitem *next = entry->next; + if (entry->prev != NULL) { + assert(entry->prev->next == entry); + entry->prev->next = next; + } + else { + assert(xidregistry->head == entry); + xidregistry->head = next; + } + if (next != NULL) { + next->prev = entry->prev; + } + Py_DECREF(entry->cls); + PyMem_RawFree(entry); + return next; +} + +static struct _xidregitem * +_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +{ + struct _xidregitem *cur = xidregistry->head; + while (cur != NULL) { + PyObject *registered = PyWeakref_GetObject(cur->cls); + if (registered == Py_None) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + } + else { + assert(PyType_Check(registered)); + if (registered == (PyObject *)cls) { + return cur; + } + cur = cur->next; + } + } + return NULL; +} + static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); int @@ -1973,19 +2028,32 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, return -1; } - // Make sure the class isn't ever deallocated. - Py_INCREF((PyObject *)cls); - struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); if (xidregistry->head == NULL) { _register_builtins_for_crossinterpreter_data(xidregistry); } - int res = _register_xidata(xidregistry, cls, getdata); + int res = _xidregistry_add_type(xidregistry, cls, getdata); PyThread_release_lock(xidregistry->mutex); return res; } +int +_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +{ + int res = 0; + struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + (void)_xidregistry_remove_entry(xidregistry, matched); + res = 1; + } + PyThread_release_lock(xidregistry->mutex); + return res; +} + + /* Cross-interpreter objects are looked up by exact match on the class. We can reassess this policy when we move from a global registry to a tp_* slot. */ @@ -1995,22 +2063,15 @@ _PyCrossInterpreterData_Lookup(PyObject *obj) { struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; PyObject *cls = PyObject_Type(obj); - crossinterpdatafunc getdata = NULL; PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - struct _xidregitem *cur = xidregistry->head; - if (cur == NULL) { + if (xidregistry->head == NULL) { _register_builtins_for_crossinterpreter_data(xidregistry); - cur = xidregistry->head; - } - for(; cur != NULL; cur = cur->next) { - if (cur->cls == (PyTypeObject *)cls) { - getdata = cur->getdata; - break; - } } + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, + (PyTypeObject *)cls); Py_DECREF(cls); PyThread_release_lock(xidregistry->mutex); - return getdata; + return matched != NULL ? matched->getdata : NULL; } /* cross-interpreter data for builtin types */ @@ -2116,22 +2177,22 @@ static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) { // None - if (_register_xidata(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { Py_FatalError("could not register None for cross-interpreter sharing"); } // int - if (_register_xidata(xidregistry, &PyLong_Type, _long_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { Py_FatalError("could not register int for cross-interpreter sharing"); } // bytes - if (_register_xidata(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { Py_FatalError("could not register bytes for cross-interpreter sharing"); } // str - if (_register_xidata(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { Py_FatalError("could not register str for cross-interpreter sharing"); } } From webhook-mailer at python.org Fri Dec 2 12:43:16 2022 From: webhook-mailer at python.org (iritkatriel) Date: Fri, 02 Dec 2022 17:43:16 -0000 Subject: [Python-checkins] gh-99955: standardize return values of functions in assembler and optimizer. (#99956) Message-ID: https://github.com/python/cpython/commit/ab02262cd0385a2fb5eb8a6ee3cedd4b4bb969f3 commit: ab02262cd0385a2fb5eb8a6ee3cedd4b4bb969f3 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-02T17:43:10Z summary: gh-99955: standardize return values of functions in assembler and optimizer. (#99956) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index e200c5abb598..d6ed6941ac1e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7268,12 +7268,12 @@ assemble_init(struct assembler *a, int firstlineno) if (a->a_except_table == NULL) { goto error; } - return 1; + return 0; error: Py_XDECREF(a->a_bytecode); Py_XDECREF(a->a_linetable); Py_XDECREF(a->a_except_table); - return 0; + return -1; } static void @@ -7667,8 +7667,9 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, bas { Py_ssize_t len = PyBytes_GET_SIZE(a->a_except_table); if (a->a_except_table_off + MAX_SIZE_OF_ENTRY >= len) { - if (_PyBytes_Resize(&a->a_except_table, len * 2) < 0) - return 0; + if (_PyBytes_Resize(&a->a_except_table, len * 2) < 0) { + return -1; + } } int size = end-start; assert(end > start); @@ -7683,7 +7684,7 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, bas assemble_emit_exception_table_item(a, size, 0); assemble_emit_exception_table_item(a, target, 0); assemble_emit_exception_table_item(a, depth_lasti, 0); - return 1; + return 0; } static int @@ -7699,7 +7700,9 @@ assemble_exception_table(struct assembler *a, basicblock *entryblock) struct instr *instr = &b->b_instr[i]; if (instr->i_except != handler) { if (handler != NULL) { - RETURN_IF_FALSE(assemble_emit_exception_table_entry(a, start, ioffset, handler)); + if (assemble_emit_exception_table_entry(a, start, ioffset, handler) < 0) { + return -1; + } } start = ioffset; handler = instr->i_except; @@ -7708,9 +7711,11 @@ assemble_exception_table(struct assembler *a, basicblock *entryblock) } } if (handler != NULL) { - RETURN_IF_FALSE(assemble_emit_exception_table_entry(a, start, ioffset, handler)); + if (assemble_emit_exception_table_entry(a, start, ioffset, handler) < 0) { + return -1; + } } - return 1; + return 0; } /* Code location emitting code. See locations.md for a description of the format. */ @@ -7813,12 +7818,12 @@ write_location_info_entry(struct assembler* a, struct instr* i, int isize) if (a->a_location_off + THEORETICAL_MAX_ENTRY_SIZE >= len) { assert(len > THEORETICAL_MAX_ENTRY_SIZE); if (_PyBytes_Resize(&a->a_linetable, len*2) < 0) { - return 0; + return -1; } } if (i->i_loc.lineno < 0) { write_location_info_none(a, isize); - return 1; + return 0; } int line_delta = i->i_loc.lineno - a->a_lineno; int column = i->i_loc.col_offset; @@ -7829,23 +7834,23 @@ write_location_info_entry(struct assembler* a, struct instr* i, int isize) if (i->i_loc.end_lineno == i->i_loc.lineno || i->i_loc.end_lineno == -1) { write_location_info_no_column(a, isize, line_delta); a->a_lineno = i->i_loc.lineno; - return 1; + return 0; } } else if (i->i_loc.end_lineno == i->i_loc.lineno) { if (line_delta == 0 && column < 80 && end_column - column < 16 && end_column >= column) { write_location_info_short_form(a, isize, column, end_column); - return 1; + return 0; } if (line_delta >= 0 && line_delta < 3 && column < 128 && end_column < 128) { write_location_info_oneline_form(a, isize, line_delta, column, end_column); a->a_lineno = i->i_loc.lineno; - return 1; + return 0; } } write_location_info_long_form(a, i, isize); a->a_lineno = i->i_loc.lineno; - return 1; + return 0; } static int @@ -7853,8 +7858,8 @@ assemble_emit_location(struct assembler* a, struct instr* i) { int isize = instr_size(i); while (isize > 8) { - if (!write_location_info_entry(a, i, 8)) { - return 0; + if (write_location_info_entry(a, i, 8) < 0) { + return -1; } isize -= 8; } @@ -7874,15 +7879,17 @@ assemble_emit(struct assembler *a, struct instr *i) int size = instr_size(i); if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) { - if (len > PY_SSIZE_T_MAX / 2) - return 0; - if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0) - return 0; + if (len > PY_SSIZE_T_MAX / 2) { + return -1; + } + if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0) { + return -1; + } } code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; a->a_offset += size; write_instr(code, i, size); - return 1; + return 0; } static int @@ -8282,17 +8289,17 @@ merge_const_one(PyObject *const_cache, PyObject **obj) PyDict_CheckExact(const_cache); PyObject *key = _PyCode_ConstantKey(*obj); if (key == NULL) { - return 0; + return -1; } // t is borrowed reference PyObject *t = PyDict_SetDefault(const_cache, key, key); Py_DECREF(key); if (t == NULL) { - return 0; + return -1; } if (t == key) { // obj is new constant. - return 1; + return 0; } if (PyTuple_CheckExact(t)) { @@ -8301,7 +8308,7 @@ merge_const_one(PyObject *const_cache, PyObject **obj) } Py_SETREF(*obj, Py_NewRef(t)); - return 1; + return 0; } // This is in codeobject.c. @@ -8367,7 +8374,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (!names) { goto error; } - if (!merge_const_one(c->c_const_cache, &names)) { + if (merge_const_one(c->c_const_cache, &names) < 0) { goto error; } @@ -8375,7 +8382,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (consts == NULL) { goto error; } - if (!merge_const_one(c->c_const_cache, &consts)) { + if (merge_const_one(c->c_const_cache, &consts) < 0) { goto error; } @@ -8426,7 +8433,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, goto error; } - if (!merge_const_one(c->c_const_cache, &localsplusnames)) { + if (merge_const_one(c->c_const_cache, &localsplusnames) < 0) { goto error; } con.localsplusnames = localsplusnames; @@ -8892,45 +8899,50 @@ assemble(struct compiler *c, int addNone) assemble_jump_offsets(g->g_entryblock); /* Create assembler */ - if (!assemble_init(&a, c->u->u_firstlineno)) + if (assemble_init(&a, c->u->u_firstlineno) < 0) { goto error; + } /* Emit code. */ for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) - if (!assemble_emit(&a, &b->b_instr[j])) + for (int j = 0; j < b->b_iused; j++) { + if (assemble_emit(&a, &b->b_instr[j]) < 0) { goto error; + } + } } /* Emit location info */ a.a_lineno = c->u->u_firstlineno; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) - if (!assemble_emit_location(&a, &b->b_instr[j])) + for (int j = 0; j < b->b_iused; j++) { + if (assemble_emit_location(&a, &b->b_instr[j]) < 0) { goto error; + } + } } - if (!assemble_exception_table(&a, g->g_entryblock)) { + if (assemble_exception_table(&a, g->g_entryblock) < 0) { goto error; } if (_PyBytes_Resize(&a.a_except_table, a.a_except_table_off) < 0) { goto error; } - if (!merge_const_one(c->c_const_cache, &a.a_except_table)) { + if (merge_const_one(c->c_const_cache, &a.a_except_table) < 0) { goto error; } if (_PyBytes_Resize(&a.a_linetable, a.a_location_off) < 0) { goto error; } - if (!merge_const_one(c->c_const_cache, &a.a_linetable)) { + if (merge_const_one(c->c_const_cache, &a.a_linetable) < 0) { goto error; } if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0) { goto error; } - if (!merge_const_one(c->c_const_cache, &a.a_bytecode)) { + if (merge_const_one(c->c_const_cache, &a.a_bytecode) < 0) { goto error; } @@ -8995,7 +9007,7 @@ fold_tuple_on_constants(PyObject *const_cache, } PyTuple_SET_ITEM(newconst, i, constant); } - if (merge_const_one(const_cache, &newconst) == 0) { + if (merge_const_one(const_cache, &newconst) < 0) { Py_DECREF(newconst); return -1; } @@ -9849,17 +9861,17 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) return err; } -static inline int +static inline bool is_exit_without_lineno(basicblock *b) { if (!basicblock_exits_scope(b)) { - return 0; + return false; } for (int i = 0; i < b->b_iused; i++) { if (b->b_instr[i].i_loc.lineno >= 0) { - return 0; + return false; } } - return 1; + return true; } /* PEP 626 mandates that the f_lineno of a frame is correct From webhook-mailer at python.org Fri Dec 2 13:37:02 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 02 Dec 2022 18:37:02 -0000 Subject: [Python-checkins] gh-99741: Clean Up the _xxsubinterpreters Module (gh-99940) Message-ID: https://github.com/python/cpython/commit/0547a981ae413248b21a6bb0cb62dda7d236fe45 commit: 0547a981ae413248b21a6bb0cb62dda7d236fe45 branch: main author: Eric Snow committer: ericsnowcurrently date: 2022-12-02T11:36:57-07:00 summary: gh-99741: Clean Up the _xxsubinterpreters Module (gh-99940) This cleanup up resolves a few subtle bugs and makes the implementation for multi-phase init much cleaner. https://github.com/python/cpython/issues/99741 files: M Include/cpython/pystate.h M Lib/test/test__xxsubinterpreters.py M Modules/_xxsubinterpretersmodule.c M Python/pystate.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7468a1c4f26f..0f56b1f21905 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -394,7 +394,7 @@ struct _xid { PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 66f29b95af10..f274b637d947 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -386,7 +386,6 @@ def test_types(self): self._assert_values([ b'spam', 9999, - self.cid, ]) def test_bytes(self): @@ -1213,6 +1212,18 @@ def test_equality(self): self.assertFalse(cid1 != cid2) self.assertTrue(cid1 != cid3) + def test_shareable(self): + chan = interpreters.channel_create() + + obj = interpreters.channel_create() + interpreters.channel_send(chan, obj) + got = interpreters.channel_recv(chan) + + self.assertEqual(got, obj) + self.assertIs(type(got), type(obj)) + # XXX Check the following in the channel tests? + #self.assertIsNot(got, obj) + class ChannelTests(TestBase): @@ -1545,6 +1556,19 @@ def test_recv_default(self): self.assertEqual(obj5, b'eggs') self.assertIs(obj6, default) + def test_recv_sending_interp_destroyed(self): + cid = interpreters.channel_create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + interpreters.destroy(interp) + + with self.assertRaisesRegex(RuntimeError, + 'unrecognized interpreter ID'): + interpreters.channel_recv(cid) + def test_run_string_arg_unresolved(self): cid = interpreters.channel_create() interp = interpreters.create() diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 2c9e0cda1ab0..3e064ca8c0b3 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -6,11 +6,15 @@ #endif #include "Python.h" +// XXX This module should not rely on internal API. #include "pycore_frame.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_interpreteridobject.h" +#define MODULE_NAME "_xxsubinterpreters" + + static char * _copy_raw_string(PyObject *strobj) { @@ -28,13 +32,126 @@ _copy_raw_string(PyObject *strobj) } static PyInterpreterState * -_get_current(void) +_get_current_interp(void) { // PyInterpreterState_Get() aborts if lookup fails, so don't need // to check the result for NULL. return PyInterpreterState_Get(); } +static PyObject * +_get_current_module(void) +{ + // We ensured it was imported in _run_script(). + PyObject *name = PyUnicode_FromString(MODULE_NAME); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + +static PyObject * +get_module_from_owned_type(PyTypeObject *cls) +{ + assert(cls != NULL); + return _get_current_module(); + // XXX Use the more efficient API now that we use heap types: + //return PyType_GetModule(cls); +} + +static struct PyModuleDef moduledef; + +static PyObject * +get_module_from_type(PyTypeObject *cls) +{ + assert(cls != NULL); + return _get_current_module(); + // XXX Use the more efficient API now that we use heap types: + //return PyType_GetModuleByDef(cls, &moduledef); +} + +static PyObject * +add_new_exception(PyObject *mod, const char *name, PyObject *base) +{ + assert(!PyObject_HasAttrString(mod, name)); + PyObject *exctype = PyErr_NewException(name, base, NULL); + if (exctype == NULL) { + return NULL; + } + int res = PyModule_AddType(mod, (PyTypeObject *)exctype); + if (res < 0) { + Py_DECREF(exctype); + return NULL; + } + return exctype; +} + +#define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ + add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) + +static PyTypeObject * +add_new_type(PyObject *mod, PyTypeObject *cls, crossinterpdatafunc shared) +{ + if (PyType_Ready(cls) != 0) { + return NULL; + } + if (PyModule_AddType(mod, cls) != 0) { + // XXX When this becomes a heap type, we need to decref here. + return NULL; + } + if (shared != NULL) { + if (_PyCrossInterpreterData_RegisterClass(cls, shared)) { + // XXX When this becomes a heap type, we need to decref here. + return NULL; + } + } + return cls; +} + +static int +_release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) +{ + PyObject *exctype, *excval, *exctb; + if (ignoreexc) { + PyErr_Fetch(&exctype, &excval, &exctb); + } + int res = _PyCrossInterpreterData_Release(data); + if (res < 0) { + // XXX Fix this! + /* The owning interpreter is already destroyed. + * Ideally, this shouldn't ever happen. When an interpreter is + * about to be destroyed, we should clear out all of its objects + * from every channel associated with that interpreter. + * For now we hack around that to resolve refleaks, by decref'ing + * the released object here, even if its the wrong interpreter. + * The owning interpreter has already been destroyed + * so we should be okay, especially since the currently + * shareable types are all very basic, with no GC. + * That said, it becomes much messier once interpreters + * no longer share a GIL, so this needs to be fixed before then. */ + // We do what _release_xidata() does in pystate.c. + if (data->free != NULL) { + data->free(data->data); + data->data = NULL; + } + Py_CLEAR(data->obj); + if (ignoreexc) { + // XXX Emit a warning? + PyErr_Clear(); + } + } + if (ignoreexc) { + PyErr_Restore(exctype, excval, exctb); + } + return res; +} + /* data-sharing-specific code ***********************************************/ @@ -66,7 +183,7 @@ _sharednsitem_clear(struct _sharednsitem *item) PyMem_Free(item->name); item->name = NULL; } - _PyCrossInterpreterData_Release(&item->data); + (void)_release_xid_data(&item->data, 1); } static int @@ -121,8 +238,10 @@ _sharedns_free(_sharedns *shared) } static _sharedns * -_get_shared_ns(PyObject *shareable) +_get_shared_ns(PyObject *shareable, PyTypeObject *channelidtype, + int *needs_import) { + *needs_import = 0; if (shareable == NULL || shareable == Py_None) { return NULL; } @@ -144,6 +263,9 @@ _get_shared_ns(PyObject *shareable) if (_sharednsitem_init(&shared->items[i], key, value) != 0) { break; } + if (Py_TYPE(value) == channelidtype) { + *needs_import = 1; + } } if (PyErr_Occurred()) { _sharedns_free(shared); @@ -287,6 +409,17 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) #define CHANNEL_BOTH 0 #define CHANNEL_RECV -1 +/* channel errors */ + +#define ERR_CHANNEL_NOT_FOUND -2 +#define ERR_CHANNEL_CLOSED -3 +#define ERR_CHANNEL_INTERP_CLOSED -4 +#define ERR_CHANNEL_EMPTY -5 +#define ERR_CHANNEL_NOT_EMPTY -6 +#define ERR_CHANNEL_MUTEX_INIT -7 +#define ERR_CHANNELS_MUTEX_INIT -8 +#define ERR_NO_NEXT_CHANNEL_ID -9 + static PyObject *ChannelError; static PyObject *ChannelNotFoundError; static PyObject *ChannelClosedError; @@ -294,61 +427,81 @@ static PyObject *ChannelEmptyError; static PyObject *ChannelNotEmptyError; static int -channel_exceptions_init(PyObject *ns) +channel_exceptions_init(PyObject *mod) { // XXX Move the exceptions into per-module memory? +#define ADD(NAME, BASE) \ + do { \ + if (NAME == NULL) { \ + NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ + if (NAME == NULL) { \ + return -1; \ + } \ + } \ + } while (0) + // A channel-related operation failed. - ChannelError = PyErr_NewException("_xxsubinterpreters.ChannelError", - PyExc_RuntimeError, NULL); - if (ChannelError == NULL) { - return -1; + ADD(ChannelError, PyExc_RuntimeError); + // An operation tried to use a channel that doesn't exist. + ADD(ChannelNotFoundError, ChannelError); + // An operation tried to use a closed channel. + ADD(ChannelClosedError, ChannelError); + // An operation tried to pop from an empty channel. + ADD(ChannelEmptyError, ChannelError); + // An operation tried to close a non-empty channel. + ADD(ChannelNotEmptyError, ChannelError); +#undef ADD + + return 0; +} + +static int +handle_channel_error(int err, PyObject *Py_UNUSED(mod), int64_t cid) +{ + if (err == 0) { + assert(!PyErr_Occurred()); + return 0; } - if (PyDict_SetItemString(ns, "ChannelError", ChannelError) != 0) { - return -1; + assert(err < 0); + if (err == ERR_CHANNEL_NOT_FOUND) { + PyErr_Format(ChannelNotFoundError, + "channel %" PRId64 " not found", cid); } - - // An operation tried to use a channel that doesn't exist. - ChannelNotFoundError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); - if (ChannelNotFoundError == NULL) { - return -1; + else if (err == ERR_CHANNEL_CLOSED) { + PyErr_Format(ChannelClosedError, + "channel %" PRId64 " is closed", cid); } - if (PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError) != 0) { - return -1; + else if (err == ERR_CHANNEL_INTERP_CLOSED) { + PyErr_Format(ChannelClosedError, + "channel %" PRId64 " is already closed", cid); } - - // An operation tried to use a closed channel. - ChannelClosedError = PyErr_NewException( - "_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); - if (ChannelClosedError == NULL) { - return -1; + else if (err == ERR_CHANNEL_EMPTY) { + PyErr_Format(ChannelEmptyError, + "channel %" PRId64 " is empty", cid); } - if (PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError) != 0) { - return -1; + else if (err == ERR_CHANNEL_NOT_EMPTY) { + PyErr_Format(ChannelNotEmptyError, + "channel %" PRId64 " may not be closed " + "if not empty (try force=True)", + cid); } - - // An operation tried to pop from an empty channel. - ChannelEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); - if (ChannelEmptyError == NULL) { - return -1; + else if (err == ERR_CHANNEL_MUTEX_INIT) { + PyErr_SetString(ChannelError, + "can't initialize mutex for new channel"); } - if (PyDict_SetItemString(ns, "ChannelEmptyError", ChannelEmptyError) != 0) { - return -1; + else if (err == ERR_CHANNELS_MUTEX_INIT) { + PyErr_SetString(ChannelError, + "can't initialize mutex for channel management"); } - - // An operation tried to close a non-empty channel. - ChannelNotEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotEmptyError", ChannelError, NULL); - if (ChannelNotEmptyError == NULL) { - return -1; + else if (err == ERR_NO_NEXT_CHANNEL_ID) { + PyErr_SetString(ChannelError, + "failed to get a channel ID"); } - if (PyDict_SetItemString(ns, "ChannelNotEmptyError", ChannelNotEmptyError) != 0) { - return -1; + else { + assert(PyErr_Occurred()); } - - return 0; + return 1; } /* the channel queue */ @@ -377,7 +530,7 @@ static void _channelitem_clear(_channelitem *item) { if (item->data != NULL) { - _PyCrossInterpreterData_Release(item->data); + (void)_release_xid_data(item->data, 1); PyMem_Free(item->data); item->data = NULL; } @@ -621,8 +774,7 @@ _channelends_associate(_channelends *ends, int64_t interp, int send) interp, &prev); if (end != NULL) { if (!end->open) { - PyErr_SetString(ChannelClosedError, "channel already closed"); - return -1; + return ERR_CHANNEL_CLOSED; } // already associated return 0; @@ -721,19 +873,13 @@ typedef struct _channel { } _PyChannelState; static _PyChannelState * -_channel_new(void) +_channel_new(PyThread_type_lock mutex) { _PyChannelState *chan = PyMem_NEW(_PyChannelState, 1); if (chan == NULL) { return NULL; } - chan->mutex = PyThread_allocate_lock(); - if (chan->mutex == NULL) { - PyMem_Free(chan); - PyErr_SetString(ChannelError, - "can't initialize mutex for new channel"); - return NULL; - } + chan->mutex = mutex; chan->queue = _channelqueue_new(); if (chan->queue == NULL) { PyMem_Free(chan); @@ -771,10 +917,11 @@ _channel_add(_PyChannelState *chan, int64_t interp, PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel closed"); + res = ERR_CHANNEL_CLOSED; goto done; } if (_channelends_associate(chan->ends, interp, 1) != 0) { + res = ERR_CHANNEL_INTERP_CLOSED; goto done; } @@ -788,31 +935,34 @@ _channel_add(_PyChannelState *chan, int64_t interp, return res; } -static _PyCrossInterpreterData * -_channel_next(_PyChannelState *chan, int64_t interp) +static int +_channel_next(_PyChannelState *chan, int64_t interp, + _PyCrossInterpreterData **res) { - _PyCrossInterpreterData *data = NULL; + int err = 0; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel closed"); + err = ERR_CHANNEL_CLOSED; goto done; } if (_channelends_associate(chan->ends, interp, 0) != 0) { + err = ERR_CHANNEL_INTERP_CLOSED; goto done; } - data = _channelqueue_get(chan->queue); + _PyCrossInterpreterData *data = _channelqueue_get(chan->queue); if (data == NULL && !PyErr_Occurred() && chan->closing != NULL) { chan->open = 0; } + *res = data; done: PyThread_release_lock(chan->mutex); if (chan->queue->count == 0) { _channel_finish_closing(chan); } - return data; + return err; } static int @@ -822,7 +972,7 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int end) int res = -1; if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel already closed"); + res = ERR_CHANNEL_CLOSED; goto done; } @@ -844,13 +994,12 @@ _channel_close_all(_PyChannelState *chan, int end, int force) PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel already closed"); + res = ERR_CHANNEL_CLOSED; goto done; } if (!force && chan->queue->count > 0) { - PyErr_SetString(ChannelNotEmptyError, - "may not be closed if not empty (try force=True)"); + res = ERR_CHANNEL_NOT_EMPTY; goto done; } @@ -935,21 +1084,24 @@ typedef struct _channels { int64_t next_id; } _channels; -static int -_channels_init(_channels *channels) +static void +_channels_init(_channels *channels, PyThread_type_lock mutex) { - if (channels->mutex == NULL) { - channels->mutex = PyThread_allocate_lock(); - if (channels->mutex == NULL) { - PyErr_SetString(ChannelError, - "can't initialize mutex for channel management"); - return -1; - } - } + channels->mutex = mutex; channels->head = NULL; channels->numopen = 0; channels->next_id = 0; - return 0; +} + +static void +_channels_fini(_channels *channels) +{ + assert(channels->numopen == 0); + assert(channels->head == NULL); + if (channels->mutex != NULL) { + PyThread_free_lock(channels->mutex); + channels->mutex = NULL; + } } static int64_t @@ -958,17 +1110,17 @@ _channels_next_id(_channels *channels) // needs lock int64_t id = channels->next_id; if (id < 0) { /* overflow */ - PyErr_SetString(ChannelError, - "failed to get a channel ID"); return -1; } channels->next_id += 1; return id; } -static _PyChannelState * -_channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) +static int +_channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex, + _PyChannelState **res) { + int err = -1; _PyChannelState *chan = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); if (pmutex != NULL) { @@ -977,11 +1129,11 @@ _channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) _channelref *ref = _channelref_find(channels->head, id, NULL); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", id); + err = ERR_CHANNEL_NOT_FOUND; goto done; } if (ref->chan == NULL || !ref->chan->open) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); + err = ERR_CHANNEL_CLOSED; goto done; } @@ -991,11 +1143,14 @@ _channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) } chan = ref->chan; + err = 0; + done: if (pmutex == NULL || *pmutex == NULL) { PyThread_release_lock(channels->mutex); } - return chan; + *res = chan; + return err; } static int64_t @@ -1007,6 +1162,7 @@ _channels_add(_channels *channels, _PyChannelState *chan) // Create a new ref. int64_t id = _channels_next_id(channels); if (id < 0) { + cid = ERR_NO_NEXT_CHANNEL_ID; goto done; } _channelref *ref = _channelref_new(id, chan); @@ -1041,31 +1197,32 @@ _channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan, _channelref *ref = _channelref_find(channels->head, cid, NULL); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", cid); + res = ERR_CHANNEL_NOT_FOUND; goto done; } if (ref->chan == NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", cid); + res = ERR_CHANNEL_CLOSED; goto done; } else if (!force && end == CHANNEL_SEND && ref->chan->closing != NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", cid); + res = ERR_CHANNEL_CLOSED; goto done; } else { - if (_channel_close_all(ref->chan, end, force) != 0) { - if (end == CHANNEL_SEND && - PyErr_ExceptionMatches(ChannelNotEmptyError)) { + int err = _channel_close_all(ref->chan, end, force); + if (err != 0) { + if (end == CHANNEL_SEND && err == ERR_CHANNEL_NOT_EMPTY) { if (ref->chan->closing != NULL) { - PyErr_Format(ChannelClosedError, - "channel %" PRId64 " closed", cid); + res = ERR_CHANNEL_CLOSED; goto done; } // Mark the channel as closing and return. The channel // will be cleaned up in _channel_next(). PyErr_Clear(); - if (_channel_set_closing(ref, channels->mutex) != 0) { + int err = _channel_set_closing(ref, channels->mutex); + if (err != 0) { + res = err; goto done; } if (pchan != NULL) { @@ -1073,6 +1230,9 @@ _channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan, } res = 0; } + else { + res = err; + } goto done; } if (pchan != NULL) { @@ -1121,7 +1281,7 @@ _channels_remove(_channels *channels, int64_t id, _PyChannelState **pchan) _channelref *prev = NULL; _channelref *ref = _channelref_find(channels->head, id, &prev); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", id); + res = ERR_CHANNEL_NOT_FOUND; goto done; } @@ -1141,7 +1301,7 @@ _channels_add_id_object(_channels *channels, int64_t id) _channelref *ref = _channelref_find(channels->head, id, NULL); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", id); + res = ERR_CHANNEL_NOT_FOUND; goto done; } ref->objcount += 1; @@ -1215,7 +1375,7 @@ _channel_set_closing(struct _channelref *ref, PyThread_type_lock mutex) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (chan->closing != NULL) { - PyErr_SetString(ChannelClosedError, "channel closed"); + res = ERR_CHANNEL_CLOSED; goto done; } chan->closing = PyMem_NEW(struct _channel_closing, 1); @@ -1258,14 +1418,18 @@ _channel_finish_closing(struct _channel *chan) { static int64_t _channel_create(_channels *channels) { - _PyChannelState *chan = _channel_new(); + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_CHANNEL_MUTEX_INIT; + } + _PyChannelState *chan = _channel_new(mutex); if (chan == NULL) { + PyThread_free_lock(mutex); return -1; } int64_t id = _channels_add(channels, chan); if (id < 0) { _channel_free(chan); - return -1; } return id; } @@ -1274,8 +1438,9 @@ static int _channel_destroy(_channels *channels, int64_t id) { _PyChannelState *chan = NULL; - if (_channels_remove(channels, id, &chan) != 0) { - return -1; + int err = _channels_remove(channels, id, &chan); + if (err != 0) { + return err; } if (chan != NULL) { _channel_free(chan); @@ -1286,23 +1451,24 @@ _channel_destroy(_channels *channels, int64_t id) static int _channel_send(_channels *channels, int64_t id, PyObject *obj) { - PyInterpreterState *interp = _get_current(); + PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { return -1; } // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = _channels_lookup(channels, id, &mutex); - if (chan == NULL) { - return -1; + _PyChannelState *chan = NULL; + int err = _channels_lookup(channels, id, &mutex, &chan); + if (err != 0) { + return err; } + assert(chan != NULL); // Past this point we are responsible for releasing the mutex. if (chan->closing != NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); PyThread_release_lock(mutex); - return -1; + return ERR_CHANNEL_CLOSED; } // Convert the object to cross-interpreter data. @@ -1321,61 +1487,87 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj) int res = _channel_add(chan, PyInterpreterState_GetID(interp), data); PyThread_release_lock(mutex); if (res != 0) { - _PyCrossInterpreterData_Release(data); + // We may chain an exception here: + (void)_release_xid_data(data, 0); PyMem_Free(data); - return -1; + return res; } return 0; } -static PyObject * -_channel_recv(_channels *channels, int64_t id) +static int +_channel_recv(_channels *channels, int64_t id, PyObject **res) { - PyInterpreterState *interp = _get_current(); + int err; + *res = NULL; + + PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { - return NULL; + // XXX Is this always an error? + if (PyErr_Occurred()) { + return -1; + } + return 0; } // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = _channels_lookup(channels, id, &mutex); - if (chan == NULL) { - return NULL; + _PyChannelState *chan = NULL; + err = _channels_lookup(channels, id, &mutex, &chan); + if (err != 0) { + return err; } + assert(chan != NULL); // Past this point we are responsible for releasing the mutex. // Pop off the next item from the channel. - _PyCrossInterpreterData *data = _channel_next(chan, PyInterpreterState_GetID(interp)); + _PyCrossInterpreterData *data = NULL; + err = _channel_next(chan, PyInterpreterState_GetID(interp), &data); PyThread_release_lock(mutex); - if (data == NULL) { - return NULL; + if (err != 0) { + return err; + } + else if (data == NULL) { + assert(!PyErr_Occurred()); + return 0; } // Convert the data back to an object. PyObject *obj = _PyCrossInterpreterData_NewObject(data); - _PyCrossInterpreterData_Release(data); - PyMem_Free(data); if (obj == NULL) { - return NULL; + assert(PyErr_Occurred()); + (void)_release_xid_data(data, 1); + PyMem_Free(data); + return -1; + } + int release_res = _release_xid_data(data, 0); + PyMem_Free(data); + if (release_res < 0) { + // The source interpreter has been destroyed already. + assert(PyErr_Occurred()); + Py_DECREF(obj); + return -1; } - return obj; + *res = obj; + return 0; } static int _channel_drop(_channels *channels, int64_t id, int send, int recv) { - PyInterpreterState *interp = _get_current(); + PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { return -1; } // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = _channels_lookup(channels, id, &mutex); - if (chan == NULL) { - return -1; + _PyChannelState *chan = NULL; + int err = _channels_lookup(channels, id, &mutex, &chan); + if (err != 0) { + return err; } // Past this point we are responsible for releasing the mutex. @@ -1395,12 +1587,13 @@ static int _channel_is_associated(_channels *channels, int64_t cid, int64_t interp, int send) { - _PyChannelState *chan = _channels_lookup(channels, cid, NULL); - if (chan == NULL) { - return -1; - } else if (send && chan->closing != NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", cid); - return -1; + _PyChannelState *chan = NULL; + int err = _channels_lookup(channels, cid, NULL, &chan); + if (err != 0) { + return err; + } + else if (send && chan->closing != NULL) { + return ERR_CHANNEL_CLOSED; } _channelend *end = _channelend_find(send ? chan->ends->send : chan->ends->recv, @@ -1411,7 +1604,7 @@ _channel_is_associated(_channels *channels, int64_t cid, int64_t interp, /* ChannelID class */ -static PyTypeObject ChannelIDtype; +static PyTypeObject ChannelIDType; typedef struct channelid { PyObject_HEAD @@ -1421,11 +1614,17 @@ typedef struct channelid { _channels *channels; } channelid; +struct channel_id_converter_data { + PyObject *module; + int64_t cid; +}; + static int channel_id_converter(PyObject *arg, void *ptr) { int64_t cid; - if (PyObject_TypeCheck(arg, &ChannelIDtype)) { + struct channel_id_converter_data *data = ptr; + if (PyObject_TypeCheck(arg, &ChannelIDType)) { cid = ((channelid *)arg)->id; } else if (PyIndex_Check(arg)) { @@ -1445,51 +1644,62 @@ channel_id_converter(PyObject *arg, void *ptr) Py_TYPE(arg)->tp_name); return 0; } - *(int64_t *)ptr = cid; + data->cid = cid; return 1; } -static channelid * +static int newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, - int force, int resolve) + int force, int resolve, channelid **res) { + *res = NULL; + channelid *self = PyObject_New(channelid, cls); if (self == NULL) { - return NULL; + return -1; } self->id = cid; self->end = end; self->resolve = resolve; self->channels = channels; - if (_channels_add_id_object(channels, cid) != 0) { - if (force && PyErr_ExceptionMatches(ChannelNotFoundError)) { - PyErr_Clear(); + int err = _channels_add_id_object(channels, cid); + if (err != 0) { + if (force && err == ERR_CHANNEL_NOT_FOUND) { + assert(!PyErr_Occurred()); } else { Py_DECREF((PyObject *)self); - return NULL; + return err; } } - return self; + *res = self; + return 0; } static _channels * _global_channels(void); static PyObject * -channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) +_channelid_new(PyObject *mod, PyTypeObject *cls, + PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", "send", "recv", "force", "_resolve", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = mod, + }; int send = -1; int recv = -1; int force = 0; int resolve = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|$pppp:ChannelID.__new__", kwlist, - channel_id_converter, &cid, &send, &recv, &force, &resolve)) + channel_id_converter, &cid_data, + &send, &recv, &force, &resolve)) { return NULL; + } + cid = cid_data.cid; // Handle "send" and "recv". if (send == 0 && recv == 0) { @@ -1508,8 +1718,16 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) end = CHANNEL_RECV; } - return (PyObject *)newchannelid(cls, cid, end, _global_channels(), - force, resolve); + PyObject *id = NULL; + int err = newchannelid(cls, cid, end, _global_channels(), + force, resolve, + (channelid **)&id); + if (handle_channel_error(err, mod, cid)) { + assert(id == NULL); + return NULL; + } + assert(id != NULL); + return id; } static void @@ -1557,43 +1775,8 @@ channelid_int(PyObject *self) } static PyNumberMethods channelid_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - 0, /* nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - 0, /* nb_and */ - 0, /* nb_xor */ - 0, /* nb_or */ - (unaryfunc)channelid_int, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - - (unaryfunc)channelid_int, /* nb_index */ + .nb_int = (unaryfunc)channelid_int, /* nb_int */ + .nb_index = (unaryfunc)channelid_int, /* nb_index */ }; static Py_hash_t @@ -1612,17 +1795,24 @@ channelid_hash(PyObject *self) static PyObject * channelid_richcompare(PyObject *self, PyObject *other, int op) { + PyObject *res = NULL; if (op != Py_EQ && op != Py_NE) { Py_RETURN_NOTIMPLEMENTED; } - if (!PyObject_TypeCheck(self, &ChannelIDtype)) { - Py_RETURN_NOTIMPLEMENTED; + PyObject *mod = get_module_from_type(Py_TYPE(self)); + if (mod == NULL) { + return NULL; + } + + if (!PyObject_TypeCheck(self, &ChannelIDType)) { + res = Py_NewRef(Py_NotImplemented); + goto done; } channelid *cid = (channelid *)self; int equal; - if (PyObject_TypeCheck(other, &ChannelIDtype)) { + if (PyObject_TypeCheck(other, &ChannelIDType)) { channelid *othercid = (channelid *)other; equal = (cid->end == othercid->end) && (cid->id == othercid->id); } @@ -1631,27 +1821,34 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) int overflow; long long othercid = PyLong_AsLongLongAndOverflow(other, &overflow); if (othercid == -1 && PyErr_Occurred()) { - return NULL; + goto done; } equal = !overflow && (othercid >= 0) && (cid->id == othercid); } else if (PyNumber_Check(other)) { PyObject *pyid = PyLong_FromLongLong(cid->id); if (pyid == NULL) { - return NULL; + goto done; } - PyObject *res = PyObject_RichCompare(pyid, other, op); + res = PyObject_RichCompare(pyid, other, op); Py_DECREF(pyid); - return res; + goto done; } else { - Py_RETURN_NOTIMPLEMENTED; + res = Py_NewRef(Py_NotImplemented); + goto done; } if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { - Py_RETURN_TRUE; + res = Py_NewRef(Py_True); } - Py_RETURN_FALSE; + else { + res = Py_NewRef(Py_False); + } + +done: + Py_DECREF(mod); + return res; } static PyObject * @@ -1690,24 +1887,42 @@ static PyObject * _channelid_from_xid(_PyCrossInterpreterData *data) { struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + + PyObject *mod = _get_current_module(); + if (mod == NULL) { + return NULL; + } + // Note that we do not preserve the "resolve" flag. - PyObject *cid = (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end, - _global_channels(), 0, 0); + PyObject *cid = NULL; + int err = newchannelid(&ChannelIDType, xid->id, xid->end, + _global_channels(), 0, 0, + (channelid **)&cid); + if (err != 0) { + assert(cid == NULL); + (void)handle_channel_error(err, mod, xid->id); + goto done; + } + assert(cid != NULL); if (xid->end == 0) { - return cid; + goto done; } if (!xid->resolve) { - return cid; + goto done; } /* Try returning a high-level channel end but fall back to the ID. */ PyObject *chan = _channel_from_cid(cid, xid->end); if (chan == NULL) { PyErr_Clear(); - return cid; + goto done; } Py_DECREF(cid); - return chan; + cid = chan; + +done: + Py_DECREF(mod); + return cid; } static int @@ -1734,8 +1949,22 @@ channelid_end(PyObject *self, void *end) int force = 1; channelid *cid = (channelid *)self; if (end != NULL) { - return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end, - cid->channels, force, cid->resolve); + PyObject *id = NULL; + int err = newchannelid(Py_TYPE(self), cid->id, *(int *)end, + cid->channels, force, cid->resolve, + (channelid **)&id); + if (err != 0) { + assert(id == NULL); + PyObject *mod = get_module_from_type(Py_TYPE(self)); + if (mod == NULL) { + return NULL; + } + (void)handle_channel_error(err, mod, cid->id); + Py_DECREF(mod); + return NULL; + } + assert(id != NULL); + return id; } if (cid->end == CHANNEL_SEND) { @@ -1763,7 +1992,7 @@ static PyGetSetDef channelid_getsets[] = { PyDoc_STRVAR(channelid_doc, "A channel ID identifies a channel and may be used as an int."); -static PyTypeObject ChannelIDtype = { +static PyTypeObject ChannelIDType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "_xxsubinterpreters.ChannelID", /* tp_name */ sizeof(channelid), /* tp_basicsize */ @@ -1807,21 +2036,23 @@ static PyTypeObject ChannelIDtype = { static PyObject * RunFailedError = NULL; static int -interp_exceptions_init(PyObject *ns) +interp_exceptions_init(PyObject *mod) { // XXX Move the exceptions into per-module memory? - if (RunFailedError == NULL) { - // An uncaught exception came out of interp_run_string(). - RunFailedError = PyErr_NewException("_xxsubinterpreters.RunFailedError", - PyExc_RuntimeError, NULL); - if (RunFailedError == NULL) { - return -1; - } - if (PyDict_SetItemString(ns, "RunFailedError", RunFailedError) != 0) { - return -1; - } - } +#define ADD(NAME, BASE) \ + do { \ + if (NAME == NULL) { \ + NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ + if (NAME == NULL) { \ + return -1; \ + } \ + } \ + } while (0) + + // An uncaught exception came out of interp_run_string(). + ADD(RunFailedError, PyExc_RuntimeError); +#undef ADD return 0; } @@ -1860,12 +2091,24 @@ _ensure_not_running(PyInterpreterState *interp) static int _run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception **exc) + _sharedns *shared, int needs_import, + _sharedexception **exc) { PyObject *exctype = NULL; PyObject *excval = NULL; PyObject *tb = NULL; + if (needs_import) { + // It might not have been imported yet in the current interpreter. + // However, it will (almost) always have been imported already + // in the main interpreter. + PyObject *mod = PyImport_ImportModule(MODULE_NAME); + if (mod == NULL) { + goto error; + } + Py_DECREF(mod); + } + PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); if (main_mod == NULL) { goto error; @@ -1918,14 +2161,16 @@ _run_script(PyInterpreterState *interp, const char *codestr, } static int -_run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, - PyObject *shareables) +_run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, + const char *codestr, PyObject *shareables) { if (_ensure_not_running(interp) < 0) { return -1; } - _sharedns *shared = _get_shared_ns(shareables); + int needs_import = 0; + _sharedns *shared = _get_shared_ns(shareables, &ChannelIDType, + &needs_import); if (shared == NULL && PyErr_Occurred()) { return -1; } @@ -1941,7 +2186,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Run the script. _sharedexception *exc = NULL; - int result = _run_script(interp, codestr, shared, &exc); + int result = _run_script(interp, codestr, shared, needs_import, &exc); // Switch back. if (save_tstate != NULL) { @@ -1972,18 +2217,41 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, the data that we need to share between interpreters, so it cannot hold PyObject values. */ static struct globals { + int module_count; _channels channels; -} _globals = {{0}}; +} _globals = {0}; static int -_init_globals(void) +_globals_init(void) { - if (_channels_init(&_globals.channels) != 0) { - return -1; + // XXX This isn't thread-safe. + _globals.module_count++; + if (_globals.module_count > 1) { + // Already initialized. + return 0; } + + assert(_globals.channels.mutex == NULL); + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_CHANNELS_MUTEX_INIT; + } + _channels_init(&_globals.channels, mutex); return 0; } +static void +_globals_fini(void) +{ + // XXX This isn't thread-safe. + _globals.module_count--; + if (_globals.module_count > 0) { + return; + } + + _channels_fini(&_globals.channels); +} + static _channels * _global_channels(void) { return &_globals.channels; @@ -2052,7 +2320,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Ensure we don't try to destroy the current interpreter. - PyInterpreterState *current = _get_current(); + PyInterpreterState *current = _get_current_interp(); if (current == NULL) { return NULL; } @@ -2129,7 +2397,7 @@ Return a list containing the ID of every existing interpreter."); static PyObject * interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyInterpreterState *interp =_get_current(); + PyInterpreterState *interp =_get_current_interp(); if (interp == NULL) { return NULL; } @@ -2187,7 +2455,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } // Run the code in the interpreter. - if (_run_script_in_interpreter(interp, codestr, shared) != 0) { + if (_run_script_in_interpreter(self, interp, codestr, shared) != 0) { return NULL; } Py_RETURN_NONE; @@ -2259,16 +2527,22 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t cid = _channel_create(&_globals.channels); if (cid < 0) { + (void)handle_channel_error(cid, self, -1); return NULL; } - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0, - &_globals.channels, 0, 0); - if (id == NULL) { - if (_channel_destroy(&_globals.channels, cid) != 0) { + PyObject *id = NULL; + int err = newchannelid(&ChannelIDType, cid, 0, + &_globals.channels, 0, 0, + (channelid **)&id); + if (handle_channel_error(err, self, cid)) { + assert(id == NULL); + err = _channel_destroy(&_globals.channels, cid); + if (handle_channel_error(err, self, cid)) { // XXX issue a warning? } return NULL; } + assert(id != NULL); assert(((channelid *)id)->channels != NULL); return id; } @@ -2283,12 +2557,17 @@ channel_destroy(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:channel_destroy", kwlist, - channel_id_converter, &cid)) { + channel_id_converter, &cid_data)) { return NULL; } + cid = cid_data.cid; - if (_channel_destroy(&_globals.channels, cid) != 0) { + int err = _channel_destroy(&_globals.channels, cid); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2317,12 +2596,16 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } int64_t *cur = cids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cur, 0, - &_globals.channels, 0, 0); - if (id == NULL) { + PyObject *id = NULL; + int err = newchannelid(&ChannelIDType, *cur, 0, + &_globals.channels, 0, 0, + (channelid **)&id); + if (handle_channel_error(err, self, *cur)) { + assert(id == NULL); Py_SETREF(ids, NULL); break; } + assert(id != NULL); PyList_SET_ITEM(ids, (Py_ssize_t)i, id); } @@ -2341,6 +2624,9 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "send", NULL}; int64_t cid; /* Channel ID */ + struct channel_id_converter_data cid_data = { + .module = self, + }; int send = 0; /* Send or receive end? */ int64_t id; PyObject *ids, *id_obj; @@ -2348,9 +2634,10 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords( args, kwds, "O&$p:channel_list_interpreters", - kwlist, channel_id_converter, &cid, &send)) { + kwlist, channel_id_converter, &cid_data, &send)) { return NULL; } + cid = cid_data.cid; ids = PyList_New(0); if (ids == NULL) { @@ -2363,6 +2650,7 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) assert(id >= 0); int res = _channel_is_associated(&_globals.channels, cid, id, send); if (res < 0) { + (void)handle_channel_error(res, self, cid); goto except; } if (res) { @@ -2402,13 +2690,18 @@ channel_send(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "obj", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; PyObject *obj; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:channel_send", kwlist, - channel_id_converter, &cid, &obj)) { + channel_id_converter, &cid_data, &obj)) { return NULL; } + cid = cid_data.cid; - if (_channel_send(&_globals.channels, cid, obj) != 0) { + int err = _channel_send(&_globals.channels, cid, obj); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2424,26 +2717,32 @@ channel_recv(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "default", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; PyObject *dflt = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O:channel_recv", kwlist, - channel_id_converter, &cid, &dflt)) { + channel_id_converter, &cid_data, &dflt)) { return NULL; } - Py_XINCREF(dflt); + cid = cid_data.cid; - PyObject *obj = _channel_recv(&_globals.channels, cid); - if (obj != NULL) { - Py_XDECREF(dflt); - return obj; - } else if (PyErr_Occurred()) { - Py_XDECREF(dflt); - return NULL; - } else if (dflt != NULL) { - return dflt; - } else { - PyErr_Format(ChannelEmptyError, "channel %" PRId64 " is empty", cid); + PyObject *obj = NULL; + int err = _channel_recv(&_globals.channels, cid, &obj); + if (handle_channel_error(err, self, cid)) { return NULL; } + Py_XINCREF(dflt); + if (obj == NULL) { + // Use the default. + if (dflt == NULL) { + (void)handle_channel_error(ERR_CHANNEL_EMPTY, self, cid); + return NULL; + } + obj = Py_NewRef(dflt); + } + Py_XDECREF(dflt); + return obj; } PyDoc_STRVAR(channel_recv_doc, @@ -2459,16 +2758,22 @@ channel_close(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "send", "recv", "force", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; int send = 0; int recv = 0; int force = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|$ppp:channel_close", kwlist, - channel_id_converter, &cid, &send, &recv, &force)) { + channel_id_converter, &cid_data, + &send, &recv, &force)) { return NULL; } + cid = cid_data.cid; - if (_channel_close(&_globals.channels, cid, send-recv, force) != 0) { + int err = _channel_close(&_globals.channels, cid, send-recv, force); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2507,14 +2812,19 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds) // Note that only the current interpreter is affected. static char *kwlist[] = {"cid", "send", "recv", "force", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; int send = 0; int recv = 0; int force = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|$ppp:channel_release", kwlist, - channel_id_converter, &cid, &send, &recv, &force)) { + channel_id_converter, &cid_data, + &send, &recv, &force)) { return NULL; } + cid = cid_data.cid; if (send == 0 && recv == 0) { send = 1; recv = 1; @@ -2523,7 +2833,8 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds) // XXX Handle force is True. // XXX Fix implicit release. - if (_channel_drop(&_globals.channels, cid, send, recv) != 0) { + int err = _channel_drop(&_globals.channels, cid, send, recv); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2539,7 +2850,14 @@ ends are closed. Closing an already closed end is a noop."); static PyObject * channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { - return channelid_new(&ChannelIDtype, args, kwds); + PyTypeObject *cls = &ChannelIDType; + PyObject *mod = get_module_from_owned_type(cls); + if (mod == NULL) { + return NULL; + } + PyObject *cid = _channelid_new(mod, cls, args, kwds); + Py_DECREF(mod); + return cid; } static PyMethodDef module_functions[] = { @@ -2590,59 +2908,57 @@ PyDoc_STRVAR(module_doc, "This module provides primitive operations to manage Python interpreters.\n\ The 'interpreters' module provides a more convenient interface."); -static struct PyModuleDef interpretersmodule = { - PyModuleDef_HEAD_INIT, - "_xxsubinterpreters", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - module_functions, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; - - -PyMODINIT_FUNC -PyInit__xxsubinterpreters(void) +static int +module_exec(PyObject *mod) { - if (_init_globals() != 0) { - return NULL; - } - - /* Initialize types */ - if (PyType_Ready(&ChannelIDtype) != 0) { - return NULL; - } - - /* Create the module */ - PyObject *module = PyModule_Create(&interpretersmodule); - if (module == NULL) { - return NULL; + if (_globals_init() != 0) { + return -1; } /* Add exception types */ - PyObject *ns = PyModule_GetDict(module); // borrowed - if (interp_exceptions_init(ns) != 0) { - return NULL; + if (interp_exceptions_init(mod) != 0) { + goto error; } - if (channel_exceptions_init(ns) != 0) { - return NULL; + if (channel_exceptions_init(mod) != 0) { + goto error; } /* Add other types */ - if (PyDict_SetItemString(ns, "ChannelID", - Py_NewRef(&ChannelIDtype)) != 0) { - return NULL; + if (add_new_type(mod, &ChannelIDType, _channelid_shared) == NULL) { + goto error; } - if (PyDict_SetItemString(ns, "InterpreterID", - Py_NewRef(&_PyInterpreterID_Type)) != 0) { - return NULL; + if (PyModule_AddType(mod, &_PyInterpreterID_Type) < 0) { + goto error; } - if (_PyCrossInterpreterData_RegisterClass(&ChannelIDtype, _channelid_shared)) { + return 0; + +error: + (void)_PyCrossInterpreterData_UnregisterClass(&ChannelIDType); + _globals_fini(); + return -1; +} + +static struct PyModuleDef moduledef = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = MODULE_NAME, + .m_doc = module_doc, + .m_size = -1, + .m_methods = module_functions, +}; + + +PyMODINIT_FUNC +PyInit__xxsubinterpreters(void) +{ + /* Create the module */ + PyObject *mod = PyModule_Create(&moduledef); + if (mod == NULL) { return NULL; } - - return module; + if (module_exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; } diff --git a/Python/pystate.c b/Python/pystate.c index 2554cc286dfa..0fdcdf1e3569 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1865,7 +1865,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) // Fill in the blanks and validate the result. data->interp = interp->id; if (_check_xidata(tstate, data) != 0) { - _PyCrossInterpreterData_Release(data); + (void)_PyCrossInterpreterData_Release(data); return -1; } @@ -1878,8 +1878,8 @@ _release_xidata(void *arg) _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg; if (data->free != NULL) { data->free(data->data); - data->data = NULL; } + data->data = NULL; Py_CLEAR(data->obj); } @@ -1910,27 +1910,29 @@ _call_in_interpreter(struct _gilstate_runtime_state *gilstate, } } -void +int _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) { - if (data->data == NULL && data->obj == NULL) { + if (data->free == NULL && data->obj == NULL) { // Nothing to release! - return; + data->data = NULL; + return 0; } // Switch to the original interpreter. PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp); if (interp == NULL) { // The interpreter was already destroyed. - if (data->free != NULL) { - // XXX Someone leaked some memory... - } - return; + // This function shouldn't have been called. + // XXX Someone leaked some memory... + assert(PyErr_Occurred()); + return -1; } // "Release" the data and/or the object. struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; _call_in_interpreter(gilstate, interp, _release_xidata, data); + return 0; } PyObject * From webhook-mailer at python.org Fri Dec 2 22:57:48 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 03 Dec 2022 03:57:48 -0000 Subject: [Python-checkins] GH-98831: Support cache effects in super- and macro instructions (#99601) Message-ID: https://github.com/python/cpython/commit/acf9184e6b68714cf7a756edefd02372dccd988b commit: acf9184e6b68714cf7a756edefd02372dccd988b branch: main author: Guido van Rossum committer: gvanrossum date: 2022-12-02T19:57:30-08:00 summary: GH-98831: Support cache effects in super- and macro instructions (#99601) files: M Python/generated_cases.c.h M Tools/cases_generator/generate_cases.py M Tools/cases_generator/lexer.py M Tools/cases_generator/parser.py diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3af60b83d84e..3a403824b499 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -436,10 +436,10 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - uint32_t type_version = read_u32(next_instr + 1); - uint16_t func_version = read_u16(next_instr + 3); PyObject *sub = PEEK(1); PyObject *container = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t func_version = read_u16(next_instr + 3); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 424b15ede2aa..2952634a3cda 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -14,23 +14,76 @@ import parser -DEFAULT_INPUT = "Python/bytecodes.c" -DEFAULT_OUTPUT = "Python/generated_cases.c.h" +DEFAULT_INPUT = os.path.relpath( + os.path.join(os.path.dirname(__file__), "../../Python/bytecodes.c") +) +DEFAULT_OUTPUT = os.path.relpath( + os.path.join(os.path.dirname(__file__), "../../Python/generated_cases.c.h") +) BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" RE_PREDICTED = r"(?s)(?:PREDICT$|GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)$;" UNUSED = "unused" BITS_PER_CODE_UNIT = 16 -arg_parser = argparse.ArgumentParser() -arg_parser.add_argument("-i", "--input", type=str, default=DEFAULT_INPUT) -arg_parser.add_argument("-o", "--output", type=str, default=DEFAULT_OUTPUT) +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) +arg_parser.add_argument( + "-i", "--input", type=str, help="Instruction definitions", default=DEFAULT_INPUT +) +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) -# This is not a data class -class Instruction(parser.InstDef): +class Formatter: + """Wraps an output stream with the ability to indent etc.""" + + stream: typing.TextIO + prefix: str + + def __init__(self, stream: typing.TextIO, indent: int) -> None: + self.stream = stream + self.prefix = " " * indent + + def write_raw(self, s: str) -> None: + self.stream.write(s) + + def emit(self, arg: str) -> None: + if arg: + self.write_raw(f"{self.prefix}{arg}\n") + else: + self.write_raw("\n") + + @contextlib.contextmanager + def indent(self): + self.prefix += " " + yield + self.prefix = self.prefix[:-4] + + @contextlib.contextmanager + def block(self, head: str): + if head: + self.emit(head + " {") + else: + self.emit("{") + with self.indent(): + yield + self.emit("}") + + + at dataclasses.dataclass +class Instruction: """An instruction with additional data and code.""" + # Parts of the underlying instruction definition + inst: parser.InstDef + kind: typing.Literal["inst", "op"] + name: str + block: parser.Block + # Computed by constructor always_exits: bool cache_offset: int @@ -43,65 +96,44 @@ class Instruction(parser.InstDef): predicted: bool = False def __init__(self, inst: parser.InstDef): - super().__init__(inst.header, inst.block) - self.context = inst.context + self.inst = inst + self.kind = inst.kind + self.name = inst.name + self.block = inst.block self.always_exits = always_exits(self.block) self.cache_effects = [ - effect for effect in self.inputs if isinstance(effect, parser.CacheEffect) + effect for effect in inst.inputs if isinstance(effect, parser.CacheEffect) ] self.cache_offset = sum(c.size for c in self.cache_effects) self.input_effects = [ - effect for effect in self.inputs if isinstance(effect, parser.StackEffect) + effect for effect in inst.inputs if isinstance(effect, parser.StackEffect) ] - self.output_effects = self.outputs # For consistency/completeness + self.output_effects = inst.outputs # For consistency/completeness - def write(self, f: typing.TextIO, indent: str, dedent: int = 0) -> None: + def write(self, out: Formatter) -> None: """Write one instruction, sans prologue and epilogue.""" - if dedent < 0: - indent += " " * -dedent # DO WE NEED THIS? - - # Get cache offset and maybe assert that it is correct + # Write a static assertion that a family's cache size is correct if family := self.family: if self.name == family.members[0]: if cache_size := family.size: - f.write( - f"{indent} static_assert({cache_size} == " - f'{self.cache_offset}, "incorrect cache size");\n' - ) - - # Write cache effect variable declarations - cache_offset = 0 - for ceffect in self.cache_effects: - if ceffect.name != UNUSED: - bits = ceffect.size * BITS_PER_CODE_UNIT - if bits == 64: - # NOTE: We assume that 64-bit data in the cache - # is always an object pointer. - # If this becomes false, we need a way to specify - # syntactically what type the cache data is. - f.write( - f"{indent} PyObject *{ceffect.name} = " - f"read_obj(next_instr + {cache_offset});\n" + out.emit( + f"static_assert({cache_size} == " + f'{self.cache_offset}, "incorrect cache size");' ) - else: - f.write(f"{indent} uint{bits}_t {ceffect.name} = " - f"read_u{bits}(next_instr + {cache_offset});\n") - cache_offset += ceffect.size - assert cache_offset == self.cache_offset # Write input stack effect variable declarations and initializations for i, seffect in enumerate(reversed(self.input_effects), 1): if seffect.name != UNUSED: - f.write(f"{indent} PyObject *{seffect.name} = PEEK({i});\n") + out.emit(f"PyObject *{seffect.name} = PEEK({i});") # Write output stack effect variable declarations input_names = {seffect.name for seffect in self.input_effects} input_names.add(UNUSED) for seffect in self.output_effects: if seffect.name not in input_names: - f.write(f"{indent} PyObject *{seffect.name};\n") + out.emit(f"PyObject *{seffect.name};") - self.write_body(f, indent, dedent) + self.write_body(out, 0) # Skip the rest if the block always exits if always_exits(self.block): @@ -110,9 +142,9 @@ def write(self, f: typing.TextIO, indent: str, dedent: int = 0) -> None: # Write net stack growth/shrinkage diff = len(self.output_effects) - len(self.input_effects) if diff > 0: - f.write(f"{indent} STACK_GROW({diff});\n") + out.emit(f"STACK_GROW({diff});") elif diff < 0: - f.write(f"{indent} STACK_SHRINK({-diff});\n") + out.emit(f"STACK_SHRINK({-diff});") # Write output stack effect assignments unmoved_names = {UNUSED} @@ -121,14 +153,32 @@ def write(self, f: typing.TextIO, indent: str, dedent: int = 0) -> None: unmoved_names.add(ieffect.name) for i, seffect in enumerate(reversed(self.output_effects)): if seffect.name not in unmoved_names: - f.write(f"{indent} POKE({i+1}, {seffect.name});\n") + out.emit(f"POKE({i+1}, {seffect.name});") # Write cache effect if self.cache_offset: - f.write(f"{indent} next_instr += {self.cache_offset};\n") + out.emit(f"next_instr += {self.cache_offset};") - def write_body(self, f: typing.TextIO, ndent: str, dedent: int) -> None: + def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None: """Write the instruction body.""" + # Write cache effect variable declarations and initializations + cache_offset = cache_adjust + for ceffect in self.cache_effects: + if ceffect.name != UNUSED: + bits = ceffect.size * BITS_PER_CODE_UNIT + if bits == 64: + # NOTE: We assume that 64-bit data in the cache + # is always an object pointer. + # If this becomes false, we need a way to specify + # syntactically what type the cache data is. + type = "PyObject *" + func = "read_obj" + else: + type = f"uint{bits}_t " + func = f"read_u{bits}" + out.emit(f"{type}{ceffect.name} = {func}(next_instr + {cache_offset});") + cache_offset += ceffect.size + assert cache_offset == self.cache_offset + cache_adjust # Get lines of text with proper dedent blocklines = self.block.to_text(dedent=dedent).splitlines(True) @@ -165,122 +215,101 @@ def write_body(self, f: typing.TextIO, ndent: str, dedent: int) -> None: else: break if ninputs: - f.write(f"{space}if ({cond}) goto pop_{ninputs}_{label};\n") + out.write_raw(f"{space}if ({cond}) goto pop_{ninputs}_{label};\n") else: - f.write(f"{space}if ({cond}) goto {label};\n") + out.write_raw(f"{space}if ({cond}) goto {label};\n") else: - f.write(line) + out.write_raw(line) + + +InstructionOrCacheEffect = Instruction | parser.CacheEffect @dataclasses.dataclass -class SuperComponent: +class Component: instr: Instruction input_mapping: dict[str, parser.StackEffect] output_mapping: dict[str, parser.StackEffect] + def write_body(self, out: Formatter, cache_adjust: int) -> None: + with out.block(""): + for var, ieffect in self.input_mapping.items(): + out.emit(f"PyObject *{ieffect.name} = {var};") + for oeffect in self.output_mapping.values(): + out.emit(f"PyObject *{oeffect.name};") + self.instr.write_body(out, dedent=-4, cache_adjust=cache_adjust) + for var, oeffect in self.output_mapping.items(): + out.emit(f"{var} = {oeffect.name};") + -class SuperInstruction(parser.Super): +# TODO: Use a common base class for {Super,Macro}Instruction + + at dataclasses.dataclass +class SuperOrMacroInstruction: + """Common fields for super- and macro instructions.""" + + name: str stack: list[str] initial_sp: int final_sp: int - parts: list[SuperComponent] - - def __init__(self, sup: parser.Super): - super().__init__(sup.kind, sup.name, sup.ops) - self.context = sup.context - - def analyze(self, a: "Analyzer") -> None: - components = self.check_components(a) - self.stack, self.initial_sp = self.super_macro_analysis(a, components) - sp = self.initial_sp - self.parts = [] - for instr in components: - input_mapping = {} - for ieffect in reversed(instr.input_effects): - sp -= 1 - if ieffect.name != UNUSED: - input_mapping[self.stack[sp]] = ieffect - output_mapping = {} - for oeffect in instr.output_effects: - if oeffect.name != UNUSED: - output_mapping[self.stack[sp]] = oeffect - sp += 1 - self.parts.append(SuperComponent(instr, input_mapping, output_mapping)) - self.final_sp = sp - - def check_components(self, a: "Analyzer") -> list[Instruction]: - components: list[Instruction] = [] - if not self.ops: - a.error(f"{self.kind.capitalize()}-instruction has no operands", self) - for name in self.ops: - if name not in a.instrs: - a.error(f"Unknown instruction {name!r}", self) - else: - instr = a.instrs[name] - if self.kind == "super" and instr.kind != "inst": - a.error(f"Super-instruction operand {instr.name} must be inst, not op", instr) - components.append(instr) - return components - def super_macro_analysis( - self, a: "Analyzer", components: list[Instruction] - ) -> tuple[list[str], int]: - """Analyze a super-instruction or macro. - Print an error if there's a cache effect (which we don't support yet). + at dataclasses.dataclass +class SuperInstruction(SuperOrMacroInstruction): + """A super-instruction.""" - Return the list of variable names and the initial stack pointer. - """ - lowest = current = highest = 0 - for instr in components: - if instr.cache_effects: - a.error( - f"Super-instruction {self.name!r} has cache effects in {instr.name!r}", - instr, - ) - current -= len(instr.input_effects) - lowest = min(lowest, current) - current += len(instr.output_effects) - highest = max(highest, current) - # At this point, 'current' is the net stack effect, - # and 'lowest' and 'highest' are the extremes. - # Note that 'lowest' may be negative. - stack = [f"_tmp_{i+1}" for i in range(highest - lowest)] - return stack, -lowest + super: parser.Super + parts: list[Component] + + + at dataclasses.dataclass +class MacroInstruction(SuperOrMacroInstruction): + """A macro instruction.""" + + macro: parser.Macro + parts: list[Component | parser.CacheEffect] class Analyzer: """Parse input, analyze it, and write to output.""" filename: str + output_filename: str src: str errors: int = 0 + def __init__(self, filename: str, output_filename: str): + """Read the input file.""" + self.filename = filename + self.output_filename = output_filename + with open(filename) as f: + self.src = f.read() + def error(self, msg: str, node: parser.Node) -> None: lineno = 0 if context := node.context: # Use line number of first non-comment in the node - for token in context.owner.tokens[context.begin : context.end]: + for token in context.owner.tokens[context.begin : context.end]: lineno = token.line if token.kind != "COMMENT": break print(f"{self.filename}:{lineno}: {msg}", file=sys.stderr) self.errors += 1 - def __init__(self, filename: str): - """Read the input file.""" - self.filename = filename - with open(filename) as f: - self.src = f.read() - instrs: dict[str, Instruction] # Includes ops - supers: dict[str, parser.Super] # Includes macros + supers: dict[str, parser.Super] super_instrs: dict[str, SuperInstruction] + macros: dict[str, parser.Macro] + macro_instrs: dict[str, MacroInstruction] families: dict[str, parser.Family] def parse(self) -> None: - """Parse the source text.""" + """Parse the source text. + + We only want the parser to see the stuff between the + begin and end markers. + """ psr = parser.Parser(self.src, filename=self.filename) # Skip until begin marker @@ -291,24 +320,38 @@ def parse(self) -> None: raise psr.make_syntax_error( f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" ) + start = psr.getpos() - # Parse until end marker + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) self.instrs = {} self.supers = {} + self.macros = {} self.families = {} - while (tkn := psr.peek(raw=True)) and tkn.text != END_MARKER: - if inst := psr.inst_def(): - self.instrs[inst.name] = instr = Instruction(inst) - elif super := psr.super_def(): - self.supers[super.name] = super - elif family := psr.family_def(): - self.families[family.name] = family - else: - raise psr.make_syntax_error(f"Unexpected token") + while thing := psr.definition(): + match thing: + case parser.InstDef(name=name): + self.instrs[name] = Instruction(thing) + case parser.Super(name): + self.supers[name] = thing + case parser.Macro(name): + self.macros[name] = thing + case parser.Family(name): + self.families[name] = thing + case _: + typing.assert_never(thing) + if not psr.eof(): + raise psr.make_syntax_error("Extra stuff at the end") print( - f"Read {len(self.instrs)} instructions, " - f"{len(self.supers)} supers/macros, " + f"Read {len(self.instrs)} instructions/ops, " + f"{len(self.supers)} supers, {len(self.macros)} macros, " f"and {len(self.families)} families from {self.filename}", file=sys.stderr, ) @@ -321,7 +364,7 @@ def analyze(self) -> None: self.find_predictions() self.map_families() self.check_families() - self.analyze_supers() + self.analyze_supers_and_macros() def find_predictions(self) -> None: """Find the instructions that need PREDICTED() labels.""" @@ -332,7 +375,7 @@ def find_predictions(self) -> None: else: self.error( f"Unknown instruction {target!r} predicted in {instr.name!r}", - instr, # TODO: Use better location + instr.inst, # TODO: Use better location ) def map_families(self) -> None: @@ -360,7 +403,9 @@ def check_families(self) -> None: members = [member for member in family.members if member in self.instrs] if members != family.members: unknown = set(family.members) - set(members) - self.error(f"Family {family.name!r} has unknown members: {unknown}", family) + self.error( + f"Family {family.name!r} has unknown members: {unknown}", family + ) if len(members) < 2: continue head = self.instrs[members[0]] @@ -381,105 +426,211 @@ def check_families(self) -> None: family, ) - def analyze_supers(self) -> None: - """Analyze each super instruction.""" + def analyze_supers_and_macros(self) -> None: + """Analyze each super- and macro instruction.""" self.super_instrs = {} - for name, sup in self.supers.items(): - dup = SuperInstruction(sup) - dup.analyze(self) - self.super_instrs[name] = dup + self.macro_instrs = {} + for name, super in self.supers.items(): + self.super_instrs[name] = self.analyze_super(super) + for name, macro in self.macros.items(): + self.macro_instrs[name] = self.analyze_macro(macro) + + def analyze_super(self, super: parser.Super) -> SuperInstruction: + components = self.check_super_components(super) + stack, initial_sp = self.stack_analysis(components) + sp = initial_sp + parts: list[Component] = [] + for component in components: + match component: + case parser.CacheEffect() as ceffect: + parts.append(ceffect) + case Instruction() as instr: + input_mapping = {} + for ieffect in reversed(instr.input_effects): + sp -= 1 + if ieffect.name != UNUSED: + input_mapping[stack[sp]] = ieffect + output_mapping = {} + for oeffect in instr.output_effects: + if oeffect.name != UNUSED: + output_mapping[stack[sp]] = oeffect + sp += 1 + parts.append(Component(instr, input_mapping, output_mapping)) + case _: + typing.assert_never(component) + final_sp = sp + return SuperInstruction(super.name, stack, initial_sp, final_sp, super, parts) + + def analyze_macro(self, macro: parser.Macro) -> MacroInstruction: + components = self.check_macro_components(macro) + stack, initial_sp = self.stack_analysis(components) + sp = initial_sp + parts: list[Component | parser.CacheEffect] = [] + for component in components: + match component: + case parser.CacheEffect() as ceffect: + parts.append(ceffect) + case Instruction() as instr: + input_mapping = {} + for ieffect in reversed(instr.input_effects): + sp -= 1 + if ieffect.name != UNUSED: + input_mapping[stack[sp]] = ieffect + output_mapping = {} + for oeffect in instr.output_effects: + if oeffect.name != UNUSED: + output_mapping[stack[sp]] = oeffect + sp += 1 + parts.append(Component(instr, input_mapping, output_mapping)) + case _: + typing.assert_never(component) + final_sp = sp + return MacroInstruction(macro.name, stack, initial_sp, final_sp, macro, parts) + + def check_super_components(self, super: parser.Super) -> list[Instruction]: + components: list[Instruction] = [] + for op in super.ops: + if op.name not in self.instrs: + self.error(f"Unknown instruction {op.name!r}", super) + else: + components.append(self.instrs[op.name]) + return components - def write_instructions(self, filename: str) -> None: + def check_macro_components( + self, macro: parser.Macro + ) -> list[InstructionOrCacheEffect]: + components: list[InstructionOrCacheEffect] = [] + for uop in macro.uops: + match uop: + case parser.OpName(name): + if name not in self.instrs: + self.error(f"Unknown instruction {name!r}", macro) + components.append(self.instrs[name]) + case parser.CacheEffect(): + components.append(uop) + case _: + typing.assert_never(uop) + return components + + def stack_analysis( + self, components: typing.Iterable[InstructionOrCacheEffect] + ) -> tuple[list[str], int]: + """Analyze a super-instruction or macro. + + Print an error if there's a cache effect (which we don't support yet). + + Return the list of variable names and the initial stack pointer. + """ + lowest = current = highest = 0 + for thing in components: + match thing: + case Instruction() as instr: + current -= len(instr.input_effects) + lowest = min(lowest, current) + current += len(instr.output_effects) + highest = max(highest, current) + case parser.CacheEffect(): + pass + case _: + typing.assert_never(thing) + # At this point, 'current' is the net stack effect, + # and 'lowest' and 'highest' are the extremes. + # Note that 'lowest' may be negative. + stack = [f"_tmp_{i+1}" for i in range(highest - lowest)] + return stack, -lowest + + def write_instructions(self) -> None: """Write instructions to output file.""" - indent = " " * 8 - with open(filename, "w") as f: + with open(self.output_filename, "w") as f: # Write provenance header f.write(f"// This file is generated by {os.path.relpath(__file__)}\n") f.write(f"// from {os.path.relpath(self.filename)}\n") f.write(f"// Do not edit!\n") - # Write regular instructions + # Create formatter; the rest of the code uses this. + self.out = Formatter(f, 8) + + # Write and count regular instructions n_instrs = 0 for name, instr in self.instrs.items(): if instr.kind != "inst": continue # ops are not real instructions n_instrs += 1 - f.write(f"\n{indent}TARGET({name}) {{\n") - if instr.predicted: - f.write(f"{indent} PREDICTED({name});\n") - instr.write(f, indent) - if not always_exits(instr.block): - f.write(f"{indent} DISPATCH();\n") - f.write(f"{indent}}}\n") - - # Write super-instructions and macros + self.out.emit("") + with self.out.block(f"TARGET({name})"): + if instr.predicted: + self.out.emit(f"PREDICTED({name});") + instr.write(self.out) + if not always_exits(instr.block): + self.out.emit(f"DISPATCH();") + + # Write and count super-instructions n_supers = 0 - n_macros = 0 for sup in self.super_instrs.values(): - if sup.kind == "super": - n_supers += 1 - elif sup.kind == "macro": - n_macros += 1 - self.write_super_macro(f, sup, indent) - - print( - f"Wrote {n_instrs} instructions, {n_supers} supers, " - f"and {n_macros} macros to {filename}", - file=sys.stderr, - ) + n_supers += 1 + self.write_super(sup) - def write_super_macro( - self, f: typing.TextIO, sup: SuperInstruction, indent: str = "" - ) -> None: + # Write and count macro instructions + n_macros = 0 + for macro in self.macro_instrs.values(): + n_macros += 1 + self.write_macro(macro) - # TODO: Make write() and block() methods of some Formatter class - def write(arg: str) -> None: - if arg: - f.write(f"{indent}{arg}\n") - else: - f.write("\n") + print( + f"Wrote {n_instrs} instructions, {n_supers} supers, " + f"and {n_macros} macros to {self.output_filename}", + file=sys.stderr, + ) - @contextlib.contextmanager - def block(head: str): - if head: - write(head + " {") - else: - write("{") - nonlocal indent - indent += " " - yield - indent = indent[:-4] - write("}") - - write("") - with block(f"TARGET({sup.name})"): - for i, var in enumerate(sup.stack): - if i < sup.initial_sp: - write(f"PyObject *{var} = PEEK({sup.initial_sp - i});") + def write_super(self, sup: SuperInstruction) -> None: + """Write code for a super-instruction.""" + with self.wrap_super_or_macro(sup): + first = True + for comp in sup.parts: + if not first: + self.out.emit("NEXTOPARG();") + self.out.emit("next_instr++;") + first = False + comp.write_body(self.out, 0) + if comp.instr.cache_offset: + self.out.emit(f"next_instr += {comp.instr.cache_offset};") + + def write_macro(self, mac: MacroInstruction) -> None: + """Write code for a macro instruction.""" + with self.wrap_super_or_macro(mac): + cache_adjust = 0 + for part in mac.parts: + match part: + case parser.CacheEffect(size=size): + cache_adjust += size + case Component() as comp: + comp.write_body(self.out, cache_adjust) + cache_adjust += comp.instr.cache_offset + + if cache_adjust: + self.out.emit(f"next_instr += {cache_adjust};") + + @contextlib.contextmanager + def wrap_super_or_macro(self, up: SuperOrMacroInstruction): + """Shared boilerplate for super- and macro instructions.""" + self.out.emit("") + with self.out.block(f"TARGET({up.name})"): + for i, var in enumerate(up.stack): + if i < up.initial_sp: + self.out.emit(f"PyObject *{var} = PEEK({up.initial_sp - i});") else: - write(f"PyObject *{var};") - - for i, comp in enumerate(sup.parts): - if i > 0 and sup.kind == "super": - write("NEXTOPARG();") - write("next_instr++;") - - with block(""): - for var, ieffect in comp.input_mapping.items(): - write(f"PyObject *{ieffect.name} = {var};") - for oeffect in comp.output_mapping.values(): - write(f"PyObject *{oeffect.name};") - comp.instr.write_body(f, indent, dedent=-4) - for var, oeffect in comp.output_mapping.items(): - write(f"{var} = {oeffect.name};") - - if sup.final_sp > sup.initial_sp: - write(f"STACK_GROW({sup.final_sp - sup.initial_sp});") - elif sup.final_sp < sup.initial_sp: - write(f"STACK_SHRINK({sup.initial_sp - sup.final_sp});") - for i, var in enumerate(reversed(sup.stack[:sup.final_sp]), 1): - write(f"POKE({i}, {var});") - write("DISPATCH();") + self.out.emit(f"PyObject *{var};") + + yield + + if up.final_sp > up.initial_sp: + self.out.emit(f"STACK_GROW({up.final_sp - up.initial_sp});") + elif up.final_sp < up.initial_sp: + self.out.emit(f"STACK_SHRINK({up.initial_sp - up.final_sp});") + for i, var in enumerate(reversed(up.stack[: up.final_sp]), 1): + self.out.emit(f"POKE({i}, {var});") + + self.out.emit(f"DISPATCH();") def always_exits(block: parser.Block) -> bool: @@ -506,13 +657,12 @@ def always_exits(block: parser.Block) -> bool: def main(): """Parse command line, parse input, analyze, write output.""" args = arg_parser.parse_args() # Prints message and sys.exit(2) on error - a = Analyzer(args.input) # Raises OSError if file not found + a = Analyzer(args.input, args.output) # Raises OSError if input unreadable a.parse() # Raises SyntaxError on failure - a.analyze() # Prints messages and raises SystemExit on failure + a.analyze() # Prints messages and sets a.errors on failure if a.errors: sys.exit(f"Found {a.errors} errors") - - a.write_instructions(args.output) # Raises OSError if file can't be written + a.write_instructions() # Raises OSError if output can't be written if __name__ == "__main__": diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 980c920bf357..39b6a212a67b 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -240,7 +240,12 @@ def to_text(tkns: list[Token], dedent: int = 0) -> str: res.append('\n') col = 1+dedent res.append(' '*(c-col)) - res.append(tkn.text) + text = tkn.text + if dedent != 0 and tkn.kind == 'COMMENT' and '\n' in text: + if dedent < 0: + text = text.replace('\n', '\n' + ' '*-dedent) + # TODO: dedent > 0 + res.append(text) line, col = tkn.end return ''.join(res) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index ae5ef1e26ea1..02a7834d2215 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -9,10 +9,12 @@ P = TypeVar("P", bound="Parser") N = TypeVar("N", bound="Node") -def contextual(func: Callable[[P], N|None]) -> Callable[[P], N|None]: + + +def contextual(func: Callable[[P], N | None]) -> Callable[[P], N | None]: # Decorator to wrap grammar methods. # Resets position if `func` returns None. - def contextual_wrapper(self: P) -> N|None: + def contextual_wrapper(self: P) -> N | None: begin = self.getpos() res = func(self) if res is None: @@ -21,6 +23,7 @@ def contextual_wrapper(self: P) -> N|None: end = self.getpos() res.context = Context(begin, end, self) return res + return contextual_wrapper @@ -35,7 +38,7 @@ def __repr__(self): @dataclass class Node: - context: Context|None = field(init=False, default=None) + context: Context | None = field(init=False, default=None) @property def text(self) -> str: @@ -68,8 +71,14 @@ class CacheEffect(Node): size: int + at dataclass +class OpName(Node): + name: str + + InputEffect = StackEffect | CacheEffect OutputEffect = StackEffect +UOp = OpName | CacheEffect @dataclass @@ -82,32 +91,23 @@ class InstHeader(Node): @dataclass class InstDef(Node): - # TODO: Merge InstHeader and InstDef - header: InstHeader + kind: Literal["inst", "op"] + name: str + inputs: list[InputEffect] + outputs: list[OutputEffect] block: Block - @property - def kind(self) -> str: - return self.header.kind - - @property - def name(self) -> str: - return self.header.name - @property - def inputs(self) -> list[InputEffect]: - return self.header.inputs - - @property - def outputs(self) -> list[OutputEffect]: - return self.header.outputs + at dataclass +class Super(Node): + name: str + ops: list[OpName] @dataclass -class Super(Node): - kind: Literal["macro", "super"] +class Macro(Node): name: str - ops: list[str] + uops: list[UOp] @dataclass @@ -118,12 +118,22 @@ class Family(Node): class Parser(PLexer): + @contextual + def definition(self) -> InstDef | Super | Macro | Family | None: + if inst := self.inst_def(): + return inst + if super := self.super_def(): + return super + if macro := self.macro_def(): + return macro + if family := self.family_def(): + return family @contextual def inst_def(self) -> InstDef | None: - if header := self.inst_header(): + if hdr := self.inst_header(): if block := self.block(): - return InstDef(header, block) + return InstDef(hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block) raise self.make_syntax_error("Expected block") return None @@ -132,17 +142,14 @@ def inst_header(self) -> InstHeader | None: # inst(NAME) # | inst(NAME, (inputs -- outputs)) # | op(NAME, (inputs -- outputs)) - # TODO: Error out when there is something unexpected. # TODO: Make INST a keyword in the lexer. if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("inst", "op"): - if (self.expect(lx.LPAREN) - and (tkn := self.expect(lx.IDENTIFIER))): + if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)): name = tkn.text if self.expect(lx.COMMA): inp, outp = self.stack_effect() if self.expect(lx.RPAREN): - if ((tkn := self.peek()) - and tkn.kind == lx.LBRACE): + if (tkn := self.peek()) and tkn.kind == lx.LBRACE: return InstHeader(kind, name, inp, outp) elif self.expect(lx.RPAREN) and kind == "inst": # No legacy stack effect if kind is "op". @@ -176,18 +183,20 @@ def inputs(self) -> list[InputEffect] | None: def input(self) -> InputEffect | None: # IDENTIFIER '/' INTEGER (CacheEffect) # IDENTIFIER (StackEffect) - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.DIVIDE): if num := self.expect(lx.NUMBER): try: size = int(num.text) except ValueError: raise self.make_syntax_error( - f"Expected integer, got {num.text!r}") + f"Expected integer, got {num.text!r}" + ) else: return CacheEffect(tkn.text, size) raise self.make_syntax_error("Expected integer") else: + # TODO: Arrays, conditions return StackEffect(tkn.text) def outputs(self) -> list[OutputEffect] | None: @@ -205,46 +214,91 @@ def outputs(self) -> list[OutputEffect] | None: @contextual def output(self) -> OutputEffect | None: - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): return StackEffect(tkn.text) @contextual def super_def(self) -> Super | None: - if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("super", "macro"): + if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "super": if self.expect(lx.LPAREN): - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if ops := self.ops(): - res = Super(kind, tkn.text, ops) + self.require(lx.SEMI) + res = Super(tkn.text, ops) return res - def ops(self) -> list[str] | None: - if tkn := self.expect(lx.IDENTIFIER): - ops = [tkn.text] + def ops(self) -> list[OpName] | None: + if op := self.op(): + ops = [op] while self.expect(lx.PLUS): - if tkn := self.require(lx.IDENTIFIER): - ops.append(tkn.text) - self.require(lx.SEMI) + if op := self.op(): + ops.append(op) return ops + @contextual + def op(self) -> OpName | None: + if tkn := self.expect(lx.IDENTIFIER): + return OpName(tkn.text) + + @contextual + def macro_def(self) -> Macro | None: + if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "macro": + if self.expect(lx.LPAREN): + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.RPAREN): + if self.expect(lx.EQUALS): + if uops := self.uops(): + self.require(lx.SEMI) + res = Macro(tkn.text, uops) + return res + + def uops(self) -> list[UOp] | None: + if uop := self.uop(): + uops = [uop] + while self.expect(lx.PLUS): + if uop := self.uop(): + uops.append(uop) + else: + raise self.make_syntax_error("Expected op name or cache effect") + return uops + + @contextual + def uop(self) -> UOp | None: + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.DIVIDE): + if num := self.expect(lx.NUMBER): + try: + size = int(num.text) + except ValueError: + raise self.make_syntax_error( + f"Expected integer, got {num.text!r}" + ) + else: + return CacheEffect(tkn.text, size) + raise self.make_syntax_error("Expected integer") + else: + return OpName(tkn.text) + @contextual def family_def(self) -> Family | None: if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "family": size = None if self.expect(lx.LPAREN): - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.COMMA): if not (size := self.expect(lx.IDENTIFIER)): - raise self.make_syntax_error( - "Expected identifier") + raise self.make_syntax_error("Expected identifier") if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if not self.expect(lx.LBRACE): raise self.make_syntax_error("Expected {") if members := self.members(): if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Family(tkn.text, size.text if size else "", members) + return Family( + tkn.text, size.text if size else "", members + ) return None def members(self) -> list[str] | None: @@ -284,6 +338,7 @@ def c_blob(self) -> list[lx.Token]: if __name__ == "__main__": import sys + if sys.argv[1:]: filename = sys.argv[1] if filename == "-c" and sys.argv[2:]: @@ -295,10 +350,10 @@ def c_blob(self) -> list[lx.Token]: srclines = src.splitlines() begin = srclines.index("// BEGIN BYTECODES //") end = srclines.index("// END BYTECODES //") - src = "\n".join(srclines[begin+1 : end]) + src = "\n".join(srclines[begin + 1 : end]) else: filename = "" src = "if (x) { x.foo; // comment\n}" parser = Parser(src, filename) - x = parser.inst_def() or parser.super_def() or parser.family_def() + x = parser.definition() print(x) From webhook-mailer at python.org Sat Dec 3 01:15:42 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 03 Dec 2022 06:15:42 -0000 Subject: [Python-checkins] GH-66285: remove redundant `time.sleep` from `test_fork_signal_handling` (GH-99963) Message-ID: https://github.com/python/cpython/commit/ee6015650017ca145a48c345311a9c481949de71 commit: ee6015650017ca145a48c345311a9c481949de71 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-02T22:15:36-08:00 summary: GH-66285: remove redundant `time.sleep` from `test_fork_signal_handling` (GH-99963) files: M Lib/test/test_asyncio/test_unix_events.py diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 092edb215854..309a1cfdb4aa 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1907,7 +1907,6 @@ def test_fork_signal_handling(self): def child_main(): signal.signal(signal.SIGTERM, lambda *args: child_handled.set()) child_started.set() - time.sleep(1) async def main(): loop = asyncio.get_running_loop() From webhook-mailer at python.org Sat Dec 3 11:53:25 2022 From: webhook-mailer at python.org (rhettinger) Date: Sat, 03 Dec 2022 16:53:25 -0000 Subject: [Python-checkins] gh-99934: test_marshal.py: add more elements in test_deterministic_sets (GH-99935) Message-ID: https://github.com/python/cpython/commit/c68573b339320409b038501fdd7d4f8a56766275 commit: c68573b339320409b038501fdd7d4f8a56766275 branch: main author: Alexander Kanavin committer: rhettinger date: 2022-12-03T10:53:20-06:00 summary: gh-99934: test_marshal.py: add more elements in test_deterministic_sets (GH-99935) Existing elements do produce different output on x86_64, but they do not on x86. Let's make the data longer to ensure it differs. files: A Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst M Lib/test/test_marshal.py diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 54c5a324897d..3d9d6d5d0aca 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -355,7 +355,7 @@ def test_deterministic_sets(self): for elements in ( "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'", # Also test for bad interactions with backreferencing: - "('Spam', 0), ('Spam', 1), ('Spam', 2)", + "('Spam', 0), ('Spam', 1), ('Spam', 2), ('Spam', 3), ('Spam', 4), ('Spam', 5)", ): s = f"{kind}([{elements}])" with self.subTest(s): diff --git a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst new file mode 100644 index 000000000000..1b1287c33dbd --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst @@ -0,0 +1 @@ +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. From webhook-mailer at python.org Sat Dec 3 12:18:29 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 03 Dec 2022 17:18:29 -0000 Subject: [Python-checkins] gh-99934: test_marshal.py: add more elements in test_deterministic_sets (GH-99935) Message-ID: https://github.com/python/cpython/commit/4710a7589aee48bec9099af308bf5938285ba336 commit: 4710a7589aee48bec9099af308bf5938285ba336 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-03T09:18:23-08:00 summary: gh-99934: test_marshal.py: add more elements in test_deterministic_sets (GH-99935) Existing elements do produce different output on x86_64, but they do not on x86. Let's make the data longer to ensure it differs. (cherry picked from commit c68573b339320409b038501fdd7d4f8a56766275) Co-authored-by: Alexander Kanavin files: A Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst M Lib/test/test_marshal.py diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index f5956fc3a15a..a1a91f661ba2 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -354,7 +354,7 @@ def test_deterministic_sets(self): for elements in ( "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'", # Also test for bad interactions with backreferencing: - "('Spam', 0), ('Spam', 1), ('Spam', 2)", + "('Spam', 0), ('Spam', 1), ('Spam', 2), ('Spam', 3), ('Spam', 4), ('Spam', 5)", ): s = f"{kind}([{elements}])" with self.subTest(s): diff --git a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst new file mode 100644 index 000000000000..1b1287c33dbd --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst @@ -0,0 +1 @@ +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. From webhook-mailer at python.org Sat Dec 3 14:52:32 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 03 Dec 2022 19:52:32 -0000 Subject: [Python-checkins] bpo-15999: Accept arbitrary values for boolean parameters. (#15609) Message-ID: https://github.com/python/cpython/commit/a87c46eab3c306b1c5b8a072b7b30ac2c50651c0 commit: a87c46eab3c306b1c5b8a072b7b30ac2c50651c0 branch: main author: Serhiy Storchaka committer: gpshead date: 2022-12-03T11:52:21-08:00 summary: bpo-15999: Accept arbitrary values for boolean parameters. (#15609) builtins and extension module functions and methods that expect boolean values for parameters now accept any Python object rather than just a bool or int type. This is more consistent with how native Python code itself behaves. files: A Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst M Doc/whatsnew/3.12.rst M Lib/test/test_itertools.py M Lib/test/test_posix.py M Modules/_codecsmodule.c M Modules/_cursesmodule.c M Modules/_io/_iomodule.c M Modules/_io/clinic/_iomodule.c.h M Modules/_io/clinic/fileio.c.h M Modules/_io/clinic/textio.c.h M Modules/_io/clinic/winconsoleio.c.h M Modules/_io/fileio.c M Modules/_io/textio.c M Modules/_io/winconsoleio.c M Modules/_json.c M Modules/_lsprof.c M Modules/_multiprocessing/clinic/semaphore.c.h M Modules/_multiprocessing/semaphore.c M Modules/_posixsubprocess.c M Modules/_sqlite/clinic/connection.c.h M Modules/_sqlite/connection.c M Modules/_ssl.c M Modules/_threadmodule.c M Modules/_tkinter.c M Modules/_winapi.c M Modules/binascii.c M Modules/cjkcodecs/clinic/multibytecodec.c.h M Modules/cjkcodecs/multibytecodec.c M Modules/clinic/_codecsmodule.c.h M Modules/clinic/_cursesmodule.c.h M Modules/clinic/_ssl.c.h M Modules/clinic/_tkinter.c.h M Modules/clinic/_winapi.c.h M Modules/clinic/binascii.c.h M Modules/clinic/posixmodule.c.h M Modules/clinic/pyexpat.c.h M Modules/faulthandler.c M Modules/itertoolsmodule.c M Modules/ossaudiodev.c M Modules/posixmodule.c M Modules/pyexpat.c M Modules/socketmodule.c M Objects/bytearrayobject.c M Objects/bytesobject.c M Objects/clinic/bytearrayobject.c.h M Objects/clinic/bytesobject.c.h M Objects/clinic/listobject.c.h M Objects/clinic/unicodeobject.c.h M Objects/listobject.c M Objects/unicodeobject.c M Python/bltinmodule.c M Python/clinic/bltinmodule.c.h diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3f1ec0f9a344..6f5ce818d961 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -178,6 +178,10 @@ Other Language Changes In a future Python version they will be eventually a :exc:`SyntaxError`. (Contributed by Victor Stinner in :gh:`98401`.) +* All builtin and extension callables expecting boolean parameters now accept + arguments of any type instead of just :class:`bool` and :class:`int`. + (Contributed by Serhiy Storchaka in :gh:`60203`.) + New Modules =========== diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index a0a740fba8e8..f028bd5fdce8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -739,10 +739,6 @@ def test_cycle_setstate(self): c.__setstate__((tuple('defg'), 0)) take(20, c) - # The second argument in the setstate tuple must be an int - with self.assertRaises(TypeError): - cycle('defg').__setstate__((list('abcdefg'), 'x')) - self.assertRaises(TypeError, cycle('').__setstate__, ()) self.assertRaises(TypeError, cycle('').__setstate__, ([],)) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 442dec8b2806..77f42f7f9c93 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1646,12 +1646,6 @@ def test_resetids(self): ) support.wait_process(pid, exitcode=0) - def test_resetids_wrong_type(self): - with self.assertRaises(TypeError): - self.spawn_func(sys.executable, - [sys.executable, "-c", "pass"], - os.environ, resetids=None) - def test_setpgroup(self): pid = self.spawn_func( sys.executable, diff --git a/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst b/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst new file mode 100644 index 000000000000..9650c694cc67 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst @@ -0,0 +1,2 @@ +All built-in functions now accept arguments of any type instead of just +``bool`` and ``int`` for boolean parameters. diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 8a0df4266e83..d5035d20600a 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -256,14 +256,14 @@ _codecs_escape_encode_impl(PyObject *module, PyObject *data, _codecs.utf_7_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=0cd3a944a32a4089 input=22c395d357815d26]*/ +/*[clinic end generated code: output=0cd3a944a32a4089 input=dbf8c8998102dc7d]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeUTF7Stateful(data->buf, data->len, @@ -276,14 +276,14 @@ _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_8_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=10f74dec8d9bb8bf input=f611b3867352ba59]*/ +/*[clinic end generated code: output=10f74dec8d9bb8bf input=ca06bc8a9c970e25]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeUTF8Stateful(data->buf, data->len, @@ -296,14 +296,14 @@ _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_16_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=783b442abcbcc2d0 input=191d360bd7309180]*/ +/*[clinic end generated code: output=783b442abcbcc2d0 input=5b0f52071ba6cadc]*/ { int byteorder = 0; /* This is overwritten unless final is true. */ @@ -318,14 +318,14 @@ _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_16_le_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=899b9e6364379dcd input=c6904fdc27fb4724]*/ +/*[clinic end generated code: output=899b9e6364379dcd input=115bd8c7b783d0bf]*/ { int byteorder = -1; /* This is overwritten unless final is true. */ @@ -340,14 +340,14 @@ _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_16_be_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_be_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=49f6465ea07669c8 input=e49012400974649b]*/ +/*[clinic end generated code: output=49f6465ea07669c8 input=63131422b01f9cb4]*/ { int byteorder = 1; /* This is overwritten unless final is true. */ @@ -370,14 +370,14 @@ _codecs.utf_16_ex_decode data: Py_buffer errors: str(accept={str, NoneType}) = None byteorder: int = 0 - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int byteorder, int final) -/*[clinic end generated code: output=0f385f251ecc1988 input=5a9c19f2e6b6cf0e]*/ +/*[clinic end generated code: output=0f385f251ecc1988 input=f368a51cf384bf4c]*/ { /* This is overwritten unless final is true. */ Py_ssize_t consumed = data->len; @@ -394,14 +394,14 @@ _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_32_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=2fc961807f7b145f input=fd7193965627eb58]*/ +/*[clinic end generated code: output=2fc961807f7b145f input=fcdf3658c5e9b5f3]*/ { int byteorder = 0; /* This is overwritten unless final is true. */ @@ -416,14 +416,14 @@ _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_32_le_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=ec8f46b67a94f3e6 input=9078ec70acfe7613]*/ +/*[clinic end generated code: output=ec8f46b67a94f3e6 input=12220556e885f817]*/ { int byteorder = -1; /* This is overwritten unless final is true. */ @@ -438,14 +438,14 @@ _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_32_be_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_be_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=ff82bae862c92c4e input=f1ae1bbbb86648ff]*/ +/*[clinic end generated code: output=ff82bae862c92c4e input=2bc669b4781598db]*/ { int byteorder = 1; /* This is overwritten unless final is true. */ @@ -468,14 +468,14 @@ _codecs.utf_32_ex_decode data: Py_buffer errors: str(accept={str, NoneType}) = None byteorder: int = 0 - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int byteorder, int final) -/*[clinic end generated code: output=6bfb177dceaf4848 input=e46a73bc859d0bd0]*/ +/*[clinic end generated code: output=6bfb177dceaf4848 input=4a2323d0013620df]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeUTF32Stateful(data->buf, data->len, @@ -490,14 +490,14 @@ _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data, _codecs.unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = True + final: bool = True / [clinic start generated code]*/ static PyObject * _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=b284f97b12c635ee input=6154f039a9f7c639]*/ +/*[clinic end generated code: output=b284f97b12c635ee input=15019f081ffe272b]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = _PyUnicode_DecodeUnicodeEscapeStateful(data->buf, data->len, @@ -510,14 +510,14 @@ _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, _codecs.raw_unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = True + final: bool = True / [clinic start generated code]*/ static PyObject * _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=11dbd96301e2879e input=2d166191beb3235a]*/ +/*[clinic end generated code: output=11dbd96301e2879e input=b93f823aa8c343ad]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = _PyUnicode_DecodeRawUnicodeEscapeStateful(data->buf, data->len, @@ -586,14 +586,14 @@ _codecs_charmap_decode_impl(PyObject *module, Py_buffer *data, _codecs.mbcs_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=39b65b8598938c4b input=1c1d50f08fa53789]*/ +/*[clinic end generated code: output=39b65b8598938c4b input=f144ad1ed6d8f5a6]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeMBCSStateful(data->buf, data->len, @@ -605,14 +605,14 @@ _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data, _codecs.oem_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_oem_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=da1617612f3fcad8 input=81b67cba811022e5]*/ +/*[clinic end generated code: output=da1617612f3fcad8 input=629bf87376d211b4]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeCodePageStateful(CP_OEMCP, @@ -625,14 +625,14 @@ _codecs.code_page_decode codepage: int data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_code_page_decode_impl(PyObject *module, int codepage, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=53008ea967da3fff input=c5f58d036cb63575]*/ +/*[clinic end generated code: output=53008ea967da3fff input=6a32589b0658c277]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeCodePageStateful(codepage, diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 5146b4004a14..5691a419a32f 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -2371,7 +2371,7 @@ _curses.window.touchline start: int count: int [ - changed: bool(accept={int}) = True + changed: bool = True ] / @@ -2384,7 +2384,7 @@ as having been changed (changed=True) or unchanged (changed=False). static PyObject * _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed) -/*[clinic end generated code: output=65d05b3f7438c61d input=918ad1cbdadf93ea]*/ +/*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ { if (!group_right_1) { return PyCursesCheckERR(touchline(self->win, start, count), "touchline"); @@ -2706,7 +2706,7 @@ NoArgTrueFalseFunctionBody(can_change_color) /*[clinic input] _curses.cbreak - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling nocbreak(). / @@ -2721,7 +2721,7 @@ Calling first raw() then cbreak() leaves the terminal in cbreak mode. static PyObject * _curses_cbreak_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9f9dee9664769751 input=150be619eb1f1458]*/ +/*[clinic end generated code: output=9f9dee9664769751 input=c7d0bddda93016c1]*/ NoArgOrFlagNoReturnFunctionBody(cbreak, flag) /*[clinic input] @@ -2870,7 +2870,7 @@ NoArgNoReturnFunctionBody(doupdate) /*[clinic input] _curses.echo - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling noecho(). / @@ -2881,7 +2881,7 @@ In echo mode, each character input is echoed to the screen as it is entered. static PyObject * _curses_echo_impl(PyObject *module, int flag) -/*[clinic end generated code: output=03acb2ddfa6c8729 input=2e9e891d637eac5d]*/ +/*[clinic end generated code: output=03acb2ddfa6c8729 input=86cd4d5bb1d569c0]*/ NoArgOrFlagNoReturnFunctionBody(echo, flag) /*[clinic input] @@ -3496,14 +3496,14 @@ _curses_set_tabsize_impl(PyObject *module, int size) /*[clinic input] _curses.intrflush - flag: bool(accept={int}) + flag: bool / [clinic start generated code]*/ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) -/*[clinic end generated code: output=c1986df35e999a0f input=fcba57bb28dfd795]*/ +/*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ { PyCursesInitialised; @@ -3605,7 +3605,7 @@ NoArgReturnStringFunctionBody(longname) /*[clinic input] _curses.meta - yes: bool(accept={int}) + yes: bool / Enable/disable meta keys. @@ -3616,7 +3616,7 @@ allow only 7-bit characters. static PyObject * _curses_meta_impl(PyObject *module, int yes) -/*[clinic end generated code: output=22f5abda46a605d8 input=af9892e3a74f35db]*/ +/*[clinic end generated code: output=22f5abda46a605d8 input=cfe7da79f51d0e30]*/ { PyCursesInitialised; @@ -3766,7 +3766,7 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, /*[clinic input] _curses.nl - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling nonl(). / @@ -3778,7 +3778,7 @@ newline into return and line-feed on output. Newline mode is initially on. static PyObject * _curses_nl_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b39cc0ffc9015003 input=cf36a63f7b86e28a]*/ +/*[clinic end generated code: output=b39cc0ffc9015003 input=18e3e9c6e8cfcf6f]*/ NoArgOrFlagNoReturnFunctionBody(nl, flag) /*[clinic input] @@ -3925,7 +3925,7 @@ _curses_putp_impl(PyObject *module, const char *string) /*[clinic input] _curses.qiflush - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling noqiflush(). / @@ -3937,7 +3937,7 @@ will be flushed when the INTR, QUIT and SUSP characters are read. static PyObject * _curses_qiflush_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9167e862f760ea30 input=e9e4a389946a0dbc]*/ +/*[clinic end generated code: output=9167e862f760ea30 input=6ec8b3e2b717ec40]*/ { PyCursesInitialised; @@ -4018,7 +4018,7 @@ _curses_update_lines_cols_impl(PyObject *module) /*[clinic input] _curses.raw - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling noraw(). / @@ -4031,7 +4031,7 @@ curses input functions one by one. static PyObject * _curses_raw_impl(PyObject *module, int flag) -/*[clinic end generated code: output=a750e4b342be015b input=e36d8db27832b848]*/ +/*[clinic end generated code: output=a750e4b342be015b input=4b447701389fb4df]*/ NoArgOrFlagNoReturnFunctionBody(raw, flag) /*[clinic input] @@ -4503,7 +4503,7 @@ _curses_unget_wch(PyObject *module, PyObject *ch) /*[clinic input] _curses.use_env - flag: bool(accept={int}) + flag: bool / Use environment variables LINES and COLUMNS. @@ -4520,7 +4520,7 @@ not set). static PyObject * _curses_use_env_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b2c445e435c0b164 input=1778eb1e9151ea37]*/ +/*[clinic end generated code: output=b2c445e435c0b164 input=06ac30948f2d78e4]*/ { use_env(flag); Py_RETURN_NONE; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 121d9617e188..af5950cf66c1 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -59,7 +59,7 @@ PyDoc_STRVAR(module_doc, " I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n" " possible.\n" ); - + /* * The main open() function @@ -74,7 +74,7 @@ _io.open encoding: str(accept={str, NoneType}) = None errors: str(accept={str, NoneType}) = None newline: str(accept={str, NoneType}) = None - closefd: bool(accept={int}) = True + closefd: bool = True opener: object = None Open file and return a stream. Raise OSError upon failure. @@ -196,7 +196,7 @@ static PyObject * _io_open_impl(PyObject *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=aefafc4ce2b46dc0 input=5bb37f174cb2fb11]*/ +/*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/ { unsigned i; @@ -204,8 +204,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, int text = 0, binary = 0; char rawmode[6], *m; - int line_buffering, is_number; - long isatty = 0; + int line_buffering, is_number, isatty = 0; PyObject *raw, *modeobj = NULL, *buffer, *wrapper, *result = NULL, *path_or_fd = NULL; @@ -345,9 +344,9 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(isatty)); if (res == NULL) goto error; - isatty = PyLong_AsLong(res); + isatty = PyObject_IsTrue(res); Py_DECREF(res); - if (isatty == -1 && PyErr_Occurred()) + if (isatty < 0) goto error; } @@ -509,7 +508,7 @@ _io_open_code_impl(PyObject *module, PyObject *path) { return PyFile_OpenCodeObject(path); } - + /* * Private helpers for the io module. */ diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index b38738486f68..4d76e333b0f2 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -280,8 +280,8 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } } if (args[6]) { - closefd = _PyLong_AsInt(args[6]); - if (closefd == -1 && PyErr_Occurred()) { + closefd = PyObject_IsTrue(args[6]); + if (closefd < 0) { goto exit; } if (!--noptargs) { @@ -407,4 +407,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=1f8001287a423470 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f387eba3f4c0254a input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index a925b94fe075..b6e9bd5b65a0 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -116,8 +116,8 @@ _io_FileIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - closefd = _PyLong_AsInt(fastargs[2]); - if (closefd == -1 && PyErr_Occurred()) { + closefd = PyObject_IsTrue(fastargs[2]); + if (closefd < 0) { goto exit; } if (!--noptargs) { @@ -466,4 +466,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=ff479a26cab0d479 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=27f883807a6c29ae input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 038f0a5c209d..db968e884cc8 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -68,8 +68,8 @@ _io_IncrementalNewlineDecoder___init__(PyObject *self, PyObject *args, PyObject goto exit; } decoder = fastargs[0]; - translate = _PyLong_AsInt(fastargs[1]); - if (translate == -1 && PyErr_Occurred()) { + translate = PyObject_IsTrue(fastargs[1]); + if (translate < 0) { goto exit; } if (!noptargs) { @@ -137,8 +137,8 @@ _io_IncrementalNewlineDecoder_decode(nldecoder_object *self, PyObject *const *ar if (!noptargs) { goto skip_optional_pos; } - final = _PyLong_AsInt(args[1]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[1]); + if (final < 0) { goto exit; } skip_optional_pos: @@ -331,16 +331,16 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[4]) { - line_buffering = _PyLong_AsInt(fastargs[4]); - if (line_buffering == -1 && PyErr_Occurred()) { + line_buffering = PyObject_IsTrue(fastargs[4]); + if (line_buffering < 0) { goto exit; } if (!--noptargs) { goto skip_optional_pos; } } - write_through = _PyLong_AsInt(fastargs[5]); - if (write_through == -1 && PyErr_Occurred()) { + write_through = PyObject_IsTrue(fastargs[5]); + if (write_through < 0) { goto exit; } skip_optional_pos: @@ -769,4 +769,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=aecd376eca3cb148 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73f84b13c343b34b input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 65820a8f2ea0..df834dbde40f 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -115,8 +115,8 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - closefd = _PyLong_AsInt(fastargs[2]); - if (closefd == -1 && PyErr_Occurred()) { + closefd = PyObject_IsTrue(fastargs[2]); + if (closefd < 0) { goto exit; } if (!--noptargs) { @@ -407,4 +407,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=08ae244e9a44da55 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4920e9068e0cf08a input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 659297ef1b1d..d1a183cedac5 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -198,7 +198,7 @@ extern int _Py_open_cloexec_works; _io.FileIO.__init__ file as nameobj: object mode: str = "r" - closefd: bool(accept={int}) = True + closefd: bool = True opener: object = None Open a file. @@ -219,7 +219,7 @@ results in functionality similar to passing None). static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=23413f68e6484bbd input=1596c9157a042a39]*/ +/*[clinic end generated code: output=23413f68e6484bbd input=588aac967e0ba74b]*/ { #ifdef MS_WINDOWS Py_UNICODE *widename = NULL; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3091f6efafcc..32ab8a44c621 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -212,7 +212,7 @@ typedef struct { /*[clinic input] _io.IncrementalNewlineDecoder.__init__ decoder: object - translate: int + translate: bool errors: object(c_default="NULL") = "strict" Codec used when reading a file in universal newlines mode. @@ -229,7 +229,7 @@ static int _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, PyObject *decoder, int translate, PyObject *errors) -/*[clinic end generated code: output=fbd04d443e764ec2 input=89db6b19c6b126bf]*/ +/*[clinic end generated code: output=fbd04d443e764ec2 input=ed547aa257616b0e]*/ { if (errors == NULL) { @@ -484,13 +484,13 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself, /*[clinic input] _io.IncrementalNewlineDecoder.decode input: object - final: bool(accept={int}) = False + final: bool = False [clinic start generated code]*/ static PyObject * _io_IncrementalNewlineDecoder_decode_impl(nldecoder_object *self, PyObject *input, int final) -/*[clinic end generated code: output=0d486755bb37a66e input=a4ea97f26372d866]*/ +/*[clinic end generated code: output=0d486755bb37a66e input=90e223c70322c5cd]*/ { return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final); } @@ -1023,8 +1023,8 @@ _io.TextIOWrapper.__init__ encoding: str(accept={str, NoneType}) = None errors: object = None newline: str(accept={str, NoneType}) = None - line_buffering: bool(accept={int}) = False - write_through: bool(accept={int}) = False + line_buffering: bool = False + write_through: bool = False Character and line based layer over a BufferedIOBase object, buffer. @@ -1061,7 +1061,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, const char *encoding, PyObject *errors, const char *newline, int line_buffering, int write_through) -/*[clinic end generated code: output=72267c0c01032ed2 input=72590963698f289b]*/ +/*[clinic end generated code: output=72267c0c01032ed2 input=e6cfaaaf6059d4f5]*/ { PyObject *raw, *codec_info = NULL; PyObject *res; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 5c1a6dd86fc5..d5de64b4ac3d 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -235,7 +235,7 @@ winconsoleio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _io._WindowsConsoleIO.__init__ file as nameobj: object mode: str = "r" - closefd: bool(accept={int}) = True + closefd: bool = True opener: object = None Open a console buffer by file descriptor. @@ -249,7 +249,7 @@ static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=3fd9cbcdd8d95429 input=06ae4b863c63244b]*/ +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=7a3eed6bbe998fd9]*/ { const char *s; wchar_t *name = NULL; diff --git a/Modules/_json.c b/Modules/_json.c index 81431aa1041c..429b4ee0fa8d 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -556,7 +556,7 @@ py_scanstring(PyObject* Py_UNUSED(self), PyObject *args) Py_ssize_t end; Py_ssize_t next_end = -1; int strict = 1; - if (!PyArg_ParseTuple(args, "On|i:scanstring", &pystr, &end, &strict)) { + if (!PyArg_ParseTuple(args, "On|p:scanstring", &pystr, &end, &strict)) { return NULL; } if (PyUnicode_Check(pystr)) { diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index cb80c8d33932..37170bbea56a 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -667,7 +667,7 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) int subcalls = -1; int builtins = -1; static char *kwlist[] = {"subcalls", "builtins", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ii:enable", + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", kwlist, &subcalls, &builtins)) return NULL; if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { @@ -770,7 +770,7 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) static char *kwlist[] = {"timer", "timeunit", "subcalls", "builtins", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|Odii:Profiler", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kw, "|Odpp:Profiler", kwlist, &timer, &timeunit, &subcalls, &builtins)) return -1; diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index dce0366c266f..35347169bc15 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -65,8 +65,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ goto skip_optional_pos; } if (args[0]) { - blocking = _PyLong_AsInt(args[0]); - if (blocking == -1 && PyErr_Occurred()) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) { goto exit; } if (!--noptargs) { @@ -162,8 +162,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ goto skip_optional_pos; } if (args[0]) { - blocking = _PyLong_AsInt(args[0]); - if (blocking == -1 && PyErr_Occurred()) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) { goto exit; } if (!--noptargs) { @@ -275,8 +275,8 @@ _multiprocessing_SemLock(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - unlink = _PyLong_AsInt(fastargs[4]); - if (unlink == -1 && PyErr_Occurred()) { + unlink = PyObject_IsTrue(fastargs[4]); + if (unlink < 0) { goto exit; } return_value = _multiprocessing_SemLock_impl(type, kind, value, maxvalue, name, unlink); @@ -542,4 +542,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=720d7d0066dc0954 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dae57a702cc01512 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index 58fb0eb96aee..897b8db7110a 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -79,7 +79,7 @@ _GetSemaphoreValue(HANDLE handle, long *value) /*[clinic input] _multiprocessing.SemLock.acquire - block as blocking: bool(accept={int}) = True + block as blocking: bool = True timeout as timeout_obj: object = None Acquire the semaphore/lock. @@ -88,7 +88,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=86f05662cf753eb4]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ { double timeout; DWORD res, full_msecs, nhandles; @@ -295,7 +295,7 @@ sem_timedwait_save(sem_t *sem, struct timespec *deadline, PyThreadState *_save) /*[clinic input] _multiprocessing.SemLock.acquire - block as blocking: bool(accept={int}) = True + block as blocking: bool = True timeout as timeout_obj: object = None Acquire the semaphore/lock. @@ -304,7 +304,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=86f05662cf753eb4]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ { int res, err = 0; struct timespec deadline = {0}; @@ -474,14 +474,14 @@ _multiprocessing.SemLock.__new__ value: int maxvalue: int name: str - unlink: bool(accept={int}) + unlink: bool [clinic start generated code]*/ static PyObject * _multiprocessing_SemLock_impl(PyTypeObject *type, int kind, int value, int maxvalue, const char *name, int unlink) -/*[clinic end generated code: output=30727e38f5f7577a input=b378c3ee27d3a0fa]*/ +/*[clinic end generated code: output=30727e38f5f7577a input=fdaeb69814471c5b]*/ { SEM_HANDLE handle = SEM_FAILED; PyObject *result; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 717b1cf22021..b7563ee8250a 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -814,7 +814,7 @@ subprocess_fork_exec(PyObject *module, PyObject *args) int allow_vfork; if (!PyArg_ParseTuple( - args, "OOpO!OOiiiiiiiiii" _Py_PARSE_PID "OOOiOp:fork_exec", + args, "OOpO!OOiiiiiiiipp" _Py_PARSE_PID "OOOiOp:fork_exec", &process_args, &executable_list, &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 1f9841c368b3..4c3fd1bd2741 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -100,8 +100,8 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[4]) { - check_same_thread = _PyLong_AsInt(fastargs[4]); - if (check_same_thread == -1 && PyErr_Occurred()) { + check_same_thread = PyObject_IsTrue(fastargs[4]); + if (check_same_thread < 0) { goto exit; } if (!--noptargs) { @@ -305,8 +305,8 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO goto skip_optional_kwonly; } if (args[3]) { - readonly = _PyLong_AsInt(args[3]); - if (readonly == -1 && PyErr_Occurred()) { + readonly = PyObject_IsTrue(args[3]); + if (readonly < 0) { goto exit; } if (!--noptargs) { @@ -831,8 +831,8 @@ pysqlite_connection_enable_load_extension(pysqlite_Connection *self, PyObject *a PyObject *return_value = NULL; int onoff; - onoff = _PyLong_AsInt(arg); - if (onoff == -1 && PyErr_Occurred()) { + onoff = PyObject_IsTrue(arg); + if (onoff < 0) { goto exit; } return_value = pysqlite_connection_enable_load_extension_impl(self, onoff); @@ -1532,4 +1532,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=20e929a7a7d62a01 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f10306e10427488b input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 2854c1b5c31b..4c07d5e0b61f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -197,7 +197,7 @@ _sqlite3.Connection.__init__ as pysqlite_connection_init timeout: double = 5.0 detect_types: int = 0 isolation_level: IsolationLevel = "" - check_same_thread: bool(accept={int}) = True + check_same_thread: bool = True factory: object(c_default='(PyObject*)clinic_state()->ConnectionType') = ConnectionType cached_statements as cache_size: int = 128 uri: bool = False @@ -212,7 +212,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int check_same_thread, PyObject *factory, int cache_size, int uri, enum autocommit_mode autocommit) -/*[clinic end generated code: output=cba057313ea7712f input=b21abce28ebcd304]*/ +/*[clinic end generated code: output=cba057313ea7712f input=9b0ab6c12f674fa3]*/ { if (PySys_Audit("sqlite3.connect", "O", database) < 0) { return -1; @@ -485,7 +485,7 @@ _sqlite3.Connection.blobopen as blobopen Row index. / * - readonly: bool(accept={int}) = False + readonly: bool = False Open the BLOB without write permissions. name: str = "main" Database name. @@ -496,7 +496,7 @@ Open and return a BLOB object. static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, int row, int readonly, const char *name) -/*[clinic end generated code: output=0c8e2e58516d0b5c input=1e7052516acfc94d]*/ +/*[clinic end generated code: output=0c8e2e58516d0b5c input=fa73c83aa7a7ddee]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1564,7 +1564,7 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, /*[clinic input] _sqlite3.Connection.enable_load_extension as pysqlite_connection_enable_load_extension - enable as onoff: bool(accept={int}) + enable as onoff: bool / Enable dynamic loading of SQLite extension modules. @@ -1573,7 +1573,7 @@ Enable dynamic loading of SQLite extension modules. static PyObject * pysqlite_connection_enable_load_extension_impl(pysqlite_Connection *self, int onoff) -/*[clinic end generated code: output=9cac37190d388baf input=5f00e93f7a9d3540]*/ +/*[clinic end generated code: output=9cac37190d388baf input=2a1e87931486380f]*/ { int rc; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2885774295b0..591eb91dd0f3 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4168,7 +4168,7 @@ _ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) /*[clinic input] _ssl._SSLContext._wrap_socket sock: object(subclass_of="get_state_ctx(self)->Sock_Type") - server_side: int + server_side: bool server_hostname as hostname_obj: object = None * owner: object = None @@ -4180,7 +4180,7 @@ static PyObject * _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=f103f238633940b4 input=f5916eadbc6eae81]*/ +/*[clinic end generated code: output=f103f238633940b4 input=700ca8fedff53994]*/ { char *hostname = NULL; PyObject *res; @@ -4205,7 +4205,7 @@ _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, _ssl._SSLContext._wrap_bio incoming: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") outgoing: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") - server_side: int + server_side: bool server_hostname as hostname_obj: object = None * owner: object = None @@ -4218,7 +4218,7 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, PySSLMemoryBIO *outgoing, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=5c5d6d9b41f99332 input=331edeec9c738382]*/ +/*[clinic end generated code: output=5c5d6d9b41f99332 input=a9205d097fd45a82]*/ { char *hostname = NULL; PyObject *res; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index ec8b6d881124..d323ca83dcff 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -135,7 +135,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, *timeout = unset_timeout ; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|iO:acquire", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pO:acquire", kwlist, &blocking, &timeout_obj)) return -1; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 93d4474f65d6..d4a129058702 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2832,7 +2832,7 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) { int wantobjects = -1; - if (!PyArg_ParseTuple(args, "|i:wantobjects", &wantobjects)) + if (!PyArg_ParseTuple(args, "|p:wantobjects", &wantobjects)) return NULL; if (wantobjects == -1) return PyBool_FromLong(((TkappObject*)self)->wantobjects); @@ -2978,11 +2978,11 @@ _tkinter.create screenName: str(accept={str, NoneType}) = None baseName: str = "" className: str = "Tk" - interactive: bool(accept={int}) = False - wantobjects: bool(accept={int}) = False - wantTk: bool(accept={int}) = True + interactive: bool = False + wantobjects: bool = False + wantTk: bool = True if false, then Tk_Init() doesn't get called - sync: bool(accept={int}) = False + sync: bool = False if true, then pass -sync to wish use: str(accept={str, NoneType}) = None if not None, then pass -use to wish @@ -2995,7 +2995,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName, const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=e3315607648e6bb4 input=da9b17ee7358d862]*/ +/*[clinic end generated code: output=e3315607648e6bb4 input=09afef9adea70a19]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index bb4514c36bc7..f4d982b15d40 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -404,13 +404,13 @@ _winapi_CloseHandle_impl(PyObject *module, HANDLE handle) _winapi.ConnectNamedPipe handle: HANDLE - overlapped as use_overlapped: bool(accept={int}) = False + overlapped as use_overlapped: bool = False [clinic start generated code]*/ static PyObject * _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle, int use_overlapped) -/*[clinic end generated code: output=335a0e7086800671 input=34f937c1c86e5e68]*/ +/*[clinic end generated code: output=335a0e7086800671 input=a80e56e8bd370e31]*/ { BOOL success; OverlappedObject *overlapped = NULL; @@ -1576,13 +1576,13 @@ _winapi.ReadFile handle: HANDLE size: DWORD - overlapped as use_overlapped: bool(accept={int}) = False + overlapped as use_overlapped: bool = False [clinic start generated code]*/ static PyObject * _winapi_ReadFile_impl(PyObject *module, HANDLE handle, DWORD size, int use_overlapped) -/*[clinic end generated code: output=d3d5b44a8201b944 input=08c439d03a11aac5]*/ +/*[clinic end generated code: output=d3d5b44a8201b944 input=4f82f8e909ad91ad]*/ { DWORD nread; PyObject *buf; @@ -1862,13 +1862,13 @@ _winapi.WriteFile handle: HANDLE buffer: object - overlapped as use_overlapped: bool(accept={int}) = False + overlapped as use_overlapped: bool = False [clinic start generated code]*/ static PyObject * _winapi_WriteFile_impl(PyObject *module, HANDLE handle, PyObject *buffer, int use_overlapped) -/*[clinic end generated code: output=2ca80f6bf3fa92e3 input=11eae2a03aa32731]*/ +/*[clinic end generated code: output=2ca80f6bf3fa92e3 input=2badb008c8a2e2a0]*/ { Py_buffer _buf, *buf; DWORD len, written; diff --git a/Modules/binascii.c b/Modules/binascii.c index ffc2c5941361..95ddb26988d6 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -303,14 +303,14 @@ binascii.b2a_uu data: Py_buffer / * - backtick: bool(accept={int}) = False + backtick: bool = False Uuencode line of data. [clinic start generated code]*/ static PyObject * binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) -/*[clinic end generated code: output=b1b99de62d9bbeb8 input=b26bc8d32b6ed2f6]*/ +/*[clinic end generated code: output=b1b99de62d9bbeb8 input=beb27822241095cd]*/ { unsigned char *ascii_data; const unsigned char *bin_data; @@ -375,7 +375,7 @@ binascii.a2b_base64 data: ascii_buffer / * - strict_mode: bool(accept={int}) = False + strict_mode: bool = False Decode a line of base64 data. @@ -386,7 +386,7 @@ Decode a line of base64 data. static PyObject * binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) -/*[clinic end generated code: output=5409557788d4f975 input=3a30c4e3528317c6]*/ +/*[clinic end generated code: output=5409557788d4f975 input=c0c15fd0f8f9a62d]*/ { assert(data->len >= 0); @@ -521,14 +521,14 @@ binascii.b2a_base64 data: Py_buffer / * - newline: bool(accept={int}) = True + newline: bool = True Base64-code line of data. [clinic start generated code]*/ static PyObject * binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) -/*[clinic end generated code: output=4ad62c8e8485d3b3 input=6083dac5777fa45d]*/ +/*[clinic end generated code: output=4ad62c8e8485d3b3 input=0e20ff59c5f2e3e1]*/ { unsigned char *ascii_data; const unsigned char *bin_data; @@ -952,14 +952,14 @@ binascii_unhexlify_impl(PyObject *module, Py_buffer *hexstr) binascii.a2b_qp data: ascii_buffer - header: bool(accept={int}) = False + header: bool = False Decode a string of qp-encoded data. [clinic start generated code]*/ static PyObject * binascii_a2b_qp_impl(PyObject *module, Py_buffer *data, int header) -/*[clinic end generated code: output=e99f7846cfb9bc53 input=bf6766fea76cce8f]*/ +/*[clinic end generated code: output=e99f7846cfb9bc53 input=bdfb31598d4e47b9]*/ { Py_ssize_t in, out; char ch; @@ -1048,9 +1048,9 @@ to_hex (unsigned char ch, unsigned char *s) binascii.b2a_qp data: Py_buffer - quotetabs: bool(accept={int}) = False - istext: bool(accept={int}) = True - header: bool(accept={int}) = False + quotetabs: bool = False + istext: bool = True + header: bool = False Encode a string using quoted-printable encoding. @@ -1062,7 +1062,7 @@ are both encoded. When quotetabs is set, space and tabs are encoded. static PyObject * binascii_b2a_qp_impl(PyObject *module, Py_buffer *data, int quotetabs, int istext, int header) -/*[clinic end generated code: output=e9884472ebb1a94c input=21fb7eea4a184ba6]*/ +/*[clinic end generated code: output=e9884472ebb1a94c input=e9102879afb0defd]*/ { Py_ssize_t in, out; const unsigned char *databuf; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b7e340e68796..1b41c231eac5 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -246,8 +246,8 @@ _multibytecodec_MultibyteIncrementalEncoder_encode(MultibyteIncrementalEncoderOb if (!noptargs) { goto skip_optional_pos; } - final = _PyLong_AsInt(args[1]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[1]); + if (final < 0) { goto exit; } skip_optional_pos: @@ -381,8 +381,8 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb if (!noptargs) { goto skip_optional_pos; } - final = _PyLong_AsInt(args[1]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[1]); + if (final < 0) { goto exit; } skip_optional_pos: @@ -690,4 +690,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=b034ec7126c11bde input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f0e8dacddb0ac76 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 1d77fd33ac3b..8564494f6262 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -893,14 +893,14 @@ decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx, _multibytecodec.MultibyteIncrementalEncoder.encode input: object - final: bool(accept={int}) = False + final: bool = False [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEncoderObject *self, PyObject *input, int final) -/*[clinic end generated code: output=123361b6c505e2c1 input=093a1ddbb2fc6721]*/ +/*[clinic end generated code: output=123361b6c505e2c1 input=bd5f7d40d43e99b0]*/ { return encoder_encode_stateful(STATEFUL_ECTX(self), input, final); } @@ -1114,14 +1114,14 @@ static PyType_Spec encoder_spec = { _multibytecodec.MultibyteIncrementalDecoder.decode input: Py_buffer - final: bool(accept={int}) = False + final: bool = False [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDecoderObject *self, Py_buffer *input, int final) -/*[clinic end generated code: output=b9b9090e8a9ce2ba input=c9132b24d503eb1d]*/ +/*[clinic end generated code: output=b9b9090e8a9ce2ba input=8795fbb20860027a]*/ { MultibyteDecodeBuffer buf; char *data, *wdata = NULL; diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 25db060cd900..f11bcc8815b9 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -450,8 +450,8 @@ _codecs_utf_7_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -520,8 +520,8 @@ _codecs_utf_8_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -590,8 +590,8 @@ _codecs_utf_16_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -660,8 +660,8 @@ _codecs_utf_16_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -730,8 +730,8 @@ _codecs_utf_16_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -809,8 +809,8 @@ _codecs_utf_16_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - final = _PyLong_AsInt(args[3]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[3]); + if (final < 0) { goto exit; } skip_optional: @@ -879,8 +879,8 @@ _codecs_utf_32_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -949,8 +949,8 @@ _codecs_utf_32_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1019,8 +1019,8 @@ _codecs_utf_32_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1098,8 +1098,8 @@ _codecs_utf_32_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - final = _PyLong_AsInt(args[3]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[3]); + if (final < 0) { goto exit; } skip_optional: @@ -1178,8 +1178,8 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1258,8 +1258,8 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1521,8 +1521,8 @@ _codecs_mbcs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1595,8 +1595,8 @@ _codecs_oem_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1674,8 +1674,8 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - final = _PyLong_AsInt(args[3]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[3]); + if (final < 0) { goto exit; } skip_optional: @@ -2869,4 +2869,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e885abad241bc54d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=603da07cf8dfeb4b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 67fadace8639..9d99d41af5d2 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -1748,7 +1748,7 @@ _curses_window_touchline(PyCursesWindowObject *self, PyObject *args) } break; case 3: - if (!PyArg_ParseTuple(args, "iii:touchline", &start, &count, &changed)) { + if (!PyArg_ParseTuple(args, "iip:touchline", &start, &count, &changed)) { goto exit; } group_right_1 = 1; @@ -1941,8 +1941,8 @@ _curses_cbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -2177,8 +2177,8 @@ _curses_echo(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -2900,8 +2900,8 @@ _curses_intrflush(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int flag; - flag = _PyLong_AsInt(arg); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(arg); + if (flag < 0) { goto exit; } return_value = _curses_intrflush_impl(module, flag); @@ -3064,8 +3064,8 @@ _curses_meta(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int yes; - yes = _PyLong_AsInt(arg); - if (yes == -1 && PyErr_Occurred()) { + yes = PyObject_IsTrue(arg); + if (yes < 0) { goto exit; } return_value = _curses_meta_impl(module, yes); @@ -3308,8 +3308,8 @@ _curses_nl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -3540,8 +3540,8 @@ _curses_qiflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -3603,8 +3603,8 @@ _curses_raw(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -4164,8 +4164,8 @@ _curses_use_env(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int flag; - flag = _PyLong_AsInt(arg); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(arg); + if (flag < 0) { goto exit; } return_value = _curses_use_env_impl(module, flag); @@ -4313,4 +4313,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=b2e71e2012f16197 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=27a2364193b503c1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 622e321fa1d8..2d7c98c4f014 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -757,8 +757,8 @@ _ssl__SSLContext__wrap_socket(PySSLContext *self, PyObject *const *args, Py_ssiz goto exit; } sock = args[0]; - server_side = _PyLong_AsInt(args[1]); - if (server_side == -1 && PyErr_Occurred()) { + server_side = PyObject_IsTrue(args[1]); + if (server_side < 0) { goto exit; } if (!noptargs) { @@ -855,8 +855,8 @@ _ssl__SSLContext__wrap_bio(PySSLContext *self, PyObject *const *args, Py_ssize_t goto exit; } outgoing = (PySSLMemoryBIO *)args[1]; - server_side = _PyLong_AsInt(args[2]); - if (server_side == -1 && PyErr_Occurred()) { + server_side = PyObject_IsTrue(args[2]); + if (server_side < 0) { goto exit; } if (!noptargs) { @@ -1543,4 +1543,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=9f477b0c709acb28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a3d97a19163bb044 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index a251202f9bba..96c6ee26f426 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -747,29 +747,29 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - interactive = _PyLong_AsInt(args[3]); - if (interactive == -1 && PyErr_Occurred()) { + interactive = PyObject_IsTrue(args[3]); + if (interactive < 0) { goto exit; } if (nargs < 5) { goto skip_optional; } - wantobjects = _PyLong_AsInt(args[4]); - if (wantobjects == -1 && PyErr_Occurred()) { + wantobjects = PyObject_IsTrue(args[4]); + if (wantobjects < 0) { goto exit; } if (nargs < 6) { goto skip_optional; } - wantTk = _PyLong_AsInt(args[5]); - if (wantTk == -1 && PyErr_Occurred()) { + wantTk = PyObject_IsTrue(args[5]); + if (wantTk < 0) { goto exit; } if (nargs < 7) { goto skip_optional; } - sync = _PyLong_AsInt(args[6]); - if (sync == -1 && PyErr_Occurred()) { + sync = PyObject_IsTrue(args[6]); + if (sync < 0) { goto exit; } if (nargs < 8) { @@ -865,4 +865,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=d022835d05fc8608 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2a4e3bf8448604b5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 13bf8b482cd6..891b3f851d12 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -133,7 +133,7 @@ _winapi_ConnectNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nar static const char * const _keywords[] = {"handle", "overlapped", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" F_HANDLE "|i:ConnectNamedPipe", + .format = "" F_HANDLE "|p:ConnectNamedPipe", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -972,7 +972,7 @@ _winapi_ReadFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb static const char * const _keywords[] = {"handle", "size", "overlapped", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" F_HANDLE "k|i:ReadFile", + .format = "" F_HANDLE "k|p:ReadFile", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -1220,7 +1220,7 @@ _winapi_WriteFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO static const char * const _keywords[] = {"handle", "buffer", "overlapped", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" F_HANDLE "O|i:WriteFile", + .format = "" F_HANDLE "O|p:WriteFile", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -1371,4 +1371,4 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=23ea9e176d86e026 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=edb1a9d1bbfd6394 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 23ebdff21082..63566dfb10e7 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -99,8 +99,8 @@ binascii_b2a_uu(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_kwonly; } - backtick = _PyLong_AsInt(args[1]); - if (backtick == -1 && PyErr_Occurred()) { + backtick = PyObject_IsTrue(args[1]); + if (backtick < 0) { goto exit; } skip_optional_kwonly: @@ -175,8 +175,8 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_kwonly; } - strict_mode = _PyLong_AsInt(args[1]); - if (strict_mode == -1 && PyErr_Occurred()) { + strict_mode = PyObject_IsTrue(args[1]); + if (strict_mode < 0) { goto exit; } skip_optional_kwonly: @@ -250,8 +250,8 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_kwonly; } - newline = _PyLong_AsInt(args[1]); - if (newline == -1 && PyErr_Occurred()) { + newline = PyObject_IsTrue(args[1]); + if (newline < 0) { goto exit; } skip_optional_kwonly: @@ -680,8 +680,8 @@ binascii_a2b_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_pos; } - header = _PyLong_AsInt(args[1]); - if (header == -1 && PyErr_Occurred()) { + header = PyObject_IsTrue(args[1]); + if (header < 0) { goto exit; } skip_optional_pos: @@ -763,8 +763,8 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj goto skip_optional_pos; } if (args[1]) { - quotetabs = _PyLong_AsInt(args[1]); - if (quotetabs == -1 && PyErr_Occurred()) { + quotetabs = PyObject_IsTrue(args[1]); + if (quotetabs < 0) { goto exit; } if (!--noptargs) { @@ -772,16 +772,16 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[2]) { - istext = _PyLong_AsInt(args[2]); - if (istext == -1 && PyErr_Occurred()) { + istext = PyObject_IsTrue(args[2]); + if (istext < 0) { goto exit; } if (!--noptargs) { goto skip_optional_pos; } } - header = _PyLong_AsInt(args[3]); - if (header == -1 && PyErr_Occurred()) { + header = PyObject_IsTrue(args[3]); + if (header < 0) { goto exit; } skip_optional_pos: @@ -795,4 +795,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=a266ba13c374aefa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab156917c9db79d2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index f9f6ca372ec6..86251008b1bd 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3110,8 +3110,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[5]) { - resetids = _PyLong_AsInt(args[5]); - if (resetids == -1 && PyErr_Occurred()) { + resetids = PyObject_IsTrue(args[5]); + if (resetids < 0) { goto exit; } if (!--noptargs) { @@ -3119,8 +3119,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[6]) { - setsid = _PyLong_AsInt(args[6]); - if (setsid == -1 && PyErr_Occurred()) { + setsid = PyObject_IsTrue(args[6]); + if (setsid < 0) { goto exit; } if (!--noptargs) { @@ -3260,8 +3260,8 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[5]) { - resetids = _PyLong_AsInt(args[5]); - if (resetids == -1 && PyErr_Occurred()) { + resetids = PyObject_IsTrue(args[5]); + if (resetids < 0) { goto exit; } if (!--noptargs) { @@ -3269,8 +3269,8 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[6]) { - setsid = _PyLong_AsInt(args[6]); - if (setsid == -1 && PyErr_Occurred()) { + setsid = PyObject_IsTrue(args[6]); + if (setsid < 0) { goto exit; } if (!--noptargs) { @@ -10225,8 +10225,8 @@ os_set_blocking(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (fd == -1 && PyErr_Occurred()) { goto exit; } - blocking = _PyLong_AsInt(args[1]); - if (blocking == -1 && PyErr_Occurred()) { + blocking = PyObject_IsTrue(args[1]); + if (blocking < 0) { goto exit; } return_value = os_set_blocking_impl(module, fd, blocking); @@ -11549,4 +11549,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=4192d8e09e216300 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=04fd23c89ab41f75 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 0454fbc99945..34937c5d594f 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -52,8 +52,8 @@ pyexpat_xmlparser_Parse(xmlparseobject *self, PyTypeObject *cls, PyObject *const if (nargs < 2) { goto skip_optional_posonly; } - isfinal = _PyLong_AsInt(args[1]); - if (isfinal == -1 && PyErr_Occurred()) { + isfinal = PyObject_IsTrue(args[1]); + if (isfinal < 0) { goto exit; } skip_optional_posonly: @@ -498,4 +498,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=de5f664ef05ef34a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=63efc62e24a7b5a7 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index fa4c2d0cccd1..341a03a244cd 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -270,7 +270,7 @@ faulthandler_dump_traceback_py(PyObject *self, int fd; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|Oi:dump_traceback", kwlist, + "|Op:dump_traceback", kwlist, &file, &all_threads)) return NULL; @@ -546,7 +546,7 @@ faulthandler_py_enable(PyObject *self, PyObject *args, PyObject *kwargs) PyThreadState *tstate; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|Oi:enable", kwlist, &file, &all_threads)) + "|Op:enable", kwlist, &file, &all_threads)) return NULL; fd = faulthandler_get_fileno(&file); @@ -916,7 +916,7 @@ faulthandler_register_py(PyObject *self, int err; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "i|Oii:register", kwlist, + "i|Opp:register", kwlist, &signum, &file, &all_threads, &chain)) return NULL; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index e8b9bc76eec9..7869d92bf31a 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1368,12 +1368,12 @@ cycle_setstate(cycleobject *lz, PyObject *state) PyErr_SetString(PyExc_TypeError, "state is not a tuple"); return NULL; } - if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { + if (!PyArg_ParseTuple(state, "O!p", &PyList_Type, &saved, &firstpass)) { return NULL; } Py_INCREF(saved); Py_XSETREF(lz->saved, saved); - lz->firstpass = firstpass != 0; + lz->firstpass = firstpass; lz->index = 0; Py_RETURN_NONE; } diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index 79f4ebad836c..2319eac449eb 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -569,7 +569,7 @@ oss_setparameters(oss_audio_t *self, PyObject *args) if (!_is_fd_valid(self->fd)) return NULL; - if (!PyArg_ParseTuple(args, "iii|i:setparameters", + if (!PyArg_ParseTuple(args, "iii|p:setparameters", &wanted_fmt, &wanted_channels, &wanted_rate, &strict)) return NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 95ecf1c7c4b2..7fc8aef9b303 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6324,9 +6324,9 @@ os.posix_spawn A sequence of file action tuples. setpgroup: object = NULL The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. - resetids: bool(accept={int}) = False + resetids: bool = False If the value is `true` the POSIX_SPAWN_RESETIDS will be activated. - setsid: bool(accept={int}) = False + setsid: bool = False If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. @@ -6344,7 +6344,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=14a1098c566bc675 input=8c6305619a00ad04]*/ +/*[clinic end generated code: output=14a1098c566bc675 input=808aed1090d84e33]*/ { return py_posix_spawn(0, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -6370,9 +6370,9 @@ os.posix_spawnp A sequence of file action tuples. setpgroup: object = NULL The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. - resetids: bool(accept={int}) = False + resetids: bool = False If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. - setsid: bool(accept={int}) = False + setsid: bool = False If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. @@ -6390,7 +6390,7 @@ os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=7b9aaefe3031238d input=c1911043a22028da]*/ +/*[clinic end generated code: output=7b9aaefe3031238d input=9e89e616116752a1]*/ { return py_posix_spawn(1, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -13528,7 +13528,7 @@ os_get_blocking_impl(PyObject *module, int fd) /*[clinic input] os.set_blocking fd: int - blocking: bool(accept={int}) + blocking: bool / Set the blocking mode of the specified file descriptor. @@ -13539,7 +13539,7 @@ clear the O_NONBLOCK flag otherwise. static PyObject * os_set_blocking_impl(PyObject *module, int fd, int blocking) -/*[clinic end generated code: output=384eb43aa0762a9d input=bf5c8efdc5860ff3]*/ +/*[clinic end generated code: output=384eb43aa0762a9d input=7e9dfc9b14804dd4]*/ { int result; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0e0a9cf7cc2c..2440798bff7e 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -710,7 +710,7 @@ pyexpat.xmlparser.Parse cls: defining_class data: object - isfinal: bool(accept={int}) = False + isfinal: bool = False / Parse XML data. @@ -721,7 +721,7 @@ Parse XML data. static PyObject * pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *data, int isfinal) -/*[clinic end generated code: output=8faffe07fe1f862a input=fc97f833558ca715]*/ +/*[clinic end generated code: output=8faffe07fe1f862a input=d0eb2a69fab3b9f1]*/ { const char *s; Py_ssize_t slen; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 4ecb88cef722..7d438448ef36 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2928,8 +2928,8 @@ sock_setblocking(PySocketSockObject *s, PyObject *arg) { long block; - block = PyLong_AsLong(arg); - if (block == -1 && PyErr_Occurred()) + block = PyObject_IsTrue(arg); + if (block < 0) return NULL; s->sock_timeout = _PyTime_FromSeconds(block ? -1 : 0); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index f24690a02bda..072089e39aa2 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2012,7 +2012,7 @@ bytearray_join(PyByteArrayObject *self, PyObject *iterable_of_bytes) /*[clinic input] bytearray.splitlines - keepends: bool(accept={int}) = False + keepends: bool = False Return a list of the lines in the bytearray, breaking at line boundaries. @@ -2022,7 +2022,7 @@ true. static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) -/*[clinic end generated code: output=4223c94b895f6ad9 input=99a27ad959b9cf6b]*/ +/*[clinic end generated code: output=4223c94b895f6ad9 input=66b2dcdea8d093bf]*/ { return stringlib_splitlines( (PyObject*) self, PyByteArray_AS_STRING(self), diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a63f396e022f..0fd10fa00d16 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2314,7 +2314,7 @@ bytes_decode_impl(PyBytesObject *self, const char *encoding, /*[clinic input] bytes.splitlines - keepends: bool(accept={int}) = False + keepends: bool = False Return a list of the lines in the bytes, breaking at line boundaries. @@ -2324,7 +2324,7 @@ true. static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends) -/*[clinic end generated code: output=3484149a5d880ffb input=a8b32eb01ff5a5ed]*/ +/*[clinic end generated code: output=3484149a5d880ffb input=5d7b898af2fe55c0]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 142f29981607..e7bf3183af85 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -1084,8 +1084,8 @@ bytearray_splitlines(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t if (!noptargs) { goto skip_optional_pos; } - keepends = _PyLong_AsInt(args[0]); - if (keepends == -1 && PyErr_Occurred()) { + keepends = PyObject_IsTrue(args[0]); + if (keepends < 0) { goto exit; } skip_optional_pos: @@ -1287,4 +1287,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=72bfa6cac2fd6832 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=022698e8b0faa272 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 904124ec479a..060056dafbd8 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -839,8 +839,8 @@ bytes_splitlines(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_pos; } - keepends = _PyLong_AsInt(args[0]); - if (keepends == -1 && PyErr_Occurred()) { + keepends = PyObject_IsTrue(args[0]); + if (keepends < 0) { goto exit; } skip_optional_pos: @@ -1063,4 +1063,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5e0a25b7ba749a04 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31a9e4af85562612 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 926eaa5d3698..94852e996170 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -215,8 +215,8 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_kwonly; } } - reverse = _PyLong_AsInt(args[1]); - if (reverse == -1 && PyErr_Occurred()) { + reverse = PyObject_IsTrue(args[1]); + if (reverse < 0) { goto exit; } skip_optional_kwonly: @@ -382,4 +382,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=782ed6c68b1c9f83 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4e6f38b655394564 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index d803a2733bd6..f640c9975773 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -1193,8 +1193,8 @@ unicode_splitlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb if (!noptargs) { goto skip_optional_pos; } - keepends = _PyLong_AsInt(args[0]); - if (keepends == -1 && PyErr_Occurred()) { + keepends = PyObject_IsTrue(args[0]); + if (keepends < 0) { goto exit; } skip_optional_pos: @@ -1497,4 +1497,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=e775ff4154f1c935 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=05d942840635dadf input=a9049054013a1b77]*/ diff --git a/Objects/listobject.c b/Objects/listobject.c index 0e696fbffb3f..1d32915b17a1 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2227,7 +2227,7 @@ list.sort * key as keyfunc: object = None - reverse: bool(accept={int}) = False + reverse: bool = False Sort the list in ascending order and return None. @@ -2242,7 +2242,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=cb56cd179a713060]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ { MergeState ms; Py_ssize_t nremaining; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 19bde13a6f23..deeca35714b7 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -12444,7 +12444,7 @@ unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) /*[clinic input] str.splitlines as unicode_splitlines - keepends: bool(accept={int}) = False + keepends: bool = False Return a list of the lines in the string, breaking at line boundaries. @@ -12454,7 +12454,7 @@ true. static PyObject * unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=b508e180459bdd8b]*/ +/*[clinic end generated code: output=f664dcdad153ec40 input=ba6ad05ee85d2b55]*/ { return PyUnicode_Splitlines(self, keepends); } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index b3b7e8d6c505..ff96c25da5eb 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -714,7 +714,7 @@ compile as builtin_compile filename: object(converter="PyUnicode_FSDecoder") mode: str flags: int = 0 - dont_inherit: bool(accept={int}) = False + dont_inherit: bool = False optimize: int = -1 * _feature_version as feature_version: int = -1 @@ -737,7 +737,7 @@ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=40171fb92c1d580d]*/ +/*[clinic end generated code: output=b0c09c84f116d3d7 input=cc78e20e7c7682ba]*/ { PyObject *source_copy; const char *str; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 19930a519be0..89f069dd97f6 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -354,8 +354,8 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[4]) { - dont_inherit = _PyLong_AsInt(args[4]); - if (dont_inherit == -1 && PyErr_Occurred()) { + dont_inherit = PyObject_IsTrue(args[4]); + if (dont_inherit < 0) { goto exit; } if (!--noptargs) { @@ -1215,4 +1215,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=3c9497e0ffeb8a30 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=973da43fa65aa727 input=a9049054013a1b77]*/ From webhook-mailer at python.org Sat Dec 3 19:48:46 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sun, 04 Dec 2022 00:48:46 -0000 Subject: [Python-checkins] gh-85747: "Preface" section of asyncio-eventloop.rst: Switch to active voice and suggest other edits (#99784) Message-ID: https://github.com/python/cpython/commit/bf26bdf6ac04878fc720e78422991aaedb9808a1 commit: bf26bdf6ac04878fc720e78422991aaedb9808a1 branch: main author: Brian Skinn committer: gvanrossum date: 2022-12-03T16:48:41-08:00 summary: gh-85747: "Preface" section of asyncio-eventloop.rst: Switch to active voice and suggest other edits (#99784) files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index d0a1ed2b99e5..0bcaed5477fa 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -33,7 +33,8 @@ an event loop: Return the running event loop in the current OS thread. - If there is no running event loop a :exc:`RuntimeError` is raised. + Raise a :exc:`RuntimeError` if there is no running event loop. + This function can only be called from a coroutine or a callback. .. versionadded:: 3.7 @@ -52,17 +53,19 @@ an event loop: :func:`get_running_loop` function is preferred to :func:`get_event_loop` in coroutines and callbacks. - Consider also using the :func:`asyncio.run` function instead of using - lower level functions to manually create and close an event loop. + As noted above, consider using the higher-level :func:`asyncio.run` function, + instead of using these lower level functions to manually create and close an + event loop. .. deprecated:: 3.10 - Deprecation warning is emitted if there is no running event loop. - In future Python releases, this function will be an alias of - :func:`get_running_loop`. + Emits a deprecation warning if there is no running event loop. + In future Python releases, this function may become an alias of + :func:`get_running_loop` and will accordingly raise a + :exc:`RuntimeError` if there is no running event loop. .. function:: set_event_loop(loop) - Set *loop* as a current event loop for the current OS thread. + Set *loop* as the current event loop for the current OS thread. .. function:: new_event_loop() From webhook-mailer at python.org Sat Dec 3 19:59:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 04 Dec 2022 00:59:11 -0000 Subject: [Python-checkins] gh-85747: "Preface" section of asyncio-eventloop.rst: Switch to active voice and suggest other edits (GH-99784) Message-ID: https://github.com/python/cpython/commit/27218d07996967cff1b997f83b7ccc58bd4940b9 commit: 27218d07996967cff1b997f83b7ccc58bd4940b9 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-03T16:59:06-08:00 summary: gh-85747: "Preface" section of asyncio-eventloop.rst: Switch to active voice and suggest other edits (GH-99784) (cherry picked from commit bf26bdf6ac04878fc720e78422991aaedb9808a1) Co-authored-by: Brian Skinn files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index cf9b3b7ce079..a10658df5df8 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -33,7 +33,8 @@ an event loop: Return the running event loop in the current OS thread. - If there is no running event loop a :exc:`RuntimeError` is raised. + Raise a :exc:`RuntimeError` if there is no running event loop. + This function can only be called from a coroutine or a callback. .. versionadded:: 3.7 @@ -52,17 +53,19 @@ an event loop: :func:`get_running_loop` function is preferred to :func:`get_event_loop` in coroutines and callbacks. - Consider also using the :func:`asyncio.run` function instead of using - lower level functions to manually create and close an event loop. + As noted above, consider using the higher-level :func:`asyncio.run` function, + instead of using these lower level functions to manually create and close an + event loop. .. deprecated:: 3.10 - Deprecation warning is emitted if there is no running event loop. - In future Python releases, this function will be an alias of - :func:`get_running_loop`. + Emits a deprecation warning if there is no running event loop. + In future Python releases, this function may become an alias of + :func:`get_running_loop` and will accordingly raise a + :exc:`RuntimeError` if there is no running event loop. .. function:: set_event_loop(loop) - Set *loop* as a current event loop for the current OS thread. + Set *loop* as the current event loop for the current OS thread. .. function:: new_event_loop() From webhook-mailer at python.org Sat Dec 3 19:59:53 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 04 Dec 2022 00:59:53 -0000 Subject: [Python-checkins] gh-85747: "Preface" section of asyncio-eventloop.rst: Switch to active voice and suggest other edits (GH-99784) Message-ID: https://github.com/python/cpython/commit/e2209cb6ecb094ecbf7d999fd2b59f64326c1559 commit: e2209cb6ecb094ecbf7d999fd2b59f64326c1559 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-03T16:59:47-08:00 summary: gh-85747: "Preface" section of asyncio-eventloop.rst: Switch to active voice and suggest other edits (GH-99784) (cherry picked from commit bf26bdf6ac04878fc720e78422991aaedb9808a1) Co-authored-by: Brian Skinn files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 4ad8dc08186d..86ca5d0f2ab5 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -33,7 +33,8 @@ an event loop: Return the running event loop in the current OS thread. - If there is no running event loop a :exc:`RuntimeError` is raised. + Raise a :exc:`RuntimeError` if there is no running event loop. + This function can only be called from a coroutine or a callback. .. versionadded:: 3.7 @@ -52,17 +53,19 @@ an event loop: :func:`get_running_loop` function is preferred to :func:`get_event_loop` in coroutines and callbacks. - Consider also using the :func:`asyncio.run` function instead of using - lower level functions to manually create and close an event loop. + As noted above, consider using the higher-level :func:`asyncio.run` function, + instead of using these lower level functions to manually create and close an + event loop. .. deprecated:: 3.10 - Deprecation warning is emitted if there is no running event loop. - In future Python releases, this function will be an alias of - :func:`get_running_loop`. + Emits a deprecation warning if there is no running event loop. + In future Python releases, this function may become an alias of + :func:`get_running_loop` and will accordingly raise a + :exc:`RuntimeError` if there is no running event loop. .. function:: set_event_loop(loop) - Set *loop* as a current event loop for the current OS thread. + Set *loop* as the current event loop for the current OS thread. .. function:: new_event_loop() From webhook-mailer at python.org Sun Dec 4 01:42:00 2022 From: webhook-mailer at python.org (Fidget-Spinner) Date: Sun, 04 Dec 2022 06:42:00 -0000 Subject: [Python-checkins] [3.11] gh-99886: Fix crash when freeing objects with managed dictionaries (#99902) Message-ID: https://github.com/python/cpython/commit/9e38553132bf7c6fc13e9f268a54ac6533e6ad41 commit: 9e38553132bf7c6fc13e9f268a54ac6533e6ad41 branch: 3.11 author: Ken Jin committer: Fidget-Spinner date: 2022-12-04T14:41:23+08:00 summary: [3.11] gh-99886: Fix crash when freeing objects with managed dictionaries (#99902) Co-authored-by: Erlend E. Aasland files: A Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst M Lib/test/test_sqlite3/test_regression.py M Objects/dictobject.c diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index 0b727cecb0e8..9a07e02ae119 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -469,6 +469,18 @@ def test_executescript_step_through_select(self): con.executescript("select step(t) from t") self.assertEqual(steps, values) + def test_custom_cursor_object_crash_gh_99886(self): + # This test segfaults on GH-99886 + class MyCursor(sqlite.Cursor): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # this can go before or after the super call; doesn't matter + self.some_attr = None + + with memory_database() as con: + cur = con.cursor(MyCursor) + cur.close() + del cur class RecursiveUseOfCursors(unittest.TestCase): # GH-80254: sqlite3 should not segfault for recursive use of cursors. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst new file mode 100644 index 000000000000..8bdaa9462750 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst @@ -0,0 +1 @@ +Fix a crash when an object which does not have a dictionary frees its instance values. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ebbd22ee7c14..4a214f8cf5b7 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5573,14 +5573,16 @@ _PyObject_FreeInstanceAttributes(PyObject *self) PyTypeObject *tp = Py_TYPE(self); assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictValues **values_ptr = _PyObject_ValuesPointer(self); - if (*values_ptr == NULL) { + PyDictValues *values = *values_ptr; + if (values == NULL) { return; } + *values_ptr = NULL; PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF((*values_ptr)->values[i]); + Py_XDECREF(values->values[i]); } - free_values(*values_ptr); + free_values(values); } PyObject * From webhook-mailer at python.org Sun Dec 4 07:29:01 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sun, 04 Dec 2022 12:29:01 -0000 Subject: [Python-checkins] gh-60203: Always pass True/False as boolean arguments in tests (GH-99983) Message-ID: https://github.com/python/cpython/commit/76f43fc09af29401cc0cec7710b03e4dbf8a4578 commit: 76f43fc09af29401cc0cec7710b03e4dbf8a4578 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2022-12-04T14:28:56+02:00 summary: gh-60203: Always pass True/False as boolean arguments in tests (GH-99983) Unless we explicitly test non-bool values. files: M Lib/test/_test_multiprocessing.py M Lib/test/test_asyncio/test_ssl.py M Lib/test/test_call.py M Lib/test/test_capi/test_misc.py M Lib/test/test_subprocess.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a66f4f5b897c..2fa75eb4d113 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6042,5 +6042,5 @@ def test_semlock_subclass(self): class SemLock(_multiprocessing.SemLock): pass name = f'test_semlock_subclass-{os.getpid()}' - s = SemLock(1, 0, 10, name, 0) + s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 5de9b7a14e87..aaf3c37101f5 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1689,7 +1689,7 @@ def stop(self): def run(self): try: with self._sock: - self._sock.setblocking(0) + self._sock.setblocking(False) self._run() finally: self._s1.close() diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index f148b5ebbc5a..c17528be97b4 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -559,7 +559,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 1d30adaee921..06a51aa3cc21 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -138,8 +138,9 @@ def test_seq_bytes_to_charp_array(self): class Z(object): def __len__(self): return 1 - self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + with self.assertRaisesRegex(TypeError, 'indexing'): + _posixsubprocess.fork_exec( + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -147,7 +148,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -157,7 +158,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 424a4a93b6f9..8713c73f87a0 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3209,7 +3209,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, "no vfork") + None, True) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: From webhook-mailer at python.org Sun Dec 4 07:38:27 2022 From: webhook-mailer at python.org (pablogsal) Date: Sun, 04 Dec 2022 12:38:27 -0000 Subject: [Python-checkins] GH-91054: Reset static events counts in code watchers tests (#99978) Message-ID: https://github.com/python/cpython/commit/1012dc1b4367e05b92d67ea6925a39d50dce31b7 commit: 1012dc1b4367e05b92d67ea6925a39d50dce31b7 branch: main author: Itamar Ostricher committer: pablogsal date: 2022-12-04T12:38:21Z summary: GH-91054: Reset static events counts in code watchers tests (#99978) files: M Lib/test/test_capi/test_watchers.py M Modules/_testcapi/watchers.c diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index ebe7d2783189..1922614ef605 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -383,11 +383,11 @@ def test_code_object_events_dispatched(self): del co3 self.assert_event_counts(2, 2, 1, 1) - # verify counts remain as they were after both watchers are cleared + # verify counts are reset and don't change after both watchers are cleared co4 = _testcapi.code_newempty("test_watchers", "dummy4", 0) - self.assert_event_counts(2, 2, 1, 1) + self.assert_event_counts(0, 0, 0, 0) del co4 - self.assert_event_counts(2, 2, 1, 1) + self.assert_event_counts(0, 0, 0, 0) def test_clear_out_of_range_watcher_id(self): with self.assertRaisesRegex(ValueError, r"Invalid code watcher ID -1"): diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index f0e51fd462e7..1d91c206f630 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -325,9 +325,13 @@ add_code_watcher(PyObject *self, PyObject *which_watcher) long which_l = PyLong_AsLong(which_watcher); if (which_l == 0) { watcher_id = PyCode_AddWatcher(first_code_object_callback); + num_code_object_created_events[0] = 0; + num_code_object_destroyed_events[0] = 0; } else if (which_l == 1) { watcher_id = PyCode_AddWatcher(second_code_object_callback); + num_code_object_created_events[1] = 0; + num_code_object_destroyed_events[1] = 0; } else { return NULL; @@ -346,6 +350,11 @@ clear_code_watcher(PyObject *self, PyObject *watcher_id) if (PyCode_ClearWatcher(watcher_id_l) < 0) { return NULL; } + // reset static events counters + if (watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS) { + num_code_object_created_events[watcher_id_l] = 0; + num_code_object_destroyed_events[watcher_id_l] = 0; + } Py_RETURN_NONE; } From webhook-mailer at python.org Sun Dec 4 07:50:52 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 04 Dec 2022 12:50:52 -0000 Subject: [Python-checkins] gh-60203: Always pass True/False as boolean arguments in tests (GH-99983) Message-ID: https://github.com/python/cpython/commit/7aa87bba056c9c548812a82cefbd122c67c71b88 commit: 7aa87bba056c9c548812a82cefbd122c67c71b88 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-04T04:50:46-08:00 summary: gh-60203: Always pass True/False as boolean arguments in tests (GH-99983) Unless we explicitly test non-bool values. (cherry picked from commit 76f43fc09af29401cc0cec7710b03e4dbf8a4578) Co-authored-by: Serhiy Storchaka files: M Lib/test/_test_multiprocessing.py M Lib/test/test_asyncio/test_ssl.py M Lib/test/test_call.py M Lib/test/test_capi/test_misc.py M Lib/test/test_subprocess.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 3117b110db09..b50a1543205c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6044,5 +6044,5 @@ def test_semlock_subclass(self): class SemLock(_multiprocessing.SemLock): pass name = f'test_semlock_subclass-{os.getpid()}' - s = SemLock(1, 0, 10, name, 0) + s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 5de9b7a14e87..aaf3c37101f5 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1689,7 +1689,7 @@ def stop(self): def run(self): try: with self._sock: - self._sock.setblocking(0) + self._sock.setblocking(False) self._run() finally: self._s1.close() diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 4c971bc5ed05..0974002a93b6 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -558,7 +558,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index b30b8d18d8aa..6cda91677054 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -139,8 +139,9 @@ def test_seq_bytes_to_charp_array(self): class Z(object): def __len__(self): return 1 - self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + with self.assertRaisesRegex(TypeError, 'indexing'): + _posixsubprocess.fork_exec( + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -148,7 +149,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -158,7 +159,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 424a4a93b6f9..8713c73f87a0 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3209,7 +3209,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, "no vfork") + None, True) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: From webhook-mailer at python.org Sun Dec 4 08:08:31 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sun, 04 Dec 2022 13:08:31 -0000 Subject: [Python-checkins] [3.10] gh-60203: Always pass True/False as boolean arguments in tests (GH-99983) (GH-99989) Message-ID: https://github.com/python/cpython/commit/ac781282712ab39cd6339e3c780c11e0027d22ab commit: ac781282712ab39cd6339e3c780c11e0027d22ab branch: 3.10 author: Serhiy Storchaka committer: serhiy-storchaka date: 2022-12-04T15:08:24+02:00 summary: [3.10] gh-60203: Always pass True/False as boolean arguments in tests (GH-99983) (GH-99989) Unless we explicitly test non-bool values. (cherry picked from commit 76f43fc09af29401cc0cec7710b03e4dbf8a4578) files: M Lib/test/_test_multiprocessing.py M Lib/test/test_call.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index be174aae3d63..57eada634c44 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5982,5 +5982,5 @@ def test_semlock_subclass(self): class SemLock(_multiprocessing.SemLock): pass name = f'test_semlock_subclass-{os.getpid()}' - s = SemLock(1, 0, 10, name, 0) + s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index eee269093b27..1bf1f79fdad9 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -546,7 +546,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. From webhook-mailer at python.org Sun Dec 4 14:38:28 2022 From: webhook-mailer at python.org (gpshead) Date: Sun, 04 Dec 2022 19:38:28 -0000 Subject: [Python-checkins] gh-98458: unittest: bugfix for infinite loop while handling chained exceptions that contain cycles (#98459) Message-ID: https://github.com/python/cpython/commit/72ec518203c3f3577a5e888b12f10bb49060e6c2 commit: 72ec518203c3f3577a5e888b12f10bb49060e6c2 branch: main author: AlexTate <0xalextate at gmail.com> committer: gpshead date: 2022-12-04T11:37:55-08:00 summary: gh-98458: unittest: bugfix for infinite loop while handling chained exceptions that contain cycles (#98459) * Bugfix addressing infinite loop while handling self-referencing chained exception in TestResult._clean_tracebacks() * Bugfix extended to properly handle exception cycles in _clean_tracebacks. The "seen" set follows the approach used in the TracebackException class (thank you @iritkatriel for pointing it out) * adds a test for a single chained exception that holds a self-loop in its __cause__ and __context__ attributes files: A Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst M Lib/test/test_unittest/test_result.py M Lib/unittest/result.py diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index e71d114751d9..efd9c9023505 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -275,6 +275,62 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + def test_addFailure_filter_traceback_frames_chained_exception_self_loop(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + loop = Exception("Loop") + loop.__cause__ = loop + loop.__context__ = loop + raise loop + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: Loop\n"), 1) + + def test_addFailure_filter_traceback_frames_chained_exception_cycle(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + # Create two directionally opposed cycles + # __cause__ in one direction, __context__ in the other + A, B, C = Exception("A"), Exception("B"), Exception("C") + edges = [(C, B), (B, A), (A, C)] + for ex1, ex2 in edges: + ex1.__cause__ = ex2 + ex2.__context__ = ex1 + raise C + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: A\n"), 1) + self.assertEqual(formatted_exc.count("Exception: B\n"), 1) + self.assertEqual(formatted_exc.count("Exception: C\n"), 1) + # "addError(test, err)" # ... # "Called when the test case test raises an unexpected exception err diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3da7005e603f..5ca4c23238b4 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -196,6 +196,7 @@ def _clean_tracebacks(self, exctype, value, tb, test): ret = None first = True excs = [(exctype, value, tb)] + seen = {id(value)} # Detect loops in chained exceptions. while excs: (exctype, value, tb) = excs.pop() # Skip test runner traceback levels @@ -214,8 +215,9 @@ def _clean_tracebacks(self, exctype, value, tb, test): if value is not None: for c in (value.__cause__, value.__context__): - if c is not None: + if c is not None and id(c) not in seen: excs.append((type(c), c, c.__traceback__)) + seen.add(id(c)) return ret def _is_relevant_tb_level(self, tb): diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst new file mode 100644 index 000000000000..f74195cc8e7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst @@ -0,0 +1 @@ +Fix infinite loop in unittest when a self-referencing chained exception is raised From webhook-mailer at python.org Sun Dec 4 14:49:36 2022 From: webhook-mailer at python.org (ethanfurman) Date: Sun, 04 Dec 2022 19:49:36 -0000 Subject: [Python-checkins] [Enum] Fix typos in the documentation (GH-99960) Message-ID: https://github.com/python/cpython/commit/2ae894b6d1995a3b9f95f4a82eec6dedd3ba5298 commit: 2ae894b6d1995a3b9f95f4a82eec6dedd3ba5298 branch: main author: G?ry Ogam committer: ethanfurman date: 2022-12-04T11:49:31-08:00 summary: [Enum] Fix typos in the documentation (GH-99960) files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 74d9e6732762..e29f5837f0ab 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -194,7 +194,7 @@ Data Types .. method:: EnumType.__getitem__(cls, name) - Returns the Enum member in *cls* matching *name*, or raises an :exc:`KeyError`:: + Returns the Enum member in *cls* matching *name*, or raises a :exc:`KeyError`:: >>> Color['BLUE'] @@ -241,7 +241,7 @@ Data Types .. note:: Enum member values - Member values can be anything: :class:`int`, :class:`str`, etc.. If + Member values can be anything: :class:`int`, :class:`str`, etc. If the exact value is unimportant you may use :class:`auto` instances and an appropriate value will be chosen for you. See :class:`auto` for the details. @@ -255,7 +255,7 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. - .. method:: Enum.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: Enum.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -272,8 +272,8 @@ Data Types :module: The name of the module the new Enum is created in. :qualname: The actual location in the module where this Enum can be found. :type: A mix-in type for the new Enum. - :start: The first integer value for the Enum (used by :class:`auto`) - :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only) + :start: The first integer value for the Enum (used by :class:`auto`). + :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only). .. method:: Enum.__dir__(self) @@ -315,7 +315,7 @@ Data Types >>> PowersOfThree.SECOND.value 6 - .. method:: Enum.__init_subclass__(cls, \**kwds) + .. method:: Enum.__init_subclass__(cls, **kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -373,7 +373,7 @@ Data Types .. method:: Enum.__format__(self) Returns the string used for *format()* and *f-string* calls. By default, - returns :meth:`__str__` returns, but can be overridden:: + returns :meth:`__str__` return value, but can be overridden:: >>> class OtherStyle(Enum): ... ALTERNATE = auto() @@ -552,11 +552,11 @@ Data Types Using :class:`auto` with :class:`Flag` results in integers that are powers of two, starting with ``1``. - .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It + .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It is now:: - >>> Color(0) # doctest: +SKIP - + >>> Color(0) # doctest: +SKIP + .. class:: IntFlag @@ -600,7 +600,7 @@ Data Types *replacement of existing constants* use-case. :meth:`~object.__format__` was already :meth:`!int.__format__` for that same reason. - Inversion of a :class:`!IntFlag` now returns a positive value that is the + Inversion of an :class:`!IntFlag` now returns a positive value that is the union of all flags not in the given flag, rather than a negative value. This matches the existing :class:`Flag` behavior. @@ -612,7 +612,7 @@ Data Types * :meth:`!int.__str__` for :class:`IntEnum` and :class:`IntFlag` * :meth:`!str.__str__` for :class:`StrEnum` - Inherit from :class:`!ReprEnum` to keep the :class:`str() / :func:`format` + Inherit from :class:`!ReprEnum` to keep the :class:`str() ` / :func:`format` of the mixed-in data type instead of using the :class:`Enum`-default :meth:`str() `. @@ -658,7 +658,7 @@ Data Types .. attribute:: NAMED_FLAGS Ensure that any flag groups/masks contain only named flags -- useful when - values are specified instead of being generated by :func:`auto` + values are specified instead of being generated by :func:`auto`:: >>> from enum import Flag, verify, NAMED_FLAGS >>> @verify(NAMED_FLAGS) @@ -885,23 +885,23 @@ Notes :class:`IntEnum`, :class:`StrEnum`, and :class:`IntFlag` - These three enum types are designed to be drop-in replacements for existing - integer- and string-based values; as such, they have extra limitations: + These three enum types are designed to be drop-in replacements for existing + integer- and string-based values; as such, they have extra limitations: - - ``__str__`` uses the value and not the name of the enum member + - ``__str__`` uses the value and not the name of the enum member - - ``__format__``, because it uses ``__str__``, will also use the value of - the enum member instead of its name + - ``__format__``, because it uses ``__str__``, will also use the value of + the enum member instead of its name - If you do not need/want those limitations, you can either create your own - base class by mixing in the ``int`` or ``str`` type yourself:: + If you do not need/want those limitations, you can either create your own + base class by mixing in the ``int`` or ``str`` type yourself:: - >>> from enum import Enum - >>> class MyIntEnum(int, Enum): - ... pass + >>> from enum import Enum + >>> class MyIntEnum(int, Enum): + ... pass or you can reassign the appropriate :meth:`str`, etc., in your enum:: - >>> from enum import IntEnum - >>> class MyIntEnum(IntEnum): - ... __str__ = IntEnum.__str__ + >>> from enum import IntEnum + >>> class MyIntEnum(IntEnum): + ... __str__ = IntEnum.__str__ From webhook-mailer at python.org Sun Dec 4 14:58:37 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 04 Dec 2022 19:58:37 -0000 Subject: [Python-checkins] gh-98458: unittest: bugfix for infinite loop while handling chained exceptions that contain cycles (GH-98459) Message-ID: https://github.com/python/cpython/commit/e699e5c20fc495952905597edfa82de0c1848f8c commit: e699e5c20fc495952905597edfa82de0c1848f8c branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-04T11:58:31-08:00 summary: gh-98458: unittest: bugfix for infinite loop while handling chained exceptions that contain cycles (GH-98459) * Bugfix addressing infinite loop while handling self-referencing chained exception in TestResult._clean_tracebacks() * Bugfix extended to properly handle exception cycles in _clean_tracebacks. The "seen" set follows the approach used in the TracebackException class (thank you @iritkatriel for pointing it out) * adds a test for a single chained exception that holds a self-loop in its __cause__ and __context__ attributes (cherry picked from commit 72ec518203c3f3577a5e888b12f10bb49060e6c2) Co-authored-by: AlexTate <0xalextate at gmail.com> files: A Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst M Lib/unittest/result.py M Lib/unittest/test/test_result.py diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3da7005e603f..5ca4c23238b4 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -196,6 +196,7 @@ def _clean_tracebacks(self, exctype, value, tb, test): ret = None first = True excs = [(exctype, value, tb)] + seen = {id(value)} # Detect loops in chained exceptions. while excs: (exctype, value, tb) = excs.pop() # Skip test runner traceback levels @@ -214,8 +215,9 @@ def _clean_tracebacks(self, exctype, value, tb, test): if value is not None: for c in (value.__cause__, value.__context__): - if c is not None: + if c is not None and id(c) not in seen: excs.append((type(c), c, c.__traceback__)) + seen.add(id(c)) return ret def _is_relevant_tb_level(self, tb): diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py index c5aaba0ff525..90484fd182f3 100644 --- a/Lib/unittest/test/test_result.py +++ b/Lib/unittest/test/test_result.py @@ -275,6 +275,62 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + def test_addFailure_filter_traceback_frames_chained_exception_self_loop(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + loop = Exception("Loop") + loop.__cause__ = loop + loop.__context__ = loop + raise loop + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: Loop\n"), 1) + + def test_addFailure_filter_traceback_frames_chained_exception_cycle(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + # Create two directionally opposed cycles + # __cause__ in one direction, __context__ in the other + A, B, C = Exception("A"), Exception("B"), Exception("C") + edges = [(C, B), (B, A), (A, C)] + for ex1, ex2 in edges: + ex1.__cause__ = ex2 + ex2.__context__ = ex1 + raise C + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: A\n"), 1) + self.assertEqual(formatted_exc.count("Exception: B\n"), 1) + self.assertEqual(formatted_exc.count("Exception: C\n"), 1) + # "addError(test, err)" # ... # "Called when the test case test raises an unexpected exception err diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst new file mode 100644 index 000000000000..f74195cc8e7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst @@ -0,0 +1 @@ +Fix infinite loop in unittest when a self-referencing chained exception is raised From webhook-mailer at python.org Sun Dec 4 15:06:47 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 04 Dec 2022 20:06:47 -0000 Subject: [Python-checkins] gh-98458: unittest: bugfix for infinite loop while handling chained exceptions that contain cycles (GH-98459) Message-ID: https://github.com/python/cpython/commit/9bcc68b045fd06af3896d15d6375fa4d96b706cc commit: 9bcc68b045fd06af3896d15d6375fa4d96b706cc branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-04T12:06:42-08:00 summary: gh-98458: unittest: bugfix for infinite loop while handling chained exceptions that contain cycles (GH-98459) * Bugfix addressing infinite loop while handling self-referencing chained exception in TestResult._clean_tracebacks() * Bugfix extended to properly handle exception cycles in _clean_tracebacks. The "seen" set follows the approach used in the TracebackException class (thank you @iritkatriel for pointing it out) * adds a test for a single chained exception that holds a self-loop in its __cause__ and __context__ attributes (cherry picked from commit 72ec518203c3f3577a5e888b12f10bb49060e6c2) Co-authored-by: AlexTate <0xalextate at gmail.com> files: A Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst M Lib/unittest/result.py M Lib/unittest/test/test_result.py diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3da7005e603f..5ca4c23238b4 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -196,6 +196,7 @@ def _clean_tracebacks(self, exctype, value, tb, test): ret = None first = True excs = [(exctype, value, tb)] + seen = {id(value)} # Detect loops in chained exceptions. while excs: (exctype, value, tb) = excs.pop() # Skip test runner traceback levels @@ -214,8 +215,9 @@ def _clean_tracebacks(self, exctype, value, tb, test): if value is not None: for c in (value.__cause__, value.__context__): - if c is not None: + if c is not None and id(c) not in seen: excs.append((type(c), c, c.__traceback__)) + seen.add(id(c)) return ret def _is_relevant_tb_level(self, tb): diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py index b0cc3d867d6e..9320b0a44b51 100644 --- a/Lib/unittest/test/test_result.py +++ b/Lib/unittest/test/test_result.py @@ -275,6 +275,62 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + def test_addFailure_filter_traceback_frames_chained_exception_self_loop(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + loop = Exception("Loop") + loop.__cause__ = loop + loop.__context__ = loop + raise loop + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: Loop\n"), 1) + + def test_addFailure_filter_traceback_frames_chained_exception_cycle(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + # Create two directionally opposed cycles + # __cause__ in one direction, __context__ in the other + A, B, C = Exception("A"), Exception("B"), Exception("C") + edges = [(C, B), (B, A), (A, C)] + for ex1, ex2 in edges: + ex1.__cause__ = ex2 + ex2.__context__ = ex1 + raise C + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: A\n"), 1) + self.assertEqual(formatted_exc.count("Exception: B\n"), 1) + self.assertEqual(formatted_exc.count("Exception: C\n"), 1) + # "addError(test, err)" # ... # "Called when the test case test raises an unexpected exception err diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst new file mode 100644 index 000000000000..f74195cc8e7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst @@ -0,0 +1 @@ +Fix infinite loop in unittest when a self-referencing chained exception is raised From webhook-mailer at python.org Sun Dec 4 15:24:24 2022 From: webhook-mailer at python.org (mdickinson) Date: Sun, 04 Dec 2022 20:24:24 -0000 Subject: [Python-checkins] gh-98248: Normalizing the error messages in function struct.pack (GH-98252) Message-ID: https://github.com/python/cpython/commit/854a878e4f09cd961ba5135567f7a5b5f86d7be9 commit: 854a878e4f09cd961ba5135567f7a5b5f86d7be9 branch: main author: Felix Ye <29754475+yanjs at users.noreply.github.com> committer: mdickinson date: 2022-12-04T20:24:18Z summary: gh-98248: Normalizing the error messages in function struct.pack (GH-98252) Provide consistent and informative error messages in function struct.pack when its integral arguments are not in range. files: A Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst M Lib/test/test_struct.py M Modules/_struct.c diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index b0f11af1a789..6b1f22f66fd1 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -723,23 +723,56 @@ def test_issue35714(self): struct.calcsize(s) @support.cpython_only - def test_issue45034_unsigned(self): - _testcapi = import_helper.import_module('_testcapi') - error_msg = f'ushort format requires 0 <= number <= {_testcapi.USHRT_MAX}' - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('H', 70000) # too large - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('H', -1) # too small + def test_issue98248(self): + def test_error_msg(prefix, int_type, is_unsigned): + fmt_str = prefix + int_type + size = struct.calcsize(fmt_str) + if is_unsigned: + max_ = 2 ** (size * 8) - 1 + min_ = 0 + else: + max_ = 2 ** (size * 8 - 1) - 1 + min_ = -2 ** (size * 8 - 1) + error_msg = f"'{int_type}' format requires {min_} <= number <= {max_}" + for number in [int(-1e50), min_ - 1, max_ + 1, int(1e50)]: + with self.subTest(format_str=fmt_str, number=number): + with self.assertRaisesRegex(struct.error, error_msg): + struct.pack(fmt_str, number) + error_msg = "required argument is not an integer" + not_number = "" + with self.subTest(format_str=fmt_str, number=not_number): + with self.assertRaisesRegex(struct.error, error_msg): + struct.pack(fmt_str, not_number) + + for prefix in '@=<>': + for int_type in 'BHILQ': + test_error_msg(prefix, int_type, True) + for int_type in 'bhilq': + test_error_msg(prefix, int_type, False) + + int_type = 'N' + test_error_msg('@', int_type, True) + + int_type = 'n' + test_error_msg('@', int_type, False) @support.cpython_only - def test_issue45034_signed(self): - _testcapi = import_helper.import_module('_testcapi') - error_msg = f'short format requires {_testcapi.SHRT_MIN} <= number <= {_testcapi.SHRT_MAX}' - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('h', 70000) # too large - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('h', -70000) # too small - + def test_issue98248_error_propagation(self): + class Div0: + def __index__(self): + 1 / 0 + + def test_error_propagation(fmt_str): + with self.subTest(format_str=fmt_str, exception="ZeroDivisionError"): + with self.assertRaises(ZeroDivisionError): + struct.pack(fmt_str, Div0()) + + for prefix in '@=<>': + for int_type in 'BHILQbhilq': + test_error_propagation(prefix + int_type) + + test_error_propagation('N') + test_error_propagation('n') class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst b/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst new file mode 100644 index 000000000000..347f6e160335 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst @@ -0,0 +1 @@ +Provide informative error messages in :func:`struct.pack` when its integral arguments are not in range. diff --git a/Modules/_struct.c b/Modules/_struct.c index 0cf34fbf9a3a..3db7b991acd0 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -167,9 +167,6 @@ get_long(_structmodulestate *state, PyObject *v, long *p) x = PyLong_AsLong(v); Py_DECREF(v); if (x == (long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -191,9 +188,6 @@ get_ulong(_structmodulestate *state, PyObject *v, unsigned long *p) x = PyLong_AsUnsignedLong(v); Py_DECREF(v); if (x == (unsigned long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -214,9 +208,6 @@ get_longlong(_structmodulestate *state, PyObject *v, long long *p) x = PyLong_AsLongLong(v); Py_DECREF(v); if (x == (long long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -237,9 +228,6 @@ get_ulonglong(_structmodulestate *state, PyObject *v, unsigned long long *p) x = PyLong_AsUnsignedLongLong(v); Py_DECREF(v); if (x == (unsigned long long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -260,9 +248,6 @@ get_ssize_t(_structmodulestate *state, PyObject *v, Py_ssize_t *p) x = PyLong_AsSsize_t(v); Py_DECREF(v); if (x == (Py_ssize_t)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -283,9 +268,6 @@ get_size_t(_structmodulestate *state, PyObject *v, size_t *p) x = PyLong_AsSize_t(v); Py_DECREF(v); if (x == (size_t)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -293,7 +275,7 @@ get_size_t(_structmodulestate *state, PyObject *v, size_t *p) } -#define RANGE_ERROR(state, x, f, flag, mask) return _range_error(state, f, flag) +#define RANGE_ERROR(state, f, flag) return _range_error(state, f, flag) /* Floating point helpers */ @@ -545,12 +527,14 @@ static int np_byte(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } if (x < -128 || x > 127) { - PyErr_SetString(state->StructError, - "byte format requires -128 <= number <= 127"); - return -1; + RANGE_ERROR(state, f, 0); } *p = (char)x; return 0; @@ -560,12 +544,14 @@ static int np_ubyte(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } if (x < 0 || x > 255) { - PyErr_SetString(state->StructError, - "ubyte format requires 0 <= number <= 255"); - return -1; + RANGE_ERROR(state, f, 1); } *(unsigned char *)p = (unsigned char)x; return 0; @@ -588,13 +574,14 @@ np_short(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; short y; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } if (x < SHRT_MIN || x > SHRT_MAX) { - PyErr_Format(state->StructError, - "short format requires %d <= number <= %d", - (int)SHRT_MIN, (int)SHRT_MAX); - return -1; + RANGE_ERROR(state, f, 0); } y = (short)x; memcpy(p, (char *)&y, sizeof y); @@ -606,13 +593,14 @@ np_ushort(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; unsigned short y; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } if (x < 0 || x > USHRT_MAX) { - PyErr_Format(state->StructError, - "ushort format requires 0 <= number <= %u", - (unsigned int)USHRT_MAX); - return -1; + RANGE_ERROR(state, f, 1); } y = (unsigned short)x; memcpy(p, (char *)&y, sizeof y); @@ -624,11 +612,15 @@ np_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; int y; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } #if (SIZEOF_LONG > SIZEOF_INT) if ((x < ((long)INT_MIN)) || (x > ((long)INT_MAX))) - RANGE_ERROR(state, x, f, 0, -1); + RANGE_ERROR(state, f, 0); #endif y = (int)x; memcpy(p, (char *)&y, sizeof y); @@ -640,12 +632,16 @@ np_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { unsigned long x; unsigned int y; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } y = (unsigned int)x; #if (SIZEOF_LONG > SIZEOF_INT) if (x > ((unsigned long)UINT_MAX)) - RANGE_ERROR(state, y, f, 1, -1); + RANGE_ERROR(state, f, 1); #endif memcpy(p, (char *)&y, sizeof y); return 0; @@ -655,8 +651,12 @@ static int np_long(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -665,8 +665,12 @@ static int np_ulong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { unsigned long x; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -675,8 +679,12 @@ static int np_ssize_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { Py_ssize_t x; - if (get_ssize_t(state, v, &x) < 0) + if (get_ssize_t(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -685,8 +693,12 @@ static int np_size_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { size_t x; - if (get_size_t(state, v, &x) < 0) + if (get_size_t(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -695,8 +707,16 @@ static int np_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long long x; - if (get_longlong(state, v, &x) < 0) + if (get_longlong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(state->StructError, + "'%c' format requires %lld <= number <= %lld", + f->format, + LLONG_MIN, + LLONG_MAX); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -705,8 +725,15 @@ static int np_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { unsigned long long x; - if (get_ulonglong(state, v, &x) < 0) + if (get_ulonglong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(state->StructError, + "'%c' format requires 0 <= number <= %llu", + f->format, + ULLONG_MAX); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -911,15 +938,19 @@ bp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { if ((i == 2) && (x < -32768 || x > 32767)) - RANGE_ERROR(state, x, f, 0, 0xffffL); + RANGE_ERROR(state, f, 0); #if (SIZEOF_LONG != 4) else if ((i == 4) && (x < -2147483648L || x > 2147483647L)) - RANGE_ERROR(state, x, f, 0, 0xffffffffL); + RANGE_ERROR(state, f, 0); #endif } do { @@ -935,14 +966,18 @@ bp_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) unsigned long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { unsigned long maxint = 1; maxint <<= (unsigned long)(i * 8); if (x >= maxint) - RANGE_ERROR(state, x, f, 1, maxint - 1); + RANGE_ERROR(state, f, 1); } do { q[--i] = (unsigned char)(x & 0xffUL); @@ -964,6 +999,14 @@ bp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) 0, /* little_endian */ 1 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires %lld <= number <= %lld", + f->format, + LLONG_MIN, + LLONG_MAX); + return -1; + } return res; } @@ -980,6 +1023,13 @@ bp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f 0, /* little_endian */ 0 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires 0 <= number <= %llu", + f->format, + ULLONG_MAX); + return -1; + } return res; } @@ -1148,15 +1198,19 @@ lp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { if ((i == 2) && (x < -32768 || x > 32767)) - RANGE_ERROR(state, x, f, 0, 0xffffL); + RANGE_ERROR(state, f, 0); #if (SIZEOF_LONG != 4) else if ((i == 4) && (x < -2147483648L || x > 2147483647L)) - RANGE_ERROR(state, x, f, 0, 0xffffffffL); + RANGE_ERROR(state, f, 0); #endif } do { @@ -1172,14 +1226,18 @@ lp_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) unsigned long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { unsigned long maxint = 1; maxint <<= (unsigned long)(i * 8); if (x >= maxint) - RANGE_ERROR(state, x, f, 1, maxint - 1); + RANGE_ERROR(state, f, 1); } do { *q++ = (unsigned char)(x & 0xffUL); @@ -1201,6 +1259,14 @@ lp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) 1, /* little_endian */ 1 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires %lld <= number <= %lld", + f->format, + LLONG_MIN, + LLONG_MAX); + return -1; + } return res; } @@ -1217,6 +1283,13 @@ lp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f 1, /* little_endian */ 0 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires 0 <= number <= %llu", + f->format, + ULLONG_MAX); + return -1; + } return res; } From webhook-mailer at python.org Mon Dec 5 00:42:44 2022 From: webhook-mailer at python.org (corona10) Date: Mon, 05 Dec 2022 05:42:44 -0000 Subject: [Python-checkins] no-issue: Fix typo in pycore_object.h (gh-99994) Message-ID: https://github.com/python/cpython/commit/bdc93b8a3563b4a3adb25fa902c0c879ccf427f6 commit: bdc93b8a3563b4a3adb25fa902c0c879ccf427f6 branch: main author: Ikko Ashimine committer: corona10 date: 2022-12-05T14:42:38+09:00 summary: no-issue: Fix typo in pycore_object.h (gh-99994) files: M Include/internal/pycore_object.h diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 33c8c0b75ea7..8796dfe2f6b8 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -373,7 +373,7 @@ PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); * match. * * Third party code unintentionally rely on problematic fpcasts. The call - * trampoline mitigates common occurences of bad fpcasts on Emscripten. + * trampoline mitigates common occurrences of bad fpcasts on Emscripten. */ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) #define _PyCFunction_TrampolineCall(meth, self, args) \ From webhook-mailer at python.org Mon Dec 5 05:35:37 2022 From: webhook-mailer at python.org (hugovk) Date: Mon, 05 Dec 2022 10:35:37 -0000 Subject: [Python-checkins] gh-93464: [Enum] Add versionchanged tag (#99997) Message-ID: https://github.com/python/cpython/commit/e3a3863cb9561705d3dd59a9367427ed45dfb5ea commit: e3a3863cb9561705d3dd59a9367427ed45dfb5ea branch: main author: Ethan Furman committer: hugovk date: 2022-12-05T12:35:31+02:00 summary: gh-93464: [Enum] Add versionchanged tag (#99997) Co-authored-by: C.A.M. Gerlach files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index e29f5837f0ab..208aecf11c80 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -804,6 +804,11 @@ Utilities and Decorators * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to create the ``THREE`` enum member) + .. versionchanged:: 3.11.1 + + In prior versions, ``auto()`` had to be the only thing + on the assignment line to work properly. + ``_generate_next_value_`` can be overridden to customize the values used by *auto*. From webhook-mailer at python.org Mon Dec 5 05:45:45 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 10:45:45 -0000 Subject: [Python-checkins] gh-93464: [Enum] Add versionchanged tag (GH-99997) Message-ID: https://github.com/python/cpython/commit/7f2bcc7aaa7b340f840065f94260884f86ba3165 commit: 7f2bcc7aaa7b340f840065f94260884f86ba3165 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T02:45:37-08:00 summary: gh-93464: [Enum] Add versionchanged tag (GH-99997) (cherry picked from commit e3a3863cb9561705d3dd59a9367427ed45dfb5ea) Co-authored-by: Ethan Furman Co-authored-by: C.A.M. Gerlach files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 8750be7470b5..99e9a193a60c 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -811,6 +811,11 @@ Utilities and Decorators * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to create the ``THREE`` enum member) + .. versionchanged:: 3.11.1 + + In prior versions, ``auto()`` had to be the only thing + on the assignment line to work properly. + ``_generate_next_value_`` can be overridden to customize the values used by *auto*. From webhook-mailer at python.org Mon Dec 5 07:39:05 2022 From: webhook-mailer at python.org (ambv) Date: Mon, 05 Dec 2022 12:39:05 -0000 Subject: [Python-checkins] [3.11] bpo-40882: Fix a memory leak in SharedMemory on Windows (GH-20684) (#99973) Message-ID: https://github.com/python/cpython/commit/374b0a2aceb378da6a3df9beffe35acd0c11c06d commit: 374b0a2aceb378da6a3df9beffe35acd0c11c06d branch: 3.11 author: Luke Garland committer: ambv date: 2022-12-05T13:38:25+01:00 summary: [3.11] bpo-40882: Fix a memory leak in SharedMemory on Windows (GH-20684) (#99973) bpo-40882: Fix a memory leak in SharedMemory on Windows (GH-20684) In multiprocessing.shared_memory.SharedMemory(), the temporary view returned by MapViewOfFile() should be unmapped when it is no longer needed. (cherry picked from commit 85c128e34daec7625b74746e127afa25888ccde1) Co-authored-by: Zackery Spytz files: A Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst M Lib/multiprocessing/shared_memory.py M Modules/_winapi.c M Modules/clinic/_winapi.c.h diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 881f2001dd59..9a1e5aa17b87 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -173,7 +173,10 @@ def __init__(self, name=None, create=False, size=0): ) finally: _winapi.CloseHandle(h_map) - size = _winapi.VirtualQuerySize(p_buf) + try: + size = _winapi.VirtualQuerySize(p_buf) + finally: + _winapi.UnmapViewOfFile(p_buf) self._mmap = mmap.mmap(-1, size, tagname=name) self._size = size diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst new file mode 100644 index 000000000000..2670aeef9a25 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst @@ -0,0 +1,2 @@ +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 9b30a9003261..f6bb07fd8b06 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1402,6 +1402,30 @@ _winapi_MapViewOfFile_impl(PyObject *module, HANDLE file_map, return address; } +/*[clinic input] +_winapi.UnmapViewOfFile + + address: LPCVOID + / +[clinic start generated code]*/ + +static PyObject * +_winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address) +/*[clinic end generated code: output=4f7e18ac75d19744 input=8c4b6119ad9288a3]*/ +{ + BOOL success; + + Py_BEGIN_ALLOW_THREADS + success = UnmapViewOfFile(address); + Py_END_ALLOW_THREADS + + if (!success) { + return PyErr_SetFromWindowsErr(0); + } + + Py_RETURN_NONE; +} + /*[clinic input] _winapi.OpenFileMapping -> HANDLE @@ -2095,6 +2119,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_READFILE_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF + _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF _WINAPI_WAITNAMEDPIPE_METHODDEF _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 118e7bfd29fe..5364d9a2d6ef 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -731,6 +731,32 @@ _winapi_MapViewOfFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_winapi_UnmapViewOfFile__doc__, +"UnmapViewOfFile($module, address, /)\n" +"--\n" +"\n"); + +#define _WINAPI_UNMAPVIEWOFFILE_METHODDEF \ + {"UnmapViewOfFile", (PyCFunction)_winapi_UnmapViewOfFile, METH_O, _winapi_UnmapViewOfFile__doc__}, + +static PyObject * +_winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address); + +static PyObject * +_winapi_UnmapViewOfFile(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + LPCVOID address; + + if (!PyArg_Parse(arg, "" F_POINTER ":UnmapViewOfFile", &address)) { + goto exit; + } + return_value = _winapi_UnmapViewOfFile_impl(module, address); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_OpenFileMapping__doc__, "OpenFileMapping($module, desired_access, inherit_handle, name, /)\n" "--\n" @@ -1216,4 +1242,4 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=60b036183b92659e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9c08a7371fcf5dd4 input=a9049054013a1b77]*/ From webhook-mailer at python.org Mon Dec 5 08:23:46 2022 From: webhook-mailer at python.org (vstinner) Date: Mon, 05 Dec 2022 13:23:46 -0000 Subject: [Python-checkins] gh-100005: Skip test_script_as_dev_fd() on FreeBSD (#100006) Message-ID: https://github.com/python/cpython/commit/038b151963d9d4a5f4c852544fb5b0402ffcb218 commit: 038b151963d9d4a5f4c852544fb5b0402ffcb218 branch: main author: Victor Stinner committer: vstinner date: 2022-12-05T14:23:35+01:00 summary: gh-100005: Skip test_script_as_dev_fd() on FreeBSD (#100006) On FreeBSD, skip test_script_as_dev_fd() of test_cmd_line_script if fdescfs is not mounted (at /dev/fd). files: M Lib/test/test_cmd_line_script.py M Lib/test/test_subprocess.py diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index c838e95ad554..f10d72ea5547 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -753,6 +753,9 @@ def test_nonexisting_script(self): self.assertNotEqual(proc.returncode, 0) @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform') + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_script_as_dev_fd(self): # GH-87235: On macOS passing a non-trivial script to /dev/fd/N can cause # problems because all open /dev/fd/N file descriptors share the same diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 8713c73f87a0..abd0dd8b2569 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2832,7 +2832,7 @@ def test_close_fds(self): @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, - "Requires fdescfs mounted on /dev/fd on FreeBSD.") + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_close_fds_when_max_fd_is_lowered(self): """Confirm that issue21618 is fixed (may fail under valgrind).""" fd_status = support.findfile("fd_status.py", subdir="subprocessdata") From webhook-mailer at python.org Mon Dec 5 09:07:54 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 14:07:54 -0000 Subject: [Python-checkins] gh-100005: Skip test_script_as_dev_fd() on FreeBSD (GH-100006) Message-ID: https://github.com/python/cpython/commit/5533cf67e7cdd873136a963e30a437bfcbbdffdf commit: 5533cf67e7cdd873136a963e30a437bfcbbdffdf branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T06:07:48-08:00 summary: gh-100005: Skip test_script_as_dev_fd() on FreeBSD (GH-100006) On FreeBSD, skip test_script_as_dev_fd() of test_cmd_line_script if fdescfs is not mounted (at /dev/fd). (cherry picked from commit 038b151963d9d4a5f4c852544fb5b0402ffcb218) Co-authored-by: Victor Stinner files: M Lib/test/test_cmd_line_script.py M Lib/test/test_subprocess.py diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 0c40dae0f441..4dadbc0b64bd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -741,6 +741,9 @@ def test_nonexisting_script(self): self.assertNotEqual(proc.returncode, 0) @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform') + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_script_as_dev_fd(self): # GH-87235: On macOS passing a non-trivial script to /dev/fd/N can cause # problems because all open /dev/fd/N file descriptors share the same diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 8713c73f87a0..abd0dd8b2569 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2832,7 +2832,7 @@ def test_close_fds(self): @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, - "Requires fdescfs mounted on /dev/fd on FreeBSD.") + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_close_fds_when_max_fd_is_lowered(self): """Confirm that issue21618 is fixed (may fail under valgrind).""" fd_status = support.findfile("fd_status.py", subdir="subprocessdata") From webhook-mailer at python.org Mon Dec 5 09:41:50 2022 From: webhook-mailer at python.org (vstinner) Date: Mon, 05 Dec 2022 14:41:50 -0000 Subject: [Python-checkins] gh-100008: Document Python build requirements (#100009) Message-ID: https://github.com/python/cpython/commit/5ea052bb0c8fa76867751046c89f69db5661ed4f commit: 5ea052bb0c8fa76867751046c89f69db5661ed4f branch: main author: Victor Stinner committer: vstinner date: 2022-12-05T15:41:44+01:00 summary: gh-100008: Document Python build requirements (#100009) Document also configure --without-freelists option added to Python 3.11. files: M Doc/using/configure.rst M Doc/whatsnew/3.11.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 0922972f9bf1..3df6ff4873b7 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -2,6 +2,46 @@ Configure Python **************** +Build Requirements +================== + +Features required to build CPython: + +* A `C11 `_ compiler. `Optional C11 + features + `_ + are not required. + +* Support for `IEEE 754 `_ floating + point numbers and `floating point Not-a-Number (NaN) + `_. + +* Support for threads. + +* OpenSSL 1.1.1 or newer for the :mod:`ssl` and :mod:`hashlib` modules. + +* On Windows, Microsoft Visual Studio 2017 or later is required. + +.. versionchanged:: 3.11 + C11 compiler, IEEE 754 and NaN support are now required. + +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required. + +.. versionchanged:: 3.7 + Thread support and OpenSSL 1.0.2 are now required. + +.. versionchanged:: 3.6 + Selected C99 features are now required, like ```` and ``static + inline`` functions. + +.. versionchanged:: 3.5 + On Windows, Visual Studio 2015 or later is required. + +See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform +support". + + .. _configure-options: Configure Options @@ -93,6 +133,12 @@ General Options See :envvar:`PYTHONCOERCECLOCALE` and the :pep:`538`. +.. cmdoption:: --without-freelists + + Disable all freelists except the empty tuple singleton. + + .. versionadded:: 3.11 + .. cmdoption:: --with-platlibdir=DIRNAME Python library directory name (default is ``lib``). diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6eb90df89cac..7931988ed0b0 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2131,7 +2131,7 @@ Build Changes (Contributed by Dong-hee Na and Brett Holman in :issue:`44340`.) * Freelists for object structs can now be disabled. A new :program:`configure` - option :option:`!--without-freelists` can be used to disable all freelists + option :option:`--without-freelists` can be used to disable all freelists except empty tuple singleton. (Contributed by Christian Heimes in :issue:`45522`.) From webhook-mailer at python.org Mon Dec 5 10:37:56 2022 From: webhook-mailer at python.org (vstinner) Date: Mon, 05 Dec 2022 15:37:56 -0000 Subject: [Python-checkins] gh-99892: test_unicodedata: skip test on download failure (#100011) Message-ID: https://github.com/python/cpython/commit/2488c1e1b66366a3a933ff248eff080fabd2351c commit: 2488c1e1b66366a3a933ff248eff080fabd2351c branch: main author: Victor Stinner committer: vstinner date: 2022-12-05T16:37:40+01:00 summary: gh-99892: test_unicodedata: skip test on download failure (#100011) Skip test_normalization() of test_unicodedata if it fails to download NormalizationTest.txt file from pythontest.net. files: A Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst M Lib/test/test_unicodedata.py diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index a85bda3144bc..74503c89e559 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,7 +12,8 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation) + cpython_only, check_disallow_instantiation, + ResourceDenied) class UnicodeMethodsTest(unittest.TestCase): @@ -364,8 +365,8 @@ def test_normalization(self): except PermissionError: self.skipTest(f"Permission error when downloading {TESTDATAURL} " f"into the test data directory") - except (OSError, HTTPException): - self.fail(f"Could not retrieve {TESTDATAURL}") + except (OSError, HTTPException) as exc: + self.skipTest(f"Failed to download {TESTDATAURL}: {exc}") with testdata: self.run_normalization_tests(testdata) diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst new file mode 100644 index 000000000000..eded0361fbeb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst @@ -0,0 +1,2 @@ +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. From webhook-mailer at python.org Mon Dec 5 11:05:24 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 16:05:24 -0000 Subject: [Python-checkins] gh-99892: test_unicodedata: skip test on download failure (GH-100011) Message-ID: https://github.com/python/cpython/commit/c067023d878225ebe0b71eb55e7e98b272c4d5b2 commit: c067023d878225ebe0b71eb55e7e98b272c4d5b2 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T08:05:17-08:00 summary: gh-99892: test_unicodedata: skip test on download failure (GH-100011) Skip test_normalization() of test_unicodedata if it fails to download NormalizationTest.txt file from pythontest.net. (cherry picked from commit 2488c1e1b66366a3a933ff248eff080fabd2351c) Co-authored-by: Victor Stinner files: A Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst M Lib/test/test_unicodedata.py diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 2a93f0fabe21..ad9d3d6afa27 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,7 +12,8 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation) + cpython_only, check_disallow_instantiation, + ResourceDenied) class UnicodeMethodsTest(unittest.TestCase): @@ -345,8 +346,8 @@ def test_normalization(self): except PermissionError: self.skipTest(f"Permission error when downloading {TESTDATAURL} " f"into the test data directory") - except (OSError, HTTPException): - self.fail(f"Could not retrieve {TESTDATAURL}") + except (OSError, HTTPException) as exc: + self.skipTest(f"Failed to download {TESTDATAURL}: {exc}") with testdata: self.run_normalization_tests(testdata) diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst new file mode 100644 index 000000000000..eded0361fbeb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst @@ -0,0 +1,2 @@ +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. From webhook-mailer at python.org Mon Dec 5 11:07:07 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 16:07:07 -0000 Subject: [Python-checkins] gh-99892: test_unicodedata: skip test on download failure (GH-100011) Message-ID: https://github.com/python/cpython/commit/4a7612fbecbdd81a6e708e29aab0dc4c6555948d commit: 4a7612fbecbdd81a6e708e29aab0dc4c6555948d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T08:07:00-08:00 summary: gh-99892: test_unicodedata: skip test on download failure (GH-100011) Skip test_normalization() of test_unicodedata if it fails to download NormalizationTest.txt file from pythontest.net. (cherry picked from commit 2488c1e1b66366a3a933ff248eff080fabd2351c) Co-authored-by: Victor Stinner files: A Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst M Lib/test/test_unicodedata.py diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 85a3c2d6d544..9e0097c892e7 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,7 +12,8 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation) + cpython_only, check_disallow_instantiation, + ResourceDenied) class UnicodeMethodsTest(unittest.TestCase): @@ -345,8 +346,8 @@ def test_normalization(self): except PermissionError: self.skipTest(f"Permission error when downloading {TESTDATAURL} " f"into the test data directory") - except (OSError, HTTPException): - self.fail(f"Could not retrieve {TESTDATAURL}") + except (OSError, HTTPException) as exc: + self.skipTest(f"Failed to download {TESTDATAURL}: {exc}") with testdata: self.run_normalization_tests(testdata) diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst new file mode 100644 index 000000000000..eded0361fbeb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst @@ -0,0 +1,2 @@ +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. From webhook-mailer at python.org Mon Dec 5 11:27:46 2022 From: webhook-mailer at python.org (ambv) Date: Mon, 05 Dec 2022 16:27:46 -0000 Subject: [Python-checkins] gh-60203: Revert changes in cycle.__setstate__ (#99982) Message-ID: https://github.com/python/cpython/commit/922a6cf6c265e2763a003291885ff74d46203fc3 commit: 922a6cf6c265e2763a003291885ff74d46203fc3 branch: main author: Serhiy Storchaka committer: ambv date: 2022-12-05T17:27:40+01:00 summary: gh-60203: Revert changes in cycle.__setstate__ (#99982) In case if only True/False be supported as boolean arguments in future, we should continue to support 1/0 here. files: M Lib/test/test_itertools.py M Modules/itertoolsmodule.c diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index f028bd5fdce8..5f5bcbc7cfb8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -675,6 +675,7 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) + def test_cycle_copy_pickle(self): # check copy, deepcopy, pickle c = cycle('abc') self.assertEqual(next(c), 'a') @@ -710,6 +711,37 @@ def test_cycle(self): d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_unpickle_compat(self): + testcases = [ + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + ] + assert len(testcases) == 20 + for t in testcases: + it = pickle.loads(t) + self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) + def test_cycle_setstate(self): # Verify both modes for restoring state @@ -739,6 +771,10 @@ def test_cycle_setstate(self): c.__setstate__((tuple('defg'), 0)) take(20, c) + # The second argument in the setstate tuple must be an int + with self.assertRaises(TypeError): + cycle('defg').__setstate__((list('abcdefg'), 'x')) + self.assertRaises(TypeError, cycle('').__setstate__, ()) self.assertRaises(TypeError, cycle('').__setstate__, ([],)) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 7869d92bf31a..d45000788c53 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1368,12 +1368,13 @@ cycle_setstate(cycleobject *lz, PyObject *state) PyErr_SetString(PyExc_TypeError, "state is not a tuple"); return NULL; } - if (!PyArg_ParseTuple(state, "O!p", &PyList_Type, &saved, &firstpass)) { + // The second item can be 1/0 in old pickles and True/False in new pickles + if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { return NULL; } Py_INCREF(saved); Py_XSETREF(lz->saved, saved); - lz->firstpass = firstpass; + lz->firstpass = firstpass != 0; lz->index = 0; Py_RETURN_NONE; } From webhook-mailer at python.org Mon Dec 5 11:58:36 2022 From: webhook-mailer at python.org (ambv) Date: Mon, 05 Dec 2022 16:58:36 -0000 Subject: [Python-checkins] [3.10] gh-60203: Revert changes in cycle.__setstate__ (GH-99982) (#100017) Message-ID: https://github.com/python/cpython/commit/b914eee22246311869d0ce127b619b356a8dfb4a commit: b914eee22246311869d0ce127b619b356a8dfb4a branch: 3.10 author: ?ukasz Langa committer: ambv date: 2022-12-05T17:58:30+01:00 summary: [3.10] gh-60203: Revert changes in cycle.__setstate__ (GH-99982) (#100017) In case if only True/False be supported as boolean arguments in future, we should continue to support 1/0 here. (cherry picked from commit 922a6cf6c265e2763a003291885ff74d46203fc3) Co-authored-by: Serhiy Storchaka files: M Lib/test/test_itertools.py M Modules/itertoolsmodule.c diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 4c9c597cc484..0ecc80d829f8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -634,6 +634,7 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) + def test_cycle_copy_pickle(self): # check copy, deepcopy, pickle c = cycle('abc') self.assertEqual(next(c), 'a') @@ -669,6 +670,37 @@ def test_cycle(self): d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_unpickle_compat(self): + testcases = [ + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + ] + assert len(testcases) == 20 + for t in testcases: + it = pickle.loads(t) + self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) + def test_cycle_setstate(self): # Verify both modes for restoring state diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index f8e2c45aecad..25c36e77edbc 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1204,6 +1204,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) PyErr_SetString(PyExc_TypeError, "state is not a tuple"); return NULL; } + // The second item can be 1/0 in old pickles and True/False in new pickles if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { return NULL; } From webhook-mailer at python.org Mon Dec 5 11:58:46 2022 From: webhook-mailer at python.org (ambv) Date: Mon, 05 Dec 2022 16:58:46 -0000 Subject: [Python-checkins] [3.11] gh-60203: Revert changes in cycle.__setstate__ (GH-99982) (#100016) Message-ID: https://github.com/python/cpython/commit/32a2193f67aac1fc740d53f4de9205aa605a7017 commit: 32a2193f67aac1fc740d53f4de9205aa605a7017 branch: 3.11 author: ?ukasz Langa committer: ambv date: 2022-12-05T17:58:40+01:00 summary: [3.11] gh-60203: Revert changes in cycle.__setstate__ (GH-99982) (#100016) In case if only True/False be supported as boolean arguments in future, we should continue to support 1/0 here. (cherry picked from commit 922a6cf6c265e2763a003291885ff74d46203fc3) Co-authored-by: Serhiy Storchaka files: M Lib/test/test_itertools.py M Modules/itertoolsmodule.c diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 238afbbd883d..311c2a3288dc 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -636,6 +636,7 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) + def test_cycle_copy_pickle(self): # check copy, deepcopy, pickle c = cycle('abc') self.assertEqual(next(c), 'a') @@ -671,6 +672,37 @@ def test_cycle(self): d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_unpickle_compat(self): + testcases = [ + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + ] + assert len(testcases) == 20 + for t in testcases: + it = pickle.loads(t) + self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) + def test_cycle_setstate(self): # Verify both modes for restoring state diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 4a7a95730395..d5cdfc591e26 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1200,6 +1200,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) PyErr_SetString(PyExc_TypeError, "state is not a tuple"); return NULL; } + // The second item can be 1/0 in old pickles and True/False in new pickles if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { return NULL; } From webhook-mailer at python.org Mon Dec 5 12:02:42 2022 From: webhook-mailer at python.org (encukou) Date: Mon, 05 Dec 2022 17:02:42 -0000 Subject: [Python-checkins] gh-98680: Add PyBUF_* constants to the Limited API listing (GH-100018) Message-ID: https://github.com/python/cpython/commit/f24738742cc5d3e00409d55ced789cd544b346b5 commit: f24738742cc5d3e00409d55ced789cd544b346b5 branch: main author: Petr Viktorin committer: encukou date: 2022-12-05T18:02:36+01:00 summary: gh-98680: Add PyBUF_* constants to the Limited API listing (GH-100018) ``PyBUF_*`` constants are marked as part of Limited API of Python 3.11+. These were available in 3.11.0 with `Py_LIMITED_API` defined for 3.11, and are necessary to use the buffer API. Omitting them in `stable_abi.toml` was a mistake. files: A Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst M Misc/stable_abi.toml diff --git a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst new file mode 100644 index 000000000000..a87090168e06 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst @@ -0,0 +1,3 @@ +``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. +These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for +3.11, and are necessary to use the buffer API. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index aa12bcc85ceb..c716f403d638 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2271,6 +2271,50 @@ [function.PyMemoryView_FromBuffer] added = '3.11' +# Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) +# (they were available with 3.11.0) +[const.PyBUF_MAX_NDIM] + added = '3.11' +[const.PyBUF_SIMPLE] + added = '3.11' +[const.PyBUF_WRITABLE] + added = '3.11' +[const.PyBUF_FORMAT] + added = '3.11' +[const.PyBUF_ND] + added = '3.11' +[const.PyBUF_STRIDES] + added = '3.11' +[const.PyBUF_C_CONTIGUOUS] + added = '3.11' +[const.PyBUF_F_CONTIGUOUS] + added = '3.11' +[const.PyBUF_ANY_CONTIGUOUS] + added = '3.11' +[const.PyBUF_INDIRECT] + added = '3.11' +[const.PyBUF_CONTIG] + added = '3.11' +[const.PyBUF_CONTIG_RO] + added = '3.11' +[const.PyBUF_STRIDED] + added = '3.11' +[const.PyBUF_STRIDED_RO] + added = '3.11' +[const.PyBUF_RECORDS] + added = '3.11' +[const.PyBUF_RECORDS_RO] + added = '3.11' +[const.PyBUF_FULL] + added = '3.11' +[const.PyBUF_FULL_RO] + added = '3.11' +[const.PyBUF_READ] + added = '3.11' +[const.PyBUF_WRITE] + added = '3.11' + + # (Detailed comments aren't really needed for further entries: from here on # we can use version control logs.) From webhook-mailer at python.org Mon Dec 5 12:13:12 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 17:13:12 -0000 Subject: [Python-checkins] gh-98680: Add PyBUF_* constants to the Limited API listing (GH-100018) Message-ID: https://github.com/python/cpython/commit/c206cc9d9ebb501a7a6e85838356ada201d6b847 commit: c206cc9d9ebb501a7a6e85838356ada201d6b847 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T09:12:44-08:00 summary: gh-98680: Add PyBUF_* constants to the Limited API listing (GH-100018) ``PyBUF_*`` constants are marked as part of Limited API of Python 3.11+. These were available in 3.11.0 with `Py_LIMITED_API` defined for 3.11, and are necessary to use the buffer API. Omitting them in `stable_abi.toml` was a mistake. (cherry picked from commit f24738742cc5d3e00409d55ced789cd544b346b5) Co-authored-by: Petr Viktorin files: A Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst M Misc/stable_abi.toml diff --git a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst new file mode 100644 index 000000000000..a87090168e06 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst @@ -0,0 +1,3 @@ +``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. +These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for +3.11, and are necessary to use the buffer API. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 50dbb0c7bcff..ebae4e44005c 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2271,6 +2271,50 @@ [function.PyMemoryView_FromBuffer] added = '3.11' +# Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) +# (they were available with 3.11.0) +[const.PyBUF_MAX_NDIM] + added = '3.11' +[const.PyBUF_SIMPLE] + added = '3.11' +[const.PyBUF_WRITABLE] + added = '3.11' +[const.PyBUF_FORMAT] + added = '3.11' +[const.PyBUF_ND] + added = '3.11' +[const.PyBUF_STRIDES] + added = '3.11' +[const.PyBUF_C_CONTIGUOUS] + added = '3.11' +[const.PyBUF_F_CONTIGUOUS] + added = '3.11' +[const.PyBUF_ANY_CONTIGUOUS] + added = '3.11' +[const.PyBUF_INDIRECT] + added = '3.11' +[const.PyBUF_CONTIG] + added = '3.11' +[const.PyBUF_CONTIG_RO] + added = '3.11' +[const.PyBUF_STRIDED] + added = '3.11' +[const.PyBUF_STRIDED_RO] + added = '3.11' +[const.PyBUF_RECORDS] + added = '3.11' +[const.PyBUF_RECORDS_RO] + added = '3.11' +[const.PyBUF_FULL] + added = '3.11' +[const.PyBUF_FULL_RO] + added = '3.11' +[const.PyBUF_READ] + added = '3.11' +[const.PyBUF_WRITE] + added = '3.11' + + # (Detailed comments aren't really needed for further entries: from here on # we can use version control logs.) From webhook-mailer at python.org Mon Dec 5 12:34:08 2022 From: webhook-mailer at python.org (zooba) Date: Mon, 05 Dec 2022 17:34:08 -0000 Subject: [Python-checkins] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) Message-ID: https://github.com/python/cpython/commit/124ecd657646f808d1d3282c37ee19aae6bcb47f commit: 124ecd657646f808d1d3282c37ee19aae6bcb47f branch: main author: Michael F?rderer committer: zooba date: 2022-12-05T17:34:00Z summary: bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) files: A Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst M Lib/ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 873c884c3bd9..265eaa8d4b95 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -663,12 +663,15 @@ def _getfinalpathname_nonstrict(path): # 21: ERROR_NOT_READY (implies drive with no media) # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file) # 50: ERROR_NOT_SUPPORTED + # 53: ERROR_BAD_NETPATH + # 65: ERROR_NETWORK_ACCESS_DENIED # 67: ERROR_BAD_NET_NAME (implies remote server unavailable) # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME + # 161: ERROR_BAD_PATHNAME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst new file mode 100644 index 000000000000..79f8c506b54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst @@ -0,0 +1,2 @@ +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) +and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). From webhook-mailer at python.org Mon Dec 5 13:56:19 2022 From: webhook-mailer at python.org (ericvsmith) Date: Mon, 05 Dec 2022 18:56:19 -0000 Subject: [Python-checkins] dataclasses.rst: Prevent horizontal scrolling (gh-100025) Message-ID: https://github.com/python/cpython/commit/51ee0a29e9b20c3e4a94a675e73a894ee2fe447b commit: 51ee0a29e9b20c3e4a94a675e73a894ee2fe447b branch: main author: Ram Rachum committer: ericvsmith date: 2022-12-05T13:56:13-05:00 summary: dataclasses.rst: Prevent horizontal scrolling (gh-100025) files: M Doc/library/dataclasses.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 847299649d1e..32c524a73487 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -79,7 +79,8 @@ Module contents class C: ... - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, + match_args=True, kw_only=False, slots=False, weakref_slot=False) class C: ... From webhook-mailer at python.org Mon Dec 5 13:59:40 2022 From: webhook-mailer at python.org (ericvsmith) Date: Mon, 05 Dec 2022 18:59:40 -0000 Subject: [Python-checkins] [3.11] dataclasses.rst: Prevent horizontal scrolling (gh-100025) (gh-100028) Message-ID: https://github.com/python/cpython/commit/f3a58cece6c7346355397a3e7848b0df38a0e20d commit: f3a58cece6c7346355397a3e7848b0df38a0e20d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ericvsmith date: 2022-12-05T13:59:34-05:00 summary: [3.11] dataclasses.rst: Prevent horizontal scrolling (gh-100025) (gh-100028) dataclasses.rst: Prevent horizontal scrolling (gh-100025) (cherry picked from commit 51ee0a29e9b20c3e4a94a675e73a894ee2fe447b) Co-authored-by: Ram Rachum Co-authored-by: Ram Rachum files: M Doc/library/dataclasses.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 847299649d1e..32c524a73487 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -79,7 +79,8 @@ Module contents class C: ... - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, + match_args=True, kw_only=False, slots=False, weakref_slot=False) class C: ... From webhook-mailer at python.org Mon Dec 5 14:19:14 2022 From: webhook-mailer at python.org (ericvsmith) Date: Mon, 05 Dec 2022 19:19:14 -0000 Subject: [Python-checkins] [3.10] dataclasses.rst: Prevent horizontal scrolling (gh-100025). (gh-100029) Message-ID: https://github.com/python/cpython/commit/e5075986a7bbc4b1f6b1b3fc85f18501d3b48ec4 commit: e5075986a7bbc4b1f6b1b3fc85f18501d3b48ec4 branch: 3.10 author: Eric V. Smith committer: ericvsmith date: 2022-12-05T14:19:09-05:00 summary: [3.10] dataclasses.rst: Prevent horizontal scrolling (gh-100025). (gh-100029) (cherry picked from commit 51ee0a29e9b20c3e4a94a675e73a894ee2fe447b) Co-authored-by: Ram Rachum Co-authored-by: Ram Rachum files: M Doc/library/dataclasses.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index f66010082ad1..add6043b6066 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -79,7 +79,8 @@ Module contents class C: ... - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False) + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, + match_args=True, kw_only=False, slots=False) class C: ... From webhook-mailer at python.org Mon Dec 5 15:40:30 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 05 Dec 2022 20:40:30 -0000 Subject: [Python-checkins] gh-99741: Implement Multi-Phase Init for the _xxsubinterpreters Module (gh-99742) Message-ID: https://github.com/python/cpython/commit/530cc9dbb61df55b83f0219d2282980c9cb1cbd8 commit: 530cc9dbb61df55b83f0219d2282980c9cb1cbd8 branch: main author: Eric Snow committer: ericsnowcurrently date: 2022-12-05T13:40:20-07:00 summary: gh-99741: Implement Multi-Phase Init for the _xxsubinterpreters Module (gh-99742) _xxsubinterpreters is an internal module used for testing. https://github.com/python/cpython/issues/99741 files: A Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst M Include/cpython/pystate.h M Lib/test/test__xxsubinterpreters.py M Modules/_xxsubinterpretersmodule.c M Python/pystate.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0f56b1f21905..0117c23f518c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -353,6 +353,9 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // is necessary to pass safely between interpreters in the same process. typedef struct _xid _PyCrossInterpreterData; +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + struct _xid { // data is the cross-interpreter-safe derivation of a Python object // (see _PyObject_GetCrossInterpreterData). It will be NULL if the @@ -379,7 +382,7 @@ struct _xid { // interpreter given the data. The resulting object (a new // reference) will be equivalent to the original object. This field // is required. - PyObject *(*new_object)(_PyCrossInterpreterData *); + xid_newobjectfunc new_object; // free is called when the data is released. If it is NULL then // nothing will be done to free the data. For some types this is // okay (e.g. bytes) and for those types this field should be set @@ -389,9 +392,20 @@ struct _xid { // leak. In that case, at the very least this field should be set // to PyMem_RawFree (the default if not explicitly set to NULL). // The call will happen with the original interpreter activated. - void (*free)(void *); + xid_freefunc free; }; +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); @@ -400,7 +414,8 @@ PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); /* cross-interpreter data registry */ -typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index f274b637d947..18900bb9f716 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -295,8 +295,8 @@ def clean_up_channels(): class TestBase(unittest.TestCase): def tearDown(self): - clean_up_interpreters() clean_up_channels() + clean_up_interpreters() ################################## @@ -411,6 +411,15 @@ def test_non_shareable_int(self): interpreters.channel_send(self.cid, i) +class ModuleTests(TestBase): + + def test_import_in_interpreter(self): + _run_output( + interpreters.create(), + 'import _xxsubinterpreters as _interpreters', + ) + + ################################## # interpreter tests diff --git a/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst new file mode 100644 index 000000000000..791726b52bfb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst @@ -0,0 +1,2 @@ +We've implemented multi-phase init (PEP 489/630/687) +for the internal (for testing) _xxsubinterpreters module. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 3e064ca8c0b3..d7d7fcaf00a8 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -96,18 +96,20 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) static PyTypeObject * -add_new_type(PyObject *mod, PyTypeObject *cls, crossinterpdatafunc shared) +add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared) { - if (PyType_Ready(cls) != 0) { + PyTypeObject *cls = (PyTypeObject *)PyType_FromMetaclass( + NULL, mod, spec, NULL); + if (cls == NULL) { return NULL; } - if (PyModule_AddType(mod, cls) != 0) { - // XXX When this becomes a heap type, we need to decref here. + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); return NULL; } if (shared != NULL) { if (_PyCrossInterpreterData_RegisterClass(cls, shared)) { - // XXX When this becomes a heap type, we need to decref here. + Py_DECREF(cls); return NULL; } } @@ -135,12 +137,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) * shareable types are all very basic, with no GC. * That said, it becomes much messier once interpreters * no longer share a GIL, so this needs to be fixed before then. */ - // We do what _release_xidata() does in pystate.c. - if (data->free != NULL) { - data->free(data->data); - data->data = NULL; - } - Py_CLEAR(data->obj); + _PyCrossInterpreterData_Clear(NULL, data); if (ignoreexc) { // XXX Emit a warning? PyErr_Clear(); @@ -153,6 +150,69 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) } +/* module state *************************************************************/ + +typedef struct { + PyTypeObject *ChannelIDType; + + /* interpreter exceptions */ + PyObject *RunFailedError; + + /* channel exceptions */ + PyObject *ChannelError; + PyObject *ChannelNotFoundError; + PyObject *ChannelClosedError; + PyObject *ChannelEmptyError; + PyObject *ChannelNotEmptyError; +} module_state; + +static inline module_state * +get_module_state(PyObject *mod) +{ + assert(mod != NULL); + module_state *state = PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static int +traverse_module_state(module_state *state, visitproc visit, void *arg) +{ + /* heap types */ + Py_VISIT(state->ChannelIDType); + + /* interpreter exceptions */ + Py_VISIT(state->RunFailedError); + + /* channel exceptions */ + Py_VISIT(state->ChannelError); + Py_VISIT(state->ChannelNotFoundError); + Py_VISIT(state->ChannelClosedError); + Py_VISIT(state->ChannelEmptyError); + Py_VISIT(state->ChannelNotEmptyError); + return 0; +} + +static int +clear_module_state(module_state *state) +{ + /* heap types */ + (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + Py_CLEAR(state->ChannelIDType); + + /* interpreter exceptions */ + Py_CLEAR(state->RunFailedError); + + /* channel exceptions */ + Py_CLEAR(state->ChannelError); + Py_CLEAR(state->ChannelNotFoundError); + Py_CLEAR(state->ChannelClosedError); + Py_CLEAR(state->ChannelEmptyError); + Py_CLEAR(state->ChannelNotEmptyError); + return 0; +} + + /* data-sharing-specific code ***********************************************/ struct _sharednsitem { @@ -420,82 +480,80 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) #define ERR_CHANNELS_MUTEX_INIT -8 #define ERR_NO_NEXT_CHANNEL_ID -9 -static PyObject *ChannelError; -static PyObject *ChannelNotFoundError; -static PyObject *ChannelClosedError; -static PyObject *ChannelEmptyError; -static PyObject *ChannelNotEmptyError; - static int channel_exceptions_init(PyObject *mod) { - // XXX Move the exceptions into per-module memory? + module_state *state = get_module_state(mod); + if (state == NULL) { + return -1; + } #define ADD(NAME, BASE) \ do { \ - if (NAME == NULL) { \ - NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ - if (NAME == NULL) { \ - return -1; \ - } \ + assert(state->NAME == NULL); \ + state->NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ + if (state->NAME == NULL) { \ + return -1; \ } \ } while (0) // A channel-related operation failed. ADD(ChannelError, PyExc_RuntimeError); // An operation tried to use a channel that doesn't exist. - ADD(ChannelNotFoundError, ChannelError); + ADD(ChannelNotFoundError, state->ChannelError); // An operation tried to use a closed channel. - ADD(ChannelClosedError, ChannelError); + ADD(ChannelClosedError, state->ChannelError); // An operation tried to pop from an empty channel. - ADD(ChannelEmptyError, ChannelError); + ADD(ChannelEmptyError, state->ChannelError); // An operation tried to close a non-empty channel. - ADD(ChannelNotEmptyError, ChannelError); + ADD(ChannelNotEmptyError, state->ChannelError); #undef ADD return 0; } static int -handle_channel_error(int err, PyObject *Py_UNUSED(mod), int64_t cid) +handle_channel_error(int err, PyObject *mod, int64_t cid) { if (err == 0) { assert(!PyErr_Occurred()); return 0; } assert(err < 0); + module_state *state = get_module_state(mod); + assert(state != NULL); if (err == ERR_CHANNEL_NOT_FOUND) { - PyErr_Format(ChannelNotFoundError, + PyErr_Format(state->ChannelNotFoundError, "channel %" PRId64 " not found", cid); } else if (err == ERR_CHANNEL_CLOSED) { - PyErr_Format(ChannelClosedError, + PyErr_Format(state->ChannelClosedError, "channel %" PRId64 " is closed", cid); } else if (err == ERR_CHANNEL_INTERP_CLOSED) { - PyErr_Format(ChannelClosedError, + PyErr_Format(state->ChannelClosedError, "channel %" PRId64 " is already closed", cid); } else if (err == ERR_CHANNEL_EMPTY) { - PyErr_Format(ChannelEmptyError, + PyErr_Format(state->ChannelEmptyError, "channel %" PRId64 " is empty", cid); } else if (err == ERR_CHANNEL_NOT_EMPTY) { - PyErr_Format(ChannelNotEmptyError, + PyErr_Format(state->ChannelNotEmptyError, "channel %" PRId64 " may not be closed " "if not empty (try force=True)", cid); } else if (err == ERR_CHANNEL_MUTEX_INIT) { - PyErr_SetString(ChannelError, + PyErr_SetString(state->ChannelError, "can't initialize mutex for new channel"); } else if (err == ERR_CHANNELS_MUTEX_INIT) { - PyErr_SetString(ChannelError, + PyErr_SetString(state->ChannelError, "can't initialize mutex for channel management"); } else if (err == ERR_NO_NEXT_CHANNEL_ID) { - PyErr_SetString(ChannelError, + PyErr_SetString(state->ChannelError, "failed to get a channel ID"); } else { @@ -1604,8 +1662,6 @@ _channel_is_associated(_channels *channels, int64_t cid, int64_t interp, /* ChannelID class */ -static PyTypeObject ChannelIDType; - typedef struct channelid { PyObject_HEAD int64_t id; @@ -1624,7 +1680,9 @@ channel_id_converter(PyObject *arg, void *ptr) { int64_t cid; struct channel_id_converter_data *data = ptr; - if (PyObject_TypeCheck(arg, &ChannelIDType)) { + module_state *state = get_module_state(data->module); + assert(state != NULL); + if (PyObject_TypeCheck(arg, state->ChannelIDType)) { cid = ((channelid *)arg)->id; } else if (PyIndex_Check(arg)) { @@ -1731,11 +1789,20 @@ _channelid_new(PyObject *mod, PyTypeObject *cls, } static void -channelid_dealloc(PyObject *v) +channelid_dealloc(PyObject *self) { - int64_t cid = ((channelid *)v)->id; - _channels *channels = ((channelid *)v)->channels; - Py_TYPE(v)->tp_free(v); + int64_t cid = ((channelid *)self)->id; + _channels *channels = ((channelid *)self)->channels; + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + /* "Instances of heap-allocated types hold a reference to their type." + * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol + * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse + */ + // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, + // like we do for _abc._abc_data? + Py_DECREF(tp); _channels_drop_id_object(channels, cid); } @@ -1774,11 +1841,6 @@ channelid_int(PyObject *self) return PyLong_FromLongLong(cid->id); } -static PyNumberMethods channelid_as_number = { - .nb_int = (unaryfunc)channelid_int, /* nb_int */ - .nb_index = (unaryfunc)channelid_int, /* nb_index */ -}; - static Py_hash_t channelid_hash(PyObject *self) { @@ -1804,15 +1866,19 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) if (mod == NULL) { return NULL; } + module_state *state = get_module_state(mod); + if (state == NULL) { + goto done; + } - if (!PyObject_TypeCheck(self, &ChannelIDType)) { + if (!PyObject_TypeCheck(self, state->ChannelIDType)) { res = Py_NewRef(Py_NotImplemented); goto done; } channelid *cid = (channelid *)self; int equal; - if (PyObject_TypeCheck(other, &ChannelIDType)) { + if (PyObject_TypeCheck(other, state->ChannelIDType)) { channelid *othercid = (channelid *)other; equal = (cid->end == othercid->end) && (cid->id == othercid->id); } @@ -1892,10 +1958,14 @@ _channelid_from_xid(_PyCrossInterpreterData *data) if (mod == NULL) { return NULL; } + module_state *state = get_module_state(mod); + if (state == NULL) { + return NULL; + } // Note that we do not preserve the "resolve" flag. PyObject *cid = NULL; - int err = newchannelid(&ChannelIDType, xid->id, xid->end, + int err = newchannelid(state->ChannelIDType, xid->id, xid->end, _global_channels(), 0, 0, (channelid **)&cid); if (err != 0) { @@ -1926,20 +1996,20 @@ _channelid_from_xid(_PyCrossInterpreterData *data) } static int -_channelid_shared(PyObject *obj, _PyCrossInterpreterData *data) -{ - struct _channelid_xid *xid = PyMem_NEW(struct _channelid_xid, 1); - if (xid == NULL) { +_channelid_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _channelid_xid), obj, + _channelid_from_xid + ) < 0) + { return -1; } + struct _channelid_xid *xid = (struct _channelid_xid *)data->data; xid->id = ((channelid *)obj)->id; xid->end = ((channelid *)obj)->end; xid->resolve = ((channelid *)obj)->resolve; - - data->data = xid; - data->obj = Py_NewRef(obj); - data->new_object = _channelid_from_xid; - data->free = PyMem_Free; return 0; } @@ -1992,61 +2062,45 @@ static PyGetSetDef channelid_getsets[] = { PyDoc_STRVAR(channelid_doc, "A channel ID identifies a channel and may be used as an int."); -static PyTypeObject ChannelIDType = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "_xxsubinterpreters.ChannelID", /* tp_name */ - sizeof(channelid), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)channelid_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)channelid_repr, /* tp_repr */ - &channelid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - channelid_hash, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)channelid_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - // Use Py_TPFLAGS_DISALLOW_INSTANTIATION so the type cannot be instantiated - // from Python code. We do this because there is a strong relationship - // between channel IDs and the channel lifecycle, so this limitation avoids - // related complications. Use the _channel_id() function instead. - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_DISALLOW_INSTANTIATION, /* tp_flags */ - channelid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - channelid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - channelid_getsets, /* tp_getset */ +static PyType_Slot ChannelIDType_slots[] = { + {Py_tp_dealloc, (destructor)channelid_dealloc}, + {Py_tp_doc, (void *)channelid_doc}, + {Py_tp_repr, (reprfunc)channelid_repr}, + {Py_tp_str, (reprfunc)channelid_str}, + {Py_tp_hash, channelid_hash}, + {Py_tp_richcompare, channelid_richcompare}, + {Py_tp_getset, channelid_getsets}, + // number slots + {Py_nb_int, (unaryfunc)channelid_int}, + {Py_nb_index, (unaryfunc)channelid_int}, + {0, NULL}, }; +static PyType_Spec ChannelIDType_spec = { + .name = "_xxsubinterpreters.ChannelID", + .basicsize = sizeof(channelid), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .slots = ChannelIDType_slots, +}; -/* interpreter-specific code ************************************************/ -static PyObject * RunFailedError = NULL; +/* interpreter-specific code ************************************************/ static int interp_exceptions_init(PyObject *mod) { - // XXX Move the exceptions into per-module memory? + module_state *state = get_module_state(mod); + if (state == NULL) { + return -1; + } #define ADD(NAME, BASE) \ do { \ - if (NAME == NULL) { \ - NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ - if (NAME == NULL) { \ - return -1; \ - } \ + assert(state->NAME == NULL); \ + state->NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ + if (state->NAME == NULL) { \ + return -1; \ } \ } while (0) @@ -2167,9 +2221,10 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, if (_ensure_not_running(interp) < 0) { return -1; } + module_state *state = get_module_state(mod); int needs_import = 0; - _sharedns *shared = _get_shared_ns(shareables, &ChannelIDType, + _sharedns *shared = _get_shared_ns(shareables, state->ChannelIDType, &needs_import); if (shared == NULL && PyErr_Occurred()) { return -1; @@ -2195,7 +2250,8 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, // Propagate any exception out to the caller. if (exc != NULL) { - _sharedexception_apply(exc, RunFailedError); + assert(state != NULL); + _sharedexception_apply(exc, state->RunFailedError); _sharedexception_free(exc); } else if (result != 0) { @@ -2530,8 +2586,12 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored)) (void)handle_channel_error(cid, self, -1); return NULL; } + module_state *state = get_module_state(self); + if (state == NULL) { + return NULL; + } PyObject *id = NULL; - int err = newchannelid(&ChannelIDType, cid, 0, + int err = newchannelid(state->ChannelIDType, cid, 0, &_globals.channels, 0, 0, (channelid **)&id); if (handle_channel_error(err, self, cid)) { @@ -2594,10 +2654,16 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } + module_state *state = get_module_state(self); + if (state == NULL) { + Py_DECREF(ids); + ids = NULL; + goto finally; + } int64_t *cur = cids; for (int64_t i=0; i < count; cur++, i++) { PyObject *id = NULL; - int err = newchannelid(&ChannelIDType, *cur, 0, + int err = newchannelid(state->ChannelIDType, *cur, 0, &_globals.channels, 0, 0, (channelid **)&id); if (handle_channel_error(err, self, *cur)) { @@ -2850,7 +2916,11 @@ ends are closed. Closing an already closed end is a noop."); static PyObject * channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *cls = &ChannelIDType; + module_state *state = get_module_state(self); + if (state == NULL) { + return NULL; + } + PyTypeObject *cls = state->ChannelIDType; PyObject *mod = get_module_from_owned_type(cls); if (mod == NULL) { return NULL; @@ -2924,9 +2994,16 @@ module_exec(PyObject *mod) } /* Add other types */ - if (add_new_type(mod, &ChannelIDType, _channelid_shared) == NULL) { + module_state *state = get_module_state(mod); + + // ChannelID + state->ChannelIDType = add_new_type( + mod, &ChannelIDType_spec, _channelid_shared); + if (state->ChannelIDType == NULL) { goto error; } + + // PyInterpreterID if (PyModule_AddType(mod, &_PyInterpreterID_Type) < 0) { goto error; } @@ -2934,31 +3011,57 @@ module_exec(PyObject *mod) return 0; error: - (void)_PyCrossInterpreterData_UnregisterClass(&ChannelIDType); + (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); _globals_fini(); return -1; } +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + traverse_module_state(state, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + clear_module_state(state); + return 0; +} + +static void +module_free(void *mod) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + clear_module_state(state); + _globals_fini(); +} + static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, .m_name = MODULE_NAME, .m_doc = module_doc, - .m_size = -1, + .m_size = sizeof(module_state), .m_methods = module_functions, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, }; - PyMODINIT_FUNC PyInit__xxsubinterpreters(void) { - /* Create the module */ - PyObject *mod = PyModule_Create(&moduledef); - if (mod == NULL) { - return NULL; - } - if (module_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - return mod; + return PyModuleDef_Init(&moduledef); } diff --git a/Python/pystate.c b/Python/pystate.c index 0fdcdf1e3569..ea3c22c5d71a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1789,30 +1789,78 @@ PyGILState_Release(PyGILState_STATE oldstate) /* cross-interpreter data */ -crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); +static inline void +_xidata_init(_PyCrossInterpreterData *data) +{ + // If the value is being reused + // then _xidata_clear() should have been called already. + assert(data->data == NULL); + assert(data->obj == NULL); + *data = (_PyCrossInterpreterData){0}; + data->interp = -1; +} -/* This is a separate func from _PyCrossInterpreterData_Lookup in order - to keep the registry code separate. */ -static crossinterpdatafunc -_lookup_getdata(PyObject *obj) +static inline void +_xidata_clear(_PyCrossInterpreterData *data) { - crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); - if (getdata == NULL && PyErr_Occurred() == 0) - PyErr_Format(PyExc_ValueError, - "%S does not support cross-interpreter data", obj); - return getdata; + if (data->free != NULL) { + data->free(data->data); + } + data->data = NULL; + Py_CLEAR(data->obj); +} + +void +_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(data != NULL); + assert(new_object != NULL); + _xidata_init(data); + data->data = shared; + if (obj != NULL) { + assert(interp != NULL); + // released in _PyCrossInterpreterData_Clear() + data->obj = Py_NewRef(obj); + } + // Ideally every object would know its owning interpreter. + // Until then, we have to rely on the caller to identify it + // (but we don't need it in all cases). + data->interp = (interp != NULL) ? interp->id : -1; + data->new_object = new_object; } int -_PyObject_CheckCrossInterpreterData(PyObject *obj) -{ - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { +_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(size > 0); + // For now we always free the shared data in the same interpreter + // where it was allocated, so the interpreter is required. + assert(interp != NULL); + _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + data->data = PyMem_Malloc(size); + if (data->data == NULL) { return -1; } + data->free = PyMem_Free; return 0; } +void +_PyCrossInterpreterData_Clear(PyInterpreterState *interp, + _PyCrossInterpreterData *data) +{ + assert(data != NULL); + // This must be called in the owning interpreter. + assert(interp == NULL || data->interp == interp->id); + _xidata_clear(data); +} + static int _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) { @@ -1835,6 +1883,30 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) return 0; } +crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); + +/* This is a separate func from _PyCrossInterpreterData_Lookup in order + to keep the registry code separate. */ +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) +{ + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S does not support cross-interpreter data", obj); + return getdata; +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + return -1; + } + return 0; +} + int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { @@ -1847,7 +1919,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) // Reset data before re-populating. *data = (_PyCrossInterpreterData){0}; - data->free = PyMem_RawFree; // Set a default that may be overridden. + data->interp = -1; // Call the "getdata" func for the object. Py_INCREF(obj); @@ -1856,7 +1928,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) Py_DECREF(obj); return -1; } - int res = getdata(obj, data); + int res = getdata(tstate, obj, data); Py_DECREF(obj); if (res != 0) { return -1; @@ -1872,21 +1944,17 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) return 0; } -static void -_release_xidata(void *arg) +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) { - _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg; - if (data->free != NULL) { - data->free(data->data); - } - data->data = NULL; - Py_CLEAR(data->obj); + return data->new_object(data); } +typedef void (*releasefunc)(PyInterpreterState *, void *); + static void _call_in_interpreter(struct _gilstate_runtime_state *gilstate, - PyInterpreterState *interp, - void (*func)(void *), void *arg) + PyInterpreterState *interp, releasefunc func, void *arg) { /* We would use Py_AddPendingCall() if it weren't specific to the * main interpreter (see bpo-33608). In the meantime we take a @@ -1902,7 +1970,7 @@ _call_in_interpreter(struct _gilstate_runtime_state *gilstate, // XXX Once the GIL is per-interpreter, this should be called with the // calling interpreter's GIL released and the target interpreter's held. - func(arg); + func(interp, arg); // Switch back. if (save_tstate != NULL) { @@ -1931,16 +1999,11 @@ _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) // "Release" the data and/or the object. struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; - _call_in_interpreter(gilstate, interp, _release_xidata, data); + _call_in_interpreter(gilstate, interp, + (releasefunc)_PyCrossInterpreterData_Clear, data); return 0; } -PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) -{ - return data->new_object(data); -} - /* registry of {type -> crossinterpdatafunc} */ /* For now we use a global registry of shareable classes. An @@ -2091,16 +2154,21 @@ _new_bytes_object(_PyCrossInterpreterData *data) } static int -_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) +_bytes_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - struct _shared_bytes_data *shared = PyMem_NEW(struct _shared_bytes_data, 1); + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_bytes_data), obj, + _new_bytes_object + ) < 0) + { + return -1; + } + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { + _PyCrossInterpreterData_Clear(tstate->interp, data); return -1; } - data->data = (void *)shared; - data->obj = Py_NewRef(obj); // Will be "released" (decref'ed) when data released. - data->new_object = _new_bytes_object; - data->free = PyMem_Free; return 0; } @@ -2118,16 +2186,20 @@ _new_str_object(_PyCrossInterpreterData *data) } static int -_str_shared(PyObject *obj, _PyCrossInterpreterData *data) +_str_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - struct _shared_str_data *shared = PyMem_NEW(struct _shared_str_data, 1); + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_str_data), obj, + _new_str_object + ) < 0) + { + return -1; + } + struct _shared_str_data *shared = (struct _shared_str_data *)data->data; shared->kind = PyUnicode_KIND(obj); shared->buffer = PyUnicode_DATA(obj); shared->len = PyUnicode_GET_LENGTH(obj); - data->data = (void *)shared; - data->obj = Py_NewRef(obj); // Will be "released" (decref'ed) when data released. - data->new_object = _new_str_object; - data->free = PyMem_Free; return 0; } @@ -2138,7 +2210,8 @@ _new_long_object(_PyCrossInterpreterData *data) } static int -_long_shared(PyObject *obj, _PyCrossInterpreterData *data) +_long_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { /* Note that this means the size of shareable ints is bounded by * sys.maxsize. Hence on 32-bit architectures that is half the @@ -2151,10 +2224,9 @@ _long_shared(PyObject *obj, _PyCrossInterpreterData *data) } return -1; } - data->data = (void *)value; - data->obj = NULL; - data->new_object = _new_long_object; - data->free = NULL; + _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, + _new_long_object); + // data->obj and data->free remain NULL return 0; } @@ -2166,12 +2238,12 @@ _new_none_object(_PyCrossInterpreterData *data) } static int -_none_shared(PyObject *obj, _PyCrossInterpreterData *data) +_none_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - data->data = NULL; - // data->obj remains NULL - data->new_object = _new_none_object; - data->free = NULL; // There is nothing to free. + _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, + _new_none_object); + // data->data, data->obj and data->free remain NULL return 0; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index cc465134a9e0..adbb319a1b6f 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -496,7 +496,6 @@ Modules/_pickle.c - PicklerMemoProxyType - Modules/_pickle.c - Pickler_Type - Modules/_pickle.c - UnpicklerMemoProxyType - Modules/_pickle.c - Unpickler_Type - -Modules/_xxsubinterpretersmodule.c - ChannelIDtype - Modules/_zoneinfo.c - PyZoneInfo_ZoneInfoType - Modules/ossaudiodev.c - OSSAudioType - Modules/ossaudiodev.c - OSSMixerType - @@ -523,12 +522,6 @@ Modules/_ctypes/_ctypes.c - PyExc_ArgError - Modules/_cursesmodule.c - PyCursesError - Modules/_decimal/_decimal.c - DecimalException - Modules/_tkinter.c - Tkinter_TclError - -Modules/_xxsubinterpretersmodule.c - ChannelError - -Modules/_xxsubinterpretersmodule.c - ChannelNotFoundError - -Modules/_xxsubinterpretersmodule.c - ChannelClosedError - -Modules/_xxsubinterpretersmodule.c - ChannelEmptyError - -Modules/_xxsubinterpretersmodule.c - ChannelNotEmptyError - -Modules/_xxsubinterpretersmodule.c - RunFailedError - Modules/ossaudiodev.c - OSSAudioError - Modules/socketmodule.c - socket_herror - Modules/socketmodule.c - socket_gaierror - From webhook-mailer at python.org Mon Dec 5 15:55:51 2022 From: webhook-mailer at python.org (gpshead) Date: Mon, 05 Dec 2022 20:55:51 -0000 Subject: [Python-checkins] gh-100001: Omit control characters in http.server stderr logs. (#100002) Message-ID: https://github.com/python/cpython/commit/d8ab0a4dfa48f881b4ac9ab857d2e9de42f72828 commit: d8ab0a4dfa48f881b4ac9ab857d2e9de42f72828 branch: main author: Gregory P. Smith committer: gpshead date: 2022-12-05T12:55:45-08:00 summary: gh-100001: Omit control characters in http.server stderr logs. (#100002) Replace control characters in http.server.BaseHTTPRequestHandler.log_message with an escaped \xHH sequence to avoid causing problems for the terminal the output is printed to. files: A Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Doc/library/http.server.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 81b6bf5373b4..154f3f3c3904 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -512,3 +512,10 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow to remote clients connecting to your +server to send nefarious control codes to your terminal. + diff --git a/Lib/http/server.py b/Lib/http/server.py index 8aee31bac275..3b5bd9eb0129 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -562,6 +563,10 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + def log_message(self, format, *args): """Log an arbitrary message. @@ -577,12 +582,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index a937258069ed..b0e2d713e3de 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -26,7 +26,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -990,6 +990,25 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst new file mode 100644 index 000000000000..a396e95cd83f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst @@ -0,0 +1,6 @@ +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` +``.log_message`` method to replace control characters with a ``\xHH`` hex escape +before printing. From webhook-mailer at python.org Mon Dec 5 16:16:23 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 21:16:23 -0000 Subject: [Python-checkins] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) Message-ID: https://github.com/python/cpython/commit/ec8c06bc28b29b62d31b953e54f1d8d8535faa80 commit: ec8c06bc28b29b62d31b953e54f1d8d8535faa80 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T13:16:14-08:00 summary: gh-100001: Omit control characters in http.server stderr logs. (GH-100002) Replace control characters in http.server.BaseHTTPRequestHandler.log_message with an escaped \xHH sequence to avoid causing problems for the terminal the output is printed to. (cherry picked from commit d8ab0a4dfa48f881b4ac9ab857d2e9de42f72828) Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Doc/library/http.server.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 89a25b8cc797..fd6d7cb4ee1c 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -499,3 +499,10 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow to remote clients connecting to your +server to send nefarious control codes to your terminal. + diff --git a/Lib/http/server.py b/Lib/http/server.py index e8517a7e4423..ca429428fdfd 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -563,6 +564,10 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + def log_message(self, format, *args): """Log an arbitrary message. @@ -578,12 +583,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 8fdbab4ec097..34e0e3548359 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -26,7 +26,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -984,6 +984,25 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst new file mode 100644 index 000000000000..a396e95cd83f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst @@ -0,0 +1,6 @@ +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` +``.log_message`` method to replace control characters with a ``\xHH`` hex escape +before printing. From webhook-mailer at python.org Mon Dec 5 16:26:34 2022 From: webhook-mailer at python.org (hugovk) Date: Mon, 05 Dec 2022 21:26:34 -0000 Subject: [Python-checkins] Use sphinxext-opengraph to generate OpenGraph metadata (#99931) Message-ID: https://github.com/python/cpython/commit/f49c735e525cf031ddbfc19161aafac4fb18837b commit: f49c735e525cf031ddbfc19161aafac4fb18837b branch: main author: Hugo van Kemenade committer: hugovk date: 2022-12-05T23:26:28+02:00 summary: Use sphinxext-opengraph to generate OpenGraph metadata (#99931) Co-authored-by: C.A.M. Gerlach files: A Doc/_static/og-image.png A Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst M Doc/conf.py M Doc/requirements.txt diff --git a/Doc/_static/og-image.png b/Doc/_static/og-image.png new file mode 100644 index 000000000000..0e80751e7403 Binary files /dev/null and b/Doc/_static/og-image.png differ diff --git a/Doc/conf.py b/Doc/conf.py index 6fad5c668dab..c7d2f43b7a8e 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -13,9 +13,25 @@ # General configuration # --------------------- -extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', - 'pyspecific', 'c_annotations', 'escape4chm', - 'asdl_highlight', 'peg_highlight', 'glossary_search'] +extensions = [ + 'asdl_highlight', + 'c_annotations', + 'escape4chm', + 'glossary_search', + 'peg_highlight', + 'pyspecific', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', +] + +# Skip if downstream redistributors haven't installed it +try: + import sphinxext.opengraph +except ImportError: + pass +else: + extensions.append('sphinxext.opengraph') + doctest_global_setup = ''' try: @@ -114,7 +130,7 @@ html_use_opensearch = 'https://docs.python.org/' + version # Additional static files. -html_static_path = ['tools/static'] +html_static_path = ['_static', 'tools/static'] # Output file base name for HTML help builder. htmlhelp_basename = 'python' + release.replace('.', '') @@ -238,3 +254,13 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' + +# sphinxext-opengraph config +ogp_site_url = 'https://docs.python.org/3/' +ogp_site_name = 'Python documentation' +ogp_image = '_static/og-image.png' +ogp_custom_meta_tags = [ + '', + '', + '', +] diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 958665db69e2..134f39d6d7b3 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -8,6 +8,7 @@ sphinx==4.5.0 blurb sphinx-lint==0.6.7 +sphinxext-opengraph>=0.7.1 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst new file mode 100644 index 000000000000..0c01a2cb2cfa --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst @@ -0,0 +1,2 @@ +Use `sphinxext-opengraph `__ +to generate `OpenGraph metadata `__. From webhook-mailer at python.org Mon Dec 5 16:39:28 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 21:39:28 -0000 Subject: [Python-checkins] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) Message-ID: https://github.com/python/cpython/commit/a726f747e659efed674db1ebf57218c20d8c0c39 commit: a726f747e659efed674db1ebf57218c20d8c0c39 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T13:39:22-08:00 summary: gh-100001: Omit control characters in http.server stderr logs. (GH-100002) Replace control characters in http.server.BaseHTTPRequestHandler.log_message with an escaped \xHH sequence to avoid causing problems for the terminal the output is printed to. (cherry picked from commit d8ab0a4dfa48f881b4ac9ab857d2e9de42f72828) Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Doc/library/http.server.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 81b6bf5373b4..154f3f3c3904 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -512,3 +512,10 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow to remote clients connecting to your +server to send nefarious control codes to your terminal. + diff --git a/Lib/http/server.py b/Lib/http/server.py index f2aeb6594202..9845b0fdbbd7 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -562,6 +563,10 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + def log_message(self, format, *args): """Log an arbitrary message. @@ -577,12 +582,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index a937258069ed..b0e2d713e3de 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -26,7 +26,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -990,6 +990,25 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst new file mode 100644 index 000000000000..a396e95cd83f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst @@ -0,0 +1,6 @@ +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` +``.log_message`` method to replace control characters with a ``\xHH`` hex escape +before printing. From webhook-mailer at python.org Mon Dec 5 16:41:12 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 05 Dec 2022 21:41:12 -0000 Subject: [Python-checkins] gh-99984: Fix Compiler Warnings (#100036) Message-ID: https://github.com/python/cpython/commit/e9e63ad8653296c199446d6f7cdad889e492a34e commit: e9e63ad8653296c199446d6f7cdad889e492a34e branch: main author: Eric Snow committer: ericsnowcurrently date: 2022-12-05T14:41:06-07:00 summary: gh-99984: Fix Compiler Warnings (#100036) https://github.com/python/cpython/issues/99984 files: M Modules/_xxsubinterpretersmodule.c diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index d7d7fcaf00a8..0892fa3a9595 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -2583,7 +2583,7 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t cid = _channel_create(&_globals.channels); if (cid < 0) { - (void)handle_channel_error(cid, self, -1); + (void)handle_channel_error(-1, self, cid); return NULL; } module_state *state = get_module_state(self); @@ -2985,6 +2985,11 @@ module_exec(PyObject *mod) return -1; } + module_state *state = get_module_state(mod); + if (state == NULL) { + goto error; + } + /* Add exception types */ if (interp_exceptions_init(mod) != 0) { goto error; @@ -2994,7 +2999,6 @@ module_exec(PyObject *mod) } /* Add other types */ - module_state *state = get_module_state(mod); // ChannelID state->ChannelIDType = add_new_type( From webhook-mailer at python.org Mon Dec 5 17:28:00 2022 From: webhook-mailer at python.org (gpshead) Date: Mon, 05 Dec 2022 22:28:00 -0000 Subject: [Python-checkins] gh-100001: Also escape \s in http.server log messages. (#100038) Message-ID: https://github.com/python/cpython/commit/7e29398407dbd53b714702abb89aa2fd7baca48a commit: 7e29398407dbd53b714702abb89aa2fd7baca48a branch: main author: Gregory P. Smith committer: gpshead date: 2022-12-05T14:27:55-08:00 summary: gh-100001: Also escape \s in http.server log messages. (#100038) Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so that it is technically possible to parse the line and reconstruct what the original data was. Without this a \xHH is ambiguious as to if it is a hex replacement we put in or the characters r"\x" came through in the original request line. files: A Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 3b5bd9eb0129..8acabff605e7 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -566,6 +566,7 @@ def log_error(self, format, *args): # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes _control_char_table = str.maketrans( {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' def log_message(self, format, *args): """Log an arbitrary message. diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index b0e2d713e3de..ca078862cca6 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -1000,6 +1000,7 @@ def test_unprintable_not_logged(self): log_message(self.handler, '/\033bar\000\033') log_message(self.handler, '/spam %s.', 'a') log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + log_message(self.handler, '"GET /foo\\b"ar\007 HTTP/1.0"') stderr = fake_stderr.getvalue() self.assertNotIn('\033', stderr) # non-printable chars are caught. self.assertNotIn('\000', stderr) # non-printable chars are caught. @@ -1008,6 +1009,7 @@ def test_unprintable_not_logged(self): self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) self.assertIn('/spam a.', lines[2]) self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + self.assertIn(r'"GET /foo\\b"ar\x07 HTTP/1.0"', lines[4]) def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst new file mode 100644 index 000000000000..e305352c7a55 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst @@ -0,0 +1,5 @@ +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. From webhook-mailer at python.org Mon Dec 5 17:48:03 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 22:48:03 -0000 Subject: [Python-checkins] gh-100001: Also escape \s in http.server log messages. (GH-100038) Message-ID: https://github.com/python/cpython/commit/aae7b43ca3d2bb2028370b8252ccb51006827429 commit: aae7b43ca3d2bb2028370b8252ccb51006827429 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T14:47:57-08:00 summary: gh-100001: Also escape \s in http.server log messages. (GH-100038) Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so that it is technically possible to parse the line and reconstruct what the original data was. Without this a \xHH is ambiguious as to if it is a hex replacement we put in or the characters r"\x" came through in the original request line. (cherry picked from commit 7e29398407dbd53b714702abb89aa2fd7baca48a) Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index ca429428fdfd..03dbaa51b798 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -567,6 +567,7 @@ def log_error(self, format, *args): # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes _control_char_table = str.maketrans( {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' def log_message(self, format, *args): """Log an arbitrary message. diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 34e0e3548359..ac8da494e9bb 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -994,6 +994,7 @@ def test_unprintable_not_logged(self): log_message(self.handler, '/\033bar\000\033') log_message(self.handler, '/spam %s.', 'a') log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + log_message(self.handler, '"GET /foo\\b"ar\007 HTTP/1.0"') stderr = fake_stderr.getvalue() self.assertNotIn('\033', stderr) # non-printable chars are caught. self.assertNotIn('\000', stderr) # non-printable chars are caught. @@ -1002,6 +1003,7 @@ def test_unprintable_not_logged(self): self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) self.assertIn('/spam a.', lines[2]) self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + self.assertIn(r'"GET /foo\\b"ar\x07 HTTP/1.0"', lines[4]) def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst new file mode 100644 index 000000000000..e305352c7a55 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst @@ -0,0 +1,5 @@ +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. From webhook-mailer at python.org Mon Dec 5 17:53:47 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 05 Dec 2022 22:53:47 -0000 Subject: [Python-checkins] gh-100001: Also escape \s in http.server log messages. (GH-100038) Message-ID: https://github.com/python/cpython/commit/b2ff0f761de4dc8e7bf0fea3c64bb25d9fcfcd36 commit: b2ff0f761de4dc8e7bf0fea3c64bb25d9fcfcd36 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T14:53:41-08:00 summary: gh-100001: Also escape \s in http.server log messages. (GH-100038) Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so that it is technically possible to parse the line and reconstruct what the original data was. Without this a \xHH is ambiguious as to if it is a hex replacement we put in or the characters r"\x" came through in the original request line. (cherry picked from commit 7e29398407dbd53b714702abb89aa2fd7baca48a) Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 9845b0fdbbd7..058ee47ba10e 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -566,6 +566,7 @@ def log_error(self, format, *args): # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes _control_char_table = str.maketrans( {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' def log_message(self, format, *args): """Log an arbitrary message. diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index b0e2d713e3de..ca078862cca6 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -1000,6 +1000,7 @@ def test_unprintable_not_logged(self): log_message(self.handler, '/\033bar\000\033') log_message(self.handler, '/spam %s.', 'a') log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + log_message(self.handler, '"GET /foo\\b"ar\007 HTTP/1.0"') stderr = fake_stderr.getvalue() self.assertNotIn('\033', stderr) # non-printable chars are caught. self.assertNotIn('\000', stderr) # non-printable chars are caught. @@ -1008,6 +1009,7 @@ def test_unprintable_not_logged(self): self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) self.assertIn('/spam a.', lines[2]) self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + self.assertIn(r'"GET /foo\\b"ar\x07 HTTP/1.0"', lines[4]) def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst new file mode 100644 index 000000000000..e305352c7a55 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst @@ -0,0 +1,5 @@ +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. From webhook-mailer at python.org Mon Dec 5 18:10:28 2022 From: webhook-mailer at python.org (ned-deily) Date: Mon, 05 Dec 2022 23:10:28 -0000 Subject: [Python-checkins] [3.7] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) (GH-100034) Message-ID: https://github.com/python/cpython/commit/b5bdf6a90d0c2a9c12b671340f7776e233ef625a commit: b5bdf6a90d0c2a9c12b671340f7776e233ef625a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ned-deily date: 2022-12-05T18:10:10-05:00 summary: [3.7] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) (GH-100034) Replace control characters in http.server.BaseHTTPRequestHandler.log_message with an escaped \xHH sequence to avoid causing problems for the terminal the output is printed to. (cherry picked from commit d8ab0a4dfa48f881b4ac9ab857d2e9de42f72828) Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Doc/library/http.server.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index a93362d96f13..28e4f414283b 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -481,3 +481,13 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow remote clients connecting to your +server to send nefarious control codes to your terminal. + +.. versionadded:: 3.7.16 + scrubbing control characters from log messages + diff --git a/Lib/http/server.py b/Lib/http/server.py index 7d51fc9976c5..ba2acbc98bf7 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -564,6 +565,11 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' + def log_message(self, format, *args): """Log an arbitrary message. @@ -579,12 +585,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index c11855ba228d..b3e15c475a40 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -24,7 +24,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -908,6 +908,25 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst new file mode 100644 index 000000000000..a396e95cd83f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst @@ -0,0 +1,6 @@ +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` +``.log_message`` method to replace control characters with a ``\xHH`` hex escape +before printing. From webhook-mailer at python.org Mon Dec 5 18:15:18 2022 From: webhook-mailer at python.org (gpshead) Date: Mon, 05 Dec 2022 23:15:18 -0000 Subject: [Python-checkins] gh-100001: Remove doc typo, add versionadded (#100042) Message-ID: https://github.com/python/cpython/commit/bed15f87eadc726122185cf41efcdda289f4a7b1 commit: bed15f87eadc726122185cf41efcdda289f4a7b1 branch: main author: Gregory P. Smith committer: gpshead date: 2022-12-05T15:15:13-08:00 summary: gh-100001: Remove doc typo, add versionadded (#100042) gh-100001: Remove new doc typo, add versionadded. files: M Doc/library/http.server.rst diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 154f3f3c3904..3290b9beab3e 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -516,6 +516,8 @@ to be served. Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` -implementation. This could allow to remote clients connecting to your +implementation. This could allow remote clients connecting to your server to send nefarious control codes to your terminal. +.. versionadded:: 3.12 + Control characters are scrubbed in stderr logs. From webhook-mailer at python.org Mon Dec 5 22:35:48 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Tue, 06 Dec 2022 03:35:48 -0000 Subject: [Python-checkins] gh-99957: Add `frozen_default` parameter on `dataclass_transform` (#99958) Message-ID: https://github.com/python/cpython/commit/5c19050546e3e37a8889a0baa2954e1444e803d3 commit: 5c19050546e3e37a8889a0baa2954e1444e803d3 branch: main author: Erik De Bonte committer: JelleZijlstra date: 2022-12-05T19:35:43-08:00 summary: gh-99957: Add `frozen_default` parameter on `dataclass_transform` (#99958) files: A Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst M Doc/library/typing.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 94c9cb11f02d..356f919a1897 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2575,6 +2575,10 @@ Functions and decorators assumed to be True or False if it is omitted by the caller. * ``kw_only_default`` indicates whether the ``kw_only`` parameter is assumed to be True or False if it is omitted by the caller. + * ``frozen_default`` indicates whether the ``frozen`` parameter is + assumed to be True or False if it is omitted by the caller. + + .. versionadded:: 3.12 * ``field_specifiers`` specifies a static list of supported classes or functions that describe fields, similar to ``dataclasses.field()``. * Arbitrary other keyword arguments are accepted in order to allow for diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index da602b0199d5..1cae1b0de714 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7719,6 +7719,7 @@ class CustomerModel: "eq_default": True, "order_default": False, "kw_only_default": True, + "frozen_default": False, "field_specifiers": (), "kwargs": {}, } @@ -7749,6 +7750,7 @@ class CustomerModel(Decorated, frozen=True): "eq_default": True, "order_default": True, "kw_only_default": False, + "frozen_default": False, "field_specifiers": (), "kwargs": {"make_everything_awesome": True}, } @@ -7765,7 +7767,7 @@ def __new__( return super().__new__(cls, name, bases, namespace) Decorated = dataclass_transform( - order_default=True, field_specifiers=(Field,) + order_default=True, frozen_default=True, field_specifiers=(Field,) )(ModelMeta) class ModelBase(metaclass=Decorated): ... @@ -7780,6 +7782,7 @@ class CustomerModel(ModelBase, init=False): "eq_default": True, "order_default": True, "kw_only_default": False, + "frozen_default": True, "field_specifiers": (Field,), "kwargs": {}, } diff --git a/Lib/typing.py b/Lib/typing.py index 38e227e3c55d..d9d6fbcdb8f0 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -3363,6 +3363,7 @@ def dataclass_transform( eq_default: bool = True, order_default: bool = False, kw_only_default: bool = False, + frozen_default: bool = False, field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, ) -> Callable[[T], T]: @@ -3416,6 +3417,8 @@ class CustomerModel(ModelBase): assumed to be True or False if it is omitted by the caller. - ``kw_only_default`` indicates whether the ``kw_only`` parameter is assumed to be True or False if it is omitted by the caller. + - ``frozen_default`` indicates whether the ``frozen`` parameter is + assumed to be True or False if it is omitted by the caller. - ``field_specifiers`` specifies a static list of supported classes or functions that describe fields, similar to ``dataclasses.field()``. - Arbitrary other keyword arguments are accepted in order to allow for @@ -3432,6 +3435,7 @@ def decorator(cls_or_fn): "eq_default": eq_default, "order_default": order_default, "kw_only_default": kw_only_default, + "frozen_default": frozen_default, "field_specifiers": field_specifiers, "kwargs": kwargs, } diff --git a/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst b/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst new file mode 100644 index 000000000000..4fd7b6b85cef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst @@ -0,0 +1 @@ +Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. From webhook-mailer at python.org Tue Dec 6 00:46:58 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 06 Dec 2022 05:46:58 -0000 Subject: [Python-checkins] [Enum] Fix typos in the documentation (GH-99960) Message-ID: https://github.com/python/cpython/commit/05031ee1f53581abca14c39585ed8d25196383eb commit: 05031ee1f53581abca14c39585ed8d25196383eb branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-05T21:46:48-08:00 summary: [Enum] Fix typos in the documentation (GH-99960) (cherry picked from commit 2ae894b6d1995a3b9f95f4a82eec6dedd3ba5298) Co-authored-by: G?ry Ogam files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 99e9a193a60c..be0686203bd1 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -201,7 +201,7 @@ Data Types .. method:: EnumType.__getitem__(cls, name) - Returns the Enum member in *cls* matching *name*, or raises an :exc:`KeyError`:: + Returns the Enum member in *cls* matching *name*, or raises a :exc:`KeyError`:: >>> Color['BLUE'] @@ -248,7 +248,7 @@ Data Types .. note:: Enum member values - Member values can be anything: :class:`int`, :class:`str`, etc.. If + Member values can be anything: :class:`int`, :class:`str`, etc. If the exact value is unimportant you may use :class:`auto` instances and an appropriate value will be chosen for you. See :class:`auto` for the details. @@ -262,7 +262,7 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. - .. method:: Enum.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: Enum.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -279,8 +279,8 @@ Data Types :module: The name of the module the new Enum is created in. :qualname: The actual location in the module where this Enum can be found. :type: A mix-in type for the new Enum. - :start: The first integer value for the Enum (used by :class:`auto`) - :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only) + :start: The first integer value for the Enum (used by :class:`auto`). + :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only). .. method:: Enum.__dir__(self) @@ -322,7 +322,7 @@ Data Types >>> PowersOfThree.SECOND.value 6 - .. method:: Enum.__init_subclass__(cls, \**kwds) + .. method:: Enum.__init_subclass__(cls, **kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -380,7 +380,7 @@ Data Types .. method:: Enum.__format__(self) Returns the string used for *format()* and *f-string* calls. By default, - returns :meth:`__str__` returns, but can be overridden:: + returns :meth:`__str__` return value, but can be overridden:: >>> class OtherStyle(Enum): ... ALTERNATE = auto() @@ -559,11 +559,11 @@ Data Types Using :class:`auto` with :class:`Flag` results in integers that are powers of two, starting with ``1``. - .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It + .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It is now:: - >>> Color(0) # doctest: +SKIP - + >>> Color(0) # doctest: +SKIP + .. class:: IntFlag @@ -607,7 +607,7 @@ Data Types *replacement of existing constants* use-case. :meth:`~object.__format__` was already :meth:`!int.__format__` for that same reason. - Inversion of a :class:`!IntFlag` now returns a positive value that is the + Inversion of an :class:`!IntFlag` now returns a positive value that is the union of all flags not in the given flag, rather than a negative value. This matches the existing :class:`Flag` behavior. @@ -619,7 +619,7 @@ Data Types * :meth:`!int.__str__` for :class:`IntEnum` and :class:`IntFlag` * :meth:`!str.__str__` for :class:`StrEnum` - Inherit from :class:`!ReprEnum` to keep the :class:`str() / :func:`format` + Inherit from :class:`!ReprEnum` to keep the :class:`str() ` / :func:`format` of the mixed-in data type instead of using the :class:`Enum`-default :meth:`str() `. @@ -665,7 +665,7 @@ Data Types .. attribute:: NAMED_FLAGS Ensure that any flag groups/masks contain only named flags -- useful when - values are specified instead of being generated by :func:`auto` + values are specified instead of being generated by :func:`auto`:: >>> from enum import Flag, verify, NAMED_FLAGS >>> @verify(NAMED_FLAGS) @@ -897,23 +897,23 @@ Notes :class:`IntEnum`, :class:`StrEnum`, and :class:`IntFlag` - These three enum types are designed to be drop-in replacements for existing - integer- and string-based values; as such, they have extra limitations: + These three enum types are designed to be drop-in replacements for existing + integer- and string-based values; as such, they have extra limitations: - - ``__str__`` uses the value and not the name of the enum member + - ``__str__`` uses the value and not the name of the enum member - - ``__format__``, because it uses ``__str__``, will also use the value of - the enum member instead of its name + - ``__format__``, because it uses ``__str__``, will also use the value of + the enum member instead of its name - If you do not need/want those limitations, you can either create your own - base class by mixing in the ``int`` or ``str`` type yourself:: + If you do not need/want those limitations, you can either create your own + base class by mixing in the ``int`` or ``str`` type yourself:: - >>> from enum import Enum - >>> class MyIntEnum(int, Enum): - ... pass + >>> from enum import Enum + >>> class MyIntEnum(int, Enum): + ... pass or you can reassign the appropriate :meth:`str`, etc., in your enum:: - >>> from enum import IntEnum - >>> class MyIntEnum(IntEnum): - ... __str__ = IntEnum.__str__ + >>> from enum import IntEnum + >>> class MyIntEnum(IntEnum): + ... __str__ = IntEnum.__str__ From webhook-mailer at python.org Tue Dec 6 05:20:29 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 10:20:29 -0000 Subject: [Python-checkins] [3.8] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) (#100033) Message-ID: https://github.com/python/cpython/commit/e43393aaacc9350116a5402e1ae5dde45daa6dfe commit: e43393aaacc9350116a5402e1ae5dde45daa6dfe branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2022-12-06T11:20:22+01:00 summary: [3.8] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) (#100033) * gh-100001: Omit control characters in http.server stderr logs. (GH-100002) Replace control characters in http.server.BaseHTTPRequestHandler.log_message with an escaped \xHH sequence to avoid causing problems for the terminal the output is printed to. (cherry picked from commit d8ab0a4dfa48f881b4ac9ab857d2e9de42f72828) Co-authored-by: Gregory P. Smith * also escape \s (backport of PR #100038). * add versionadded and remove extraneous 'to' Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Doc/library/http.server.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 0ba509181acd..7b548ecf7336 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -488,3 +488,12 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow remote clients connecting to your +server to send nefarious control codes to your terminal. + +.. versionadded:: 3.8.16 + scrubbing control characters from log messages diff --git a/Lib/http/server.py b/Lib/http/server.py index 39de35458c38..ac04543827e6 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -565,6 +566,11 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' + def log_message(self, format, *args): """Log an arbitrary message. @@ -580,12 +586,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index fb026188f0b4..5a49b73c2f84 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -25,7 +25,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -910,6 +910,25 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst new file mode 100644 index 000000000000..a396e95cd83f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst @@ -0,0 +1,6 @@ +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` +``.log_message`` method to replace control characters with a ``\xHH`` hex escape +before printing. From webhook-mailer at python.org Tue Dec 6 05:22:18 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 10:22:18 -0000 Subject: [Python-checkins] [3.9] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) (#100032) Message-ID: https://github.com/python/cpython/commit/3b81c13ac3e0acfdbfb0d916a4ea627c9a546049 commit: 3b81c13ac3e0acfdbfb0d916a4ea627c9a546049 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2022-12-06T11:22:12+01:00 summary: [3.9] gh-100001: Omit control characters in http.server stderr logs. (GH-100002) (#100032) * gh-100001: Omit control characters in http.server stderr logs. (GH-100002) Replace control characters in http.server.BaseHTTPRequestHandler.log_message with an escaped \xHH sequence to avoid causing problems for the terminal the output is printed to. (cherry picked from commit d8ab0a4dfa48f881b4ac9ab857d2e9de42f72828) Co-authored-by: Gregory P. Smith * also escape \s (backport of PR #100038). * add versionadded and remove extra 'to' Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Doc/library/http.server.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 4aa10e26f619..94647e99724b 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -499,3 +499,12 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow remote clients connecting to your +server to send nefarious control codes to your terminal. + +.. versionadded:: 3.9.16 + scrubbing control characters from log messages diff --git a/Lib/http/server.py b/Lib/http/server.py index 6bf9084341a6..cf8933c3dbd9 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -563,6 +564,11 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' + def log_message(self, format, *args): """Log an arbitrary message. @@ -578,12 +584,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 4acf7a6fea44..db9ee29e5fbe 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -26,7 +26,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -982,6 +982,25 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst new file mode 100644 index 000000000000..a396e95cd83f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst @@ -0,0 +1,6 @@ +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` +``.log_message`` method to replace control characters with a ``\xHH`` hex escape +before printing. From webhook-mailer at python.org Tue Dec 6 05:34:40 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 10:34:40 -0000 Subject: [Python-checkins] [3.11] gh-100001: Remove doc typo, add versionadded (GH-100042) (#100043) Message-ID: https://github.com/python/cpython/commit/235f5fd2ca4c6acb4b04efeaaa1ecb46d41d5a6d commit: 235f5fd2ca4c6acb4b04efeaaa1ecb46d41d5a6d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2022-12-06T11:34:20+01:00 summary: [3.11] gh-100001: Remove doc typo, add versionadded (GH-100042) (#100043) (cherry picked from commit bed15f87eadc726122185cf41efcdda289f4a7b1) Co-authored-by: Gregory P. Smith files: M Doc/library/http.server.rst diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 154f3f3c3904..6a5644007081 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -516,6 +516,8 @@ to be served. Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` -implementation. This could allow to remote clients connecting to your +implementation. This could allow remote clients connecting to your server to send nefarious control codes to your terminal. +.. versionadded:: 3.11.1 + Control characters are scrubbed in stderr logs. From webhook-mailer at python.org Tue Dec 6 05:34:55 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 10:34:55 -0000 Subject: [Python-checkins] [3.10] gh-100001: Remove doc typo, add versionadded (GH-100042) (#100044) Message-ID: https://github.com/python/cpython/commit/b7ae1d22857c141ea4b0cc1f107e128596d02a94 commit: b7ae1d22857c141ea4b0cc1f107e128596d02a94 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2022-12-06T11:34:49+01:00 summary: [3.10] gh-100001: Remove doc typo, add versionadded (GH-100042) (#100044) (cherry picked from commit bed15f87eadc726122185cf41efcdda289f4a7b1) Co-authored-by: Gregory P. Smith files: M Doc/library/http.server.rst diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index fd6d7cb4ee1c..8bd22eac831a 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -503,6 +503,8 @@ to be served. Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` -implementation. This could allow to remote clients connecting to your +implementation. This could allow remote clients connecting to your server to send nefarious control codes to your terminal. +.. versionadded:: 3.10.9 + Control characters are scrubbed in stderr logs. From webhook-mailer at python.org Tue Dec 6 06:15:27 2022 From: webhook-mailer at python.org (markshannon) Date: Tue, 06 Dec 2022 11:15:27 -0000 Subject: [Python-checkins] GH-100026: Include the number of raw input files in summarize_stats.py (GH-100027) Message-ID: https://github.com/python/cpython/commit/9dc787ea96916552695e79397588fdfa68f22024 commit: 9dc787ea96916552695e79397588fdfa68f22024 branch: main author: Michael Droettboom committer: markshannon date: 2022-12-06T11:14:47Z summary: GH-100026: Include the number of raw input files in summarize_stats.py (GH-100027) files: M Tools/scripts/summarize_stats.py diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9c098064fe54..c15501bdc761 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -184,6 +184,7 @@ def gather_stats(input): key = key.strip() value = int(value) stats[key] += value + stats['__nfiles__'] += 1 return stats else: raise ValueError(f"{input:r} is not a file or directory path") @@ -561,6 +562,9 @@ def output_single_stats(stats): emit_specialization_overview(opcode_stats, total) emit_call_stats(stats) emit_object_stats(stats) + with Section("Meta stats", summary="Meta statistics"): + emit_table(("", "Count:"), [('Number of data files', stats['__nfiles__'])]) + def output_comparative_stats(base_stats, head_stats): base_opcode_stats = extract_opcode_stats(base_stats) From webhook-mailer at python.org Tue Dec 6 08:36:11 2022 From: webhook-mailer at python.org (vstinner) Date: Tue, 06 Dec 2022 13:36:11 -0000 Subject: [Python-checkins] gh-100008: VS 2017 is required since Python 3.11 (#100045) Message-ID: https://github.com/python/cpython/commit/5837e5f3478a6f3afb64502edc757f312c2db507 commit: 5837e5f3478a6f3afb64502edc757f312c2db507 branch: main author: Victor Stinner committer: vstinner date: 2022-12-06T14:35:32+01:00 summary: gh-100008: VS 2017 is required since Python 3.11 (#100045) files: M Doc/using/configure.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 3df6ff4873b7..db4bf7412292 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -24,6 +24,7 @@ Features required to build CPython: .. versionchanged:: 3.11 C11 compiler, IEEE 754 and NaN support are now required. + On Windows, Visual Studio 2017 or later is required. .. versionchanged:: 3.10 OpenSSL 1.1.1 is now required. From webhook-mailer at python.org Tue Dec 6 08:37:47 2022 From: webhook-mailer at python.org (ewdurbin) Date: Tue, 06 Dec 2022 13:37:47 -0000 Subject: [Python-checkins] bpo-37860: re-add netlify.toml to set up deploy previews for docs (#92852) Message-ID: https://github.com/python/cpython/commit/85d5a7e8ef472a4a64e5de883cf313c111a8ec77 commit: 85d5a7e8ef472a4a64e5de883cf313c111a8ec77 branch: main author: Ashwin Ramaswami committer: ewdurbin date: 2022-12-06T08:37:41-05:00 summary: bpo-37860: re-add netlify.toml to set up deploy previews for docs (#92852) * Revert "bpo-46184: remove `netlify.toml` (#30272)" This reverts commit fbaf2e604cd354f1ebc6be029480010c6715a8ca. * Delete runtime.txt * Create runtime.txt * Delete runtime.txt * Update netlify.toml * Update netlify.toml * Add netlify badge * Update Doc/tools/templates/layout.html Co-authored-by: Hugo van Kemenade * Update layout.html Co-authored-by: Hugo van Kemenade files: A netlify.toml M Doc/conf.py M Doc/tools/templates/layout.html diff --git a/Doc/conf.py b/Doc/conf.py index c7d2f43b7a8e..b3da8fa9ec44 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -105,6 +105,14 @@ # Short title used e.g. for HTML tags. html_short_title = '%s Documentation' % release +# Deployment preview information, from Netlify +# (See netlify.toml and https://docs.netlify.com/configure-builds/environment-variables/#git-metadata) +html_context = { + "is_deployment_preview": os.getenv("IS_DEPLOYMENT_PREVIEW"), + "repository_url": os.getenv("REPOSITORY_URL"), + "pr_id": os.getenv("REVIEW_ID") +} + # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. html_last_updated_fmt = '%b %d, %Y' diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 98ccf4224804..460161cd3202 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -8,6 +8,19 @@ <a href="/3/{{ pagename }}{{ file_suffix }}">{% trans %} Python documentation for the current stable release{% endtrans %}</a>. </div> {%- endif %} + +{%- if is_deployment_preview %} +<div id="deployment-preview-warning" style="padding: .5em; text-align: center; background-color: #fff2ba; color: #6a580e;"> + <div style="float: right; margin-top: -10px; margin-left: 10px;"> + <a href="https://www.netlify.com"> + <img src="https://www.netlify.com/img/global/badges/netlify-color-accent.svg" alt="Deploys by Netlify" /> + </a> + </div> + {% trans %}This is a deploy preview created from a <a href="{{ repository_url }}/pull/{{ pr_id }}">pull request</a>. + For authoritative documentation, see the {% endtrans %} + <a href="https://docs.python.org/3/{{ pagename }}{{ file_suffix }}">{% trans %} the current stable release{% endtrans %}</a>. +</div> +{%- endif %} {% endblock %} {% block rootrellink %} diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 000000000000..52675b3d701e --- /dev/null +++ b/netlify.toml @@ -0,0 +1,8 @@ +[build] + base = "Doc/" + command = "make html" + publish = "build/html" + +[build.environment] + PYTHON_VERSION = "3.8" + IS_DEPLOYMENT_PREVIEW = "true" \ No newline at end of file From webhook-mailer at python.org Tue Dec 6 09:01:50 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 14:01:50 -0000 Subject: [Python-checkins] GH-99729: Unlink frames before clearing them (GH-100030) Message-ID: <mailman.2823.1670335311.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b72014c783e5698beb18ee1249597e510b8bcb5a commit: b72014c783e5698beb18ee1249597e510b8bcb5a branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T14:01:38Z summary: GH-99729: Unlink frames before clearing them (GH-100030) files: A Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst M Lib/test/test_frame.py M Python/bytecodes.c M Python/ceval.c M Python/frame.c M Python/generated_cases.c.h diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index a7db22007ded..ed413f105e5b 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -2,6 +2,7 @@ import re import sys import textwrap +import threading import types import unittest import weakref @@ -11,6 +12,7 @@ _testcapi = None from test import support +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -329,6 +331,46 @@ def f(): if old_enabled: gc.enable() + @support.cpython_only + @threading_helper.requires_working_threading() + def test_sneaky_frame_object_teardown(self): + + class SneakyDel: + def __del__(self): + """ + Stash a reference to the entire stack for walking later. + + It may look crazy, but you'd be surprised how common this is + when using a test runner (like pytest). The typical recipe is: + ResourceWarning + -Werror + a custom sys.unraisablehook. + """ + nonlocal sneaky_frame_object + sneaky_frame_object = sys._getframe() + + class SneakyThread(threading.Thread): + """ + A separate thread isn't needed to make this code crash, but it does + make crashes more consistent, since it means sneaky_frame_object is + backed by freed memory after the thread completes! + """ + + def run(self): + """Run SneakyDel.__del__ as this frame is popped.""" + ref = SneakyDel() + + sneaky_frame_object = None + t = SneakyThread() + t.start() + t.join() + # sneaky_frame_object can be anything, really, but it's crucial that + # SneakyThread.run's frame isn't anywhere on the stack while it's being + # torn down: + self.assertIsNotNone(sneaky_frame_object) + while sneaky_frame_object is not None: + self.assertIsNot( + sneaky_frame_object.f_code, SneakyThread.run.__code__ + ) + sneaky_frame_object = sneaky_frame_object.f_back @unittest.skipIf(_testcapi is None, 'need _testcapi') class TestCAPI(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst new file mode 100644 index 000000000000..3fe21a8a21bf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst @@ -0,0 +1,3 @@ +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 41dd1acc937d..d0480ac01eb6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -619,7 +619,10 @@ dummy_func( DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; } diff --git a/Python/ceval.c b/Python/ceval.c index 80bfa21ad0b6..9e4179e56071 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1009,14 +1009,6 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject return 0; } -static _PyInterpreterFrame * -pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - _PyInterpreterFrame *prev_frame = frame->previous; - _PyEvalFrameClearAndPop(tstate, frame); - return prev_frame; -} - int _Py_CheckRecursiveCallPy( PyThreadState *tstate) @@ -1432,7 +1424,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); if (frame == &entry_frame) { /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; diff --git a/Python/frame.c b/Python/frame.c index 52f6ef428291..b1525cca5112 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -127,6 +127,9 @@ _PyFrame_Clear(_PyInterpreterFrame *frame) * to have cleared the enclosing generator, if any. */ assert(frame->owner != FRAME_OWNED_BY_GENERATOR || _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); + // GH-99729: Clearing this frame can expose the stack (via finalizers). It's + // crucial that this frame has been unlinked, and is no longer visible: + assert(_PyThreadState_GET()->cframe->current_frame != frame); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3a403824b499..0805386866b3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -628,7 +628,10 @@ DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; } From webhook-mailer at python.org Tue Dec 6 10:16:20 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 15:16:20 -0000 Subject: [Python-checkins] [3.11] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#99949) Message-ID: <mailman.2824.1670339781.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3fae04b10e2655a20a3aadb5e0d63e87206d0c67 commit: 3fae04b10e2655a20a3aadb5e0d63e87206d0c67 branch: 3.11 author: Serhiy Storchaka <storchaka at gmail.com> committer: ambv <lukasz at langa.pl> date: 2022-12-06T16:15:44+01:00 summary: [3.11] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#99949) It no longer emits a deprecation warning if the current event loop was set. Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst M Doc/library/asyncio-eventloop.rst M Doc/library/asyncio-llapi-index.rst M Doc/library/asyncio-policy.rst M Lib/asyncio/events.py M Lib/test/test_asyncio/test_base_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_futures.py M Lib/test/test_asyncio/test_streams.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_asyncio/test_unix_events.py M Lib/test/test_coroutines.py M Modules/_asynciomodule.c diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index a10658df5df8..28b7a900583b 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -43,10 +43,12 @@ an event loop: Get the current event loop. - If there is no current event loop set in the current OS thread, - the OS thread is main, and :func:`set_event_loop` has not yet - been called, asyncio will create a new event loop and set it as the - current one. + When called from a coroutine or a callback (e.g. scheduled with + call_soon or similar API), this function will always return the + running event loop. + + If there is no running event loop set, the function will return + the result of ``get_event_loop_policy().get_event_loop()`` call. Because this function has rather complex behavior (especially when custom event loop policies are in use), using the @@ -58,10 +60,14 @@ an event loop: event loop. .. deprecated:: 3.10 - Emits a deprecation warning if there is no running event loop. - In future Python releases, this function may become an alias of - :func:`get_running_loop` and will accordingly raise a - :exc:`RuntimeError` if there is no running event loop. + Deprecation warning is emitted if there is no current event loop. + In Python 3.12 it will be an error. + + .. note:: + In Python versions 3.10.0--3.10.8 and 3.11.0 this function + (and other functions which used it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if + the current loop was set. .. function:: set_event_loop(loop) diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index b7ad888a7b67..9ce48a24444e 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,7 +19,7 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (current or via the policy). + - Get an event loop instance (running or current via the current policy). * - :func:`asyncio.set_event_loop` - Set the event loop as current via the current policy. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index bfc3e3090fdc..d0af45febd14 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -112,6 +112,11 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. + .. deprecated:: 3.11.1 + :meth:`get_event_loop` now emits a :exc:`DeprecationWarning` if there + is no current event loop set and a new event loop has been implicitly + created. In Python 3.12 it will be an error. + .. class:: WindowsSelectorEventLoopPolicy diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0d26ea545baa..af3f9e970b51 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -671,6 +671,21 @@ def get_event_loop(self): if (self._local._loop is None and not self._local._set_called and threading.current_thread() is threading.main_thread()): + stacklevel = 2 + try: + f = sys._getframe(1) + except AttributeError: + pass + else: + while f: + module = f.f_globals.get('__name__') + if not (module == 'asyncio' or module.startswith('asyncio.')): + break + f = f.f_back + stacklevel += 1 + import warnings + warnings.warn('There is no current event loop', + DeprecationWarning, stacklevel=stacklevel) self.set_event_loop(self.new_event_loop()) if self._local._loop is None: @@ -786,12 +801,13 @@ def get_event_loop(): def _get_event_loop(stacklevel=3): + # This internal method is going away in Python 3.12, left here only for + # backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. + # Similarly, this method's C equivalent in _asyncio is going away as well. + # See GH-99949 for more details. current_loop = _get_running_loop() if current_loop is not None: return current_loop - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 063174adacfa..6ba602dd619f 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -746,7 +746,7 @@ async def coro(): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio', - 'loop = asyncio.get_event_loop()', + 'loop = asyncio.new_event_loop()', 'print(loop.get_debug())')) # Test with -E to not fail if the unit test was run with diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 05d9107b28e2..c431fea40163 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2547,8 +2547,9 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.assertEqual(cm.filename, __file__) self.assertIsInstance(loop, asyncio.AbstractEventLoop) self.assertIs(policy._local._loop, loop) @@ -2562,7 +2563,10 @@ def test_get_event_loop_calls_set_event_loop(self): policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.addCleanup(loop.close) + self.assertEqual(cm.filename, __file__) # policy._local._loop must be set through .set_event_loop() # (the unix DefaultEventLoopPolicy needs this call to attach @@ -2596,7 +2600,8 @@ def test_new_event_loop(self): def test_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() - old_loop = policy.get_event_loop() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) self.assertRaises(TypeError, policy.set_event_loop, object()) @@ -2709,15 +2714,11 @@ def get_event_loop(self): asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2731,16 +2732,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) - + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) @@ -2764,10 +2760,8 @@ def test_get_event_loop_returns_running_loop2(self): self.addCleanup(loop2.close) self.assertEqual(cm.filename, __file__) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2781,15 +2775,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - self.assertIs(asyncio.get_event_loop(), loop) - self.assertEqual(cm.filename, __file__) + self.assertIs(asyncio.get_event_loop(), loop) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index f8fe2e76b622..987772e0b196 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -145,10 +145,8 @@ def test_initial_state(self): self.assertTrue(f.cancelled()) def test_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - self._new_future() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() def test_constructor_use_running_loop(self): async def test(): @@ -158,12 +156,10 @@ async def test(): self.assertIs(f.get_loop(), self.loop) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - f = self._new_future() - self.assertEqual(cm.filename, __file__) + f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -499,10 +495,8 @@ def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) ex.shutdown(wait=True) def test_wrap_future_use_running_loop(self): @@ -517,16 +511,14 @@ async def test(): ex.shutdown(wait=True) def test_wrap_future_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - f2 = asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) ex.shutdown(wait=True) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 0c49099bc499..95fc7a159bae 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -810,10 +810,8 @@ def test_read_all_from_pipe_reader(self): self.assertEqual(data, b'data') def test_streamreader_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() def test_streamreader_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -827,21 +825,17 @@ async def test(): def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) - with self.assertWarns(DeprecationWarning) as cm: - reader = asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) def test_streamreaderprotocol_constructor_without_loop(self): reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) def test_streamreaderprotocol_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -855,13 +849,11 @@ async def test(): def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - protocol = asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) def test_multiple_drain(self): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 6b875cfb4dcb..9900e307a477 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -200,10 +200,8 @@ async def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -213,12 +211,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -1536,10 +1532,8 @@ async def coro(): self.addCleanup(a.close) futs = asyncio.as_completed([a]) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - list(futs) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + list(futs) def test_as_completed_coroutine_use_running_loop(self): loop = self.new_test_loop() @@ -1969,10 +1963,8 @@ async def coro(): inner = coro() self.addCleanup(inner.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.shield(inner) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) def test_shield_coroutine_use_running_loop(self): async def coro(): @@ -1986,15 +1978,13 @@ async def test(): self.assertEqual(res, 42) def test_shield_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 async def coro(): return 42 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - outer = asyncio.shield(coro()) - self.assertEqual(cm.filename, __file__) + outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) @@ -2820,7 +2810,7 @@ def test_current_task_no_running_loop(self): self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): asyncio.current_task() def test_current_task_with_implicit_loop(self): @@ -2984,10 +2974,8 @@ def _gather(self, *args, **kwargs): return asyncio.gather(*args, **kwargs) def test_constructor_empty_sequence_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() def test_constructor_empty_sequence_use_running_loop(self): async def gather(): @@ -3000,12 +2988,10 @@ async def gather(): self.assertEqual(fut.result(), []) def test_constructor_empty_sequence_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather() - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) @@ -3093,10 +3079,8 @@ async def coro(): self.addCleanup(gen1.close) gen2 = coro() self.addCleanup(gen2.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) def test_constructor_use_running_loop(self): async def coro(): @@ -3110,16 +3094,14 @@ async def gather(): self.one_loop.run_until_complete(fut) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 async def coro(): return 'abc' asyncio.set_event_loop(self.other_loop) self.addCleanup(asyncio.set_event_loop, None) gen1 = coro() gen2 = coro() - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather(gen1, gen2) self.assertIs(fut._loop, self.other_loop) self.other_loop.run_until_complete(fut) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 1d922783ce3b..01c1214c7f7c 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1740,7 +1740,8 @@ def f(): def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() - loop = policy.get_event_loop() + loop = policy.new_event_loop() + policy.set_event_loop(loop) # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index a15736e9f945..10f1a9efbcbd 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2411,7 +2411,8 @@ class UnawaitedWarningDuringShutdownTest(unittest.TestCase): def test_unawaited_warning_during_shutdown(self): code = ("import asyncio\n" "async def f(): pass\n" - "asyncio.gather(f())\n") + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") assert_python_ok("-c", code) code = ("import sys\n" diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst new file mode 100644 index 000000000000..dd4f43351ac7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst @@ -0,0 +1,3 @@ +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index d503018ea6c3..40f1f80be447 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -339,13 +339,6 @@ get_event_loop(int stacklevel) return loop; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "There is no current event loop", - stacklevel)) - { - return NULL; - } - policy = PyObject_CallNoArgs(asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; @@ -3129,6 +3122,11 @@ _asyncio_get_event_loop_impl(PyObject *module) return get_event_loop(1); } +// This internal method is going away in Python 3.12, left here only for +// backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. +// Similarly, this method's Python equivalent in asyncio.events is going +// away as well. +// See GH-99949 for more details. /*[clinic input] _asyncio._get_event_loop stacklevel: int = 3 From webhook-mailer at python.org Tue Dec 6 12:02:31 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 17:02:31 -0000 Subject: [Python-checkins] [3.11] GH-99729: Unlink frames before clearing them (#100047) Message-ID: <mailman.2825.1670346152.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2182a71eedbc8e95c3cf2d8c0aa2fd66c7a93db4 commit: 2182a71eedbc8e95c3cf2d8c0aa2fd66c7a93db4 branch: 3.11 author: Brandt Bucher <brandtbucher at microsoft.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T17:02:19Z summary: [3.11] GH-99729: Unlink frames before clearing them (#100047) files: A Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst M Lib/test/test_frame.py M Python/ceval.c M Python/frame.c diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 4b86a60d2f4c..9cb2686f1759 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -2,11 +2,13 @@ import re import sys import textwrap +import threading import types import unittest import weakref from test import support +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -325,6 +327,46 @@ def f(): if old_enabled: gc.enable() + @support.cpython_only + @threading_helper.requires_working_threading() + def test_sneaky_frame_object_teardown(self): + + class SneakyDel: + def __del__(self): + """ + Stash a reference to the entire stack for walking later. + + It may look crazy, but you'd be surprised how common this is + when using a test runner (like pytest). The typical recipe is: + ResourceWarning + -Werror + a custom sys.unraisablehook. + """ + nonlocal sneaky_frame_object + sneaky_frame_object = sys._getframe() + + class SneakyThread(threading.Thread): + """ + A separate thread isn't needed to make this code crash, but it does + make crashes more consistent, since it means sneaky_frame_object is + backed by freed memory after the thread completes! + """ + + def run(self): + """Run SneakyDel.__del__ as this frame is popped.""" + ref = SneakyDel() + + sneaky_frame_object = None + t = SneakyThread() + t.start() + t.join() + # sneaky_frame_object can be anything, really, but it's crucial that + # SneakyThread.run's frame isn't anywhere on the stack while it's being + # torn down: + self.assertIsNotNone(sneaky_frame_object) + while sneaky_frame_object is not None: + self.assertIsNot( + sneaky_frame_object.f_code, SneakyThread.run.__code__ + ) + sneaky_frame_object = sneaky_frame_object.f_back if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst new file mode 100644 index 000000000000..3fe21a8a21bf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst @@ -0,0 +1,3 @@ +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. diff --git a/Python/ceval.c b/Python/ceval.c index 8cbe838ddf2b..a34e4ffb72f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1617,14 +1617,6 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject return 0; } -static _PyInterpreterFrame * -pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - _PyInterpreterFrame *prev_frame = frame->previous; - _PyEvalFrameClearAndPop(tstate, frame); - return prev_frame; -} - /* It is only between the PRECALL instruction and the following CALL, * that this has any meaning. */ @@ -2441,7 +2433,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallTstate(tstate); if (!frame->is_entry) { - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; } @@ -5833,7 +5828,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(tstate->cframe->current_frame == frame->previous); return NULL; } - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); resume_with_error: SET_LOCALS_FROM_FRAME(); diff --git a/Python/frame.c b/Python/frame.c index d8f2f801f38c..3ea3a2ced40d 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -123,6 +123,9 @@ _PyFrame_Clear(_PyInterpreterFrame *frame) * to have cleared the enclosing generator, if any. */ assert(frame->owner != FRAME_OWNED_BY_GENERATOR || _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); + // GH-99729: Clearing this frame can expose the stack (via finalizers). It's + // crucial that this frame has been unlinked, and is no longer visible: + assert(_PyThreadState_GET()->cframe->current_frame != frame); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; From webhook-mailer at python.org Tue Dec 6 12:40:59 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 17:40:59 -0000 Subject: [Python-checkins] [3.10] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#100059) Message-ID: <mailman.2826.1670348461.3313.python-checkins@python.org> https://github.com/python/cpython/commit/300d812fd1c4d9244e71de0d228cc72439d312a7 commit: 300d812fd1c4d9244e71de0d228cc72439d312a7 branch: 3.10 author: ?ukasz Langa <lukasz at langa.pl> committer: ambv <lukasz at langa.pl> date: 2022-12-06T18:40:30+01:00 summary: [3.10] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#100059) It no longer emits a deprecation warning if the current event loop was set. (cherry picked from commit 3fae04b10e2655a20a3aadb5e0d63e87206d0c67) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst M Doc/library/asyncio-eventloop.rst M Doc/library/asyncio-llapi-index.rst M Doc/library/asyncio-policy.rst M Lib/asyncio/events.py M Lib/test/test_asyncio/test_base_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_futures.py M Lib/test/test_asyncio/test_streams.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_asyncio/test_unix_events.py M Lib/test/test_coroutines.py M Modules/_asynciomodule.c diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 86ca5d0f2ab5..a23be64ec39a 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -43,10 +43,12 @@ an event loop: Get the current event loop. - If there is no current event loop set in the current OS thread, - the OS thread is main, and :func:`set_event_loop` has not yet - been called, asyncio will create a new event loop and set it as the - current one. + When called from a coroutine or a callback (e.g. scheduled with + call_soon or similar API), this function will always return the + running event loop. + + If there is no running event loop set, the function will return + the result of ``get_event_loop_policy().get_event_loop()`` call. Because this function has rather complex behavior (especially when custom event loop policies are in use), using the @@ -58,10 +60,14 @@ an event loop: event loop. .. deprecated:: 3.10 - Emits a deprecation warning if there is no running event loop. - In future Python releases, this function may become an alias of - :func:`get_running_loop` and will accordingly raise a - :exc:`RuntimeError` if there is no running event loop. + Deprecation warning is emitted if there is no current event loop. + In Python 3.12 it will be an error. + + .. note:: + In Python versions 3.10.0--3.10.8 this function + (and other functions which used it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if + the current loop was set. .. function:: set_event_loop(loop) diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index 63ab1483e1fb..d903ea0dc247 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,7 +19,7 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (current or via the policy). + - Get an event loop instance (running or current via the current policy). * - :func:`asyncio.set_event_loop` - Set the event loop as current via the current policy. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index bfc3e3090fdc..a4cd5aa992af 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -112,6 +112,11 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. + .. deprecated:: 3.10.9 + :meth:`get_event_loop` now emits a :exc:`DeprecationWarning` if there + is no current event loop set and a new event loop has been implicitly + created. In Python 3.12 it will be an error. + .. class:: WindowsSelectorEventLoopPolicy diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 5ab1acc41bf3..faac53769412 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -650,6 +650,21 @@ def get_event_loop(self): if (self._local._loop is None and not self._local._set_called and threading.current_thread() is threading.main_thread()): + stacklevel = 2 + try: + f = sys._getframe(1) + except AttributeError: + pass + else: + while f: + module = f.f_globals.get('__name__') + if not (module == 'asyncio' or module.startswith('asyncio.')): + break + f = f.f_back + stacklevel += 1 + import warnings + warnings.warn('There is no current event loop', + DeprecationWarning, stacklevel=stacklevel) self.set_event_loop(self.new_event_loop()) if self._local._loop is None: @@ -763,12 +778,13 @@ def get_event_loop(): def _get_event_loop(stacklevel=3): + # This internal method is going away in Python 3.12, left here only for + # backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. + # Similarly, this method's C equivalent in _asyncio is going away as well. + # See GH-99949 for more details. current_loop = _get_running_loop() if current_loop is not None: return current_loop - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index d77bf95a7b37..7f7e371db1b4 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -752,7 +752,7 @@ async def coro(): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio', - 'loop = asyncio.get_event_loop()', + 'loop = asyncio.new_event_loop()', 'print(loop.get_debug())')) # Test with -E to not fail if the unit test was run with diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index bed7b5d30b85..92c69de742ba 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2561,8 +2561,9 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.assertEqual(cm.filename, __file__) self.assertIsInstance(loop, asyncio.AbstractEventLoop) self.assertIs(policy._local._loop, loop) @@ -2576,7 +2577,10 @@ def test_get_event_loop_calls_set_event_loop(self): policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.addCleanup(loop.close) + self.assertEqual(cm.filename, __file__) # policy._local._loop must be set through .set_event_loop() # (the unix DefaultEventLoopPolicy needs this call to attach @@ -2610,7 +2614,8 @@ def test_new_event_loop(self): def test_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() - old_loop = policy.get_event_loop() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) self.assertRaises(AssertionError, policy.set_event_loop, object()) @@ -2723,15 +2728,11 @@ def get_event_loop(self): asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2745,16 +2746,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) - + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) @@ -2778,10 +2774,8 @@ def test_get_event_loop_returns_running_loop2(self): self.addCleanup(loop2.close) self.assertEqual(cm.warnings[0].filename, __file__) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2795,15 +2789,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - self.assertIs(asyncio.get_event_loop(), loop) - self.assertEqual(cm.warnings[0].filename, __file__) + self.assertIs(asyncio.get_event_loop(), loop) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 012674260370..0b47aa28df66 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -145,10 +145,8 @@ def test_initial_state(self): self.assertTrue(f.cancelled()) def test_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - self._new_future() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() def test_constructor_use_running_loop(self): async def test(): @@ -158,12 +156,10 @@ async def test(): self.assertIs(f.get_loop(), self.loop) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - f = self._new_future() - self.assertEqual(cm.warnings[0].filename, __file__) + f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -499,10 +495,8 @@ def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.wrap_future(f1) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) ex.shutdown(wait=True) def test_wrap_future_use_running_loop(self): @@ -517,16 +511,14 @@ async def test(): ex.shutdown(wait=True) def test_wrap_future_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - f2 = asyncio.wrap_future(f1) - self.assertEqual(cm.warnings[0].filename, __file__) + f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) ex.shutdown(wait=True) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 44fcd6589c80..994041c10f0a 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -747,10 +747,8 @@ def test_read_all_from_pipe_reader(self): self.assertEqual(data, b'data') def test_streamreader_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReader() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() def test_streamreader_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -764,21 +762,17 @@ async def test(): def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) - with self.assertWarns(DeprecationWarning) as cm: - reader = asyncio.StreamReader() - self.assertEqual(cm.warnings[0].filename, __file__) + reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) def test_streamreaderprotocol_constructor_without_loop(self): reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) def test_streamreaderprotocol_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -792,13 +786,11 @@ async def test(): def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - protocol = asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.warnings[0].filename, __file__) + protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) def test_multiple_drain(self): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 05a822ba458b..b08068e6173e 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -210,10 +210,8 @@ async def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -223,12 +221,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.warnings[0].filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -247,10 +243,8 @@ def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -260,12 +254,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.warnings[0].filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -1488,10 +1480,8 @@ async def coro(): self.addCleanup(a.close) futs = asyncio.as_completed([a]) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - list(futs) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + list(futs) def test_as_completed_coroutine_use_running_loop(self): loop = self.new_test_loop() @@ -1507,17 +1497,14 @@ async def test(): loop.run_until_complete(test()) def test_as_completed_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 async def coro(): return 42 loop = self.new_test_loop() asyncio.set_event_loop(loop) self.addCleanup(asyncio.set_event_loop, None) - futs = asyncio.as_completed([coro()]) - with self.assertWarns(DeprecationWarning) as cm: - futs = list(futs) - self.assertEqual(cm.warnings[0].filename, __file__) + futs = list(asyncio.as_completed([coro()])) self.assertEqual(len(futs), 1) self.assertEqual(loop.run_until_complete(futs[0]), 42) @@ -1987,10 +1974,8 @@ async def coro(): inner = coro() self.addCleanup(inner.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.shield(inner) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) def test_shield_coroutine_use_running_loop(self): async def coro(): @@ -2004,15 +1989,13 @@ async def test(): self.assertEqual(res, 42) def test_shield_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 async def coro(): return 42 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - outer = asyncio.shield(coro()) - self.assertEqual(cm.warnings[0].filename, __file__) + outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) @@ -2950,7 +2933,7 @@ def test_current_task_no_running_loop(self): self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): asyncio.current_task() def test_current_task_with_implicit_loop(self): @@ -3114,10 +3097,8 @@ def _gather(self, *args, **kwargs): return asyncio.gather(*args, **kwargs) def test_constructor_empty_sequence_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather() - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() def test_constructor_empty_sequence_use_running_loop(self): async def gather(): @@ -3130,12 +3111,10 @@ async def gather(): self.assertEqual(fut.result(), []) def test_constructor_empty_sequence_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather() - self.assertEqual(cm.warnings[0].filename, __file__) + fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) @@ -3223,10 +3202,8 @@ async def coro(): self.addCleanup(gen1.close) gen2 = coro() self.addCleanup(gen2.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather(gen1, gen2) - self.assertEqual(cm.warnings[0].filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) def test_constructor_use_running_loop(self): async def coro(): @@ -3240,16 +3217,14 @@ async def gather(): self.one_loop.run_until_complete(fut) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10.0, undeprecated in 3.10.9 async def coro(): return 'abc' asyncio.set_event_loop(self.other_loop) self.addCleanup(asyncio.set_event_loop, None) gen1 = coro() gen2 = coro() - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather(gen1, gen2) - self.assertEqual(cm.warnings[0].filename, __file__) + fut = asyncio.gather(gen1, gen2) self.assertIs(fut._loop, self.other_loop) self.other_loop.run_until_complete(fut) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 1d922783ce3b..01c1214c7f7c 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1740,7 +1740,8 @@ def f(): def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() - loop = policy.get_event_loop() + loop = policy.new_event_loop() + policy.set_event_loop(loop) # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index acff24537cf8..f4c526890718 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2319,7 +2319,8 @@ class UnawaitedWarningDuringShutdownTest(unittest.TestCase): def test_unawaited_warning_during_shutdown(self): code = ("import asyncio\n" "async def f(): pass\n" - "asyncio.gather(f())\n") + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") assert_python_ok("-c", code) code = ("import sys\n" diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst new file mode 100644 index 000000000000..dd4f43351ac7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst @@ -0,0 +1,3 @@ +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c627382a53f0..8d0ff690b538 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -332,13 +332,6 @@ get_event_loop(int stacklevel) return loop; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "There is no current event loop", - stacklevel)) - { - return NULL; - } - policy = PyObject_CallNoArgs(asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; @@ -3092,6 +3085,11 @@ _asyncio_get_event_loop_impl(PyObject *module) return get_event_loop(1); } +// This internal method is going away in Python 3.12, left here only for +// backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. +// Similarly, this method's Python equivalent in asyncio.events is going +// away as well. +// See GH-99949 for more details. /*[clinic input] _asyncio._get_event_loop stacklevel: int = 3 From webhook-mailer at python.org Tue Dec 6 12:42:20 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 17:42:20 -0000 Subject: [Python-checkins] gh-93453: No longer create an event loop in get_event_loop() (#98440) Message-ID: <mailman.2827.1670348540.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fd38a2f0ec03b4eec5e3cfd41241d198b1ee555a commit: fd38a2f0ec03b4eec5e3cfd41241d198b1ee555a branch: main author: Serhiy Storchaka <storchaka at gmail.com> committer: ambv <lukasz at langa.pl> date: 2022-12-06T18:42:12+01:00 summary: gh-93453: No longer create an event loop in get_event_loop() (#98440) asyncio.get_event_loop() now always return either running event loop or the result of get_event_loop_policy().get_event_loop() call. The latter should now raise an RuntimeError if no current event loop was set instead of creating and setting a new event loop. It affects also a number of asyncio functions and constructors which call get_event_loop() implicitly: ensure_future(), shield(), gather(), etc. DeprecationWarning is no longer emitted if there is no running event loop but the current event loop was set. Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst M Doc/library/asyncio-eventloop.rst M Doc/library/asyncio-llapi-index.rst M Doc/library/asyncio-policy.rst M Doc/whatsnew/3.12.rst M Lib/asyncio/events.py M Lib/asyncio/futures.py M Lib/asyncio/streams.py M Lib/asyncio/tasks.py M Lib/test/test_asyncio/test_base_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_futures.py M Lib/test/test_asyncio/test_streams.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_asyncio/test_unix_events.py M Lib/test/test_coroutines.py M Modules/_asynciomodule.c M Modules/clinic/_asynciomodule.c.h diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 0bcaed5477fa..fd47b0c24d8a 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -43,10 +43,12 @@ an event loop: Get the current event loop. - If there is no current event loop set in the current OS thread, - the OS thread is main, and :func:`set_event_loop` has not yet - been called, asyncio will create a new event loop and set it as the - current one. + When called from a coroutine or a callback (e.g. scheduled with + call_soon or similar API), this function will always return the + running event loop. + + If there is no running event loop set, the function will return + the result of calling ``get_event_loop_policy().get_event_loop()``. Because this function has rather complex behavior (especially when custom event loop policies are in use), using the @@ -57,11 +59,11 @@ an event loop: instead of using these lower level functions to manually create and close an event loop. - .. deprecated:: 3.10 - Emits a deprecation warning if there is no running event loop. - In future Python releases, this function may become an alias of - :func:`get_running_loop` and will accordingly raise a - :exc:`RuntimeError` if there is no running event loop. + .. note:: + In Python versions 3.10.0--3.10.8 and 3.11.0 this function + (and other functions which used it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if + the current loop was set. .. function:: set_event_loop(loop) diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index b7ad888a7b67..9ce48a24444e 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,7 +19,7 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (current or via the policy). + - Get an event loop instance (running or current via the current policy). * - :func:`asyncio.set_event_loop` - Set the event loop as current via the current policy. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 98c850158746..ccd952449475 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -116,6 +116,10 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. + .. versionchanged:: 3.12 + :meth:`get_event_loop` now raises a :exc:`RuntimeError` if there is no + current event loop set. + .. class:: WindowsSelectorEventLoopPolicy diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 6f5ce818d961..e9e25b9dc466 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -686,6 +686,18 @@ Changes in the Python API around process-global resources, which are best managed from the main interpreter. (Contributed by Dong-hee Na in :gh:`99127`.) +* :func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like + :func:`~asyncio.ensure_future`, :func:`~asyncio.shield` or + :func:`~asyncio.gather`, and also the + :meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of + :class:`~asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` + if called when there is no running event loop and the current event loop was + not set. + Previously they implicitly created and set a new current event loop. + :exc:`DeprecationWarning` is no longer emitted if there is no running + event loop but the current event loop is set in the policy. + (Contributed by Serhiy Storchaka in :gh:`93453`.) + Build Changes ============= diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 2836bbcc463f..34a8869dff8d 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -619,7 +619,7 @@ def get_event_loop(self): Returns an event loop object implementing the BaseEventLoop interface, or raises an exception in case no event loop has been set for the - current context and the current policy does not specify to create one. + current context. It should never return None.""" raise NotImplementedError @@ -672,11 +672,6 @@ def get_event_loop(self): Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) @@ -786,16 +781,9 @@ def get_event_loop(): the result of `get_event_loop_policy().get_event_loop()` call. """ # NOTE: this function is implemented in C (see _asynciomodule.c) - return _py__get_event_loop() - - -def _get_event_loop(stacklevel=3): current_loop = _get_running_loop() if current_loop is not None: return current_loop - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() @@ -825,7 +813,6 @@ def set_child_watcher(watcher): _py__set_running_loop = _set_running_loop _py_get_running_loop = get_running_loop _py_get_event_loop = get_event_loop -_py__get_event_loop = _get_event_loop try: @@ -833,7 +820,7 @@ def set_child_watcher(watcher): # functions in asyncio. Pure Python implementation is # about 4 times slower than C-accelerated. from _asyncio import (_get_running_loop, _set_running_loop, - get_running_loop, get_event_loop, _get_event_loop) + get_running_loop, get_event_loop) except ImportError: pass else: @@ -842,7 +829,6 @@ def set_child_watcher(watcher): _c__set_running_loop = _set_running_loop _c_get_running_loop = get_running_loop _c_get_event_loop = get_event_loop - _c__get_event_loop = _get_event_loop if hasattr(os, 'fork'): diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 3a6b44a09108..97fc4e3fcb60 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -77,7 +77,7 @@ def __init__(self, *, loop=None): the default event loop. """ if loop is None: - self._loop = events._get_event_loop() + self._loop = events.get_event_loop() else: self._loop = loop self._callbacks = [] @@ -413,7 +413,7 @@ def wrap_future(future, *, loop=None): assert isinstance(future, concurrent.futures.Future), \ f'concurrent.futures.Future is expected, got {future!r}' if loop is None: - loop = events._get_event_loop() + loop = events.get_event_loop() new_future = loop.create_future() _chain_future(future, new_future) return new_future diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index c4d837a11708..3bd99043d096 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -125,7 +125,7 @@ class FlowControlMixin(protocols.Protocol): def __init__(self, loop=None): if loop is None: - self._loop = events._get_event_loop(stacklevel=4) + self._loop = events.get_event_loop() else: self._loop = loop self._paused = False @@ -404,7 +404,7 @@ def __init__(self, limit=_DEFAULT_LIMIT, loop=None): self._limit = limit if loop is None: - self._loop = events._get_event_loop() + self._loop = events.get_event_loop() else: self._loop = loop self._buffer = bytearray() diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 571013745aa0..fa853283c0c5 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -582,7 +582,7 @@ def as_completed(fs, *, timeout=None): from .queues import Queue # Import here to avoid circular import problem. done = Queue() - loop = events._get_event_loop() + loop = events.get_event_loop() todo = {ensure_future(f, loop=loop) for f in set(fs)} timeout_handle = None @@ -668,7 +668,7 @@ def _ensure_future(coro_or_future, *, loop=None): 'is required') if loop is None: - loop = events._get_event_loop(stacklevel=4) + loop = events.get_event_loop() try: return loop.create_task(coro_or_future) except RuntimeError: @@ -749,7 +749,7 @@ def gather(*coros_or_futures, return_exceptions=False): gather won't cancel any other awaitables. """ if not coros_or_futures: - loop = events._get_event_loop() + loop = events.get_event_loop() outer = loop.create_future() outer.set_result([]) return outer diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 7421d18dc636..65dd4d42708b 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -746,7 +746,7 @@ async def coro(): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio', - 'loop = asyncio.get_event_loop()', + 'loop = asyncio.new_event_loop()', 'print(loop.get_debug())')) # Test with -E to not fail if the unit test was run with diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index cabe75f56d9f..153b2de81722 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2550,29 +2550,8 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - - loop = policy.get_event_loop() - self.assertIsInstance(loop, asyncio.AbstractEventLoop) - - self.assertIs(policy._local._loop, loop) - self.assertIs(loop, policy.get_event_loop()) - loop.close() - - def test_get_event_loop_calls_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() - - with mock.patch.object( - policy, "set_event_loop", - wraps=policy.set_event_loop) as m_set_event_loop: - - loop = policy.get_event_loop() - - # policy._local._loop must be set through .set_event_loop() - # (the unix DefaultEventLoopPolicy needs this call to attach - # the child watcher correctly) - m_set_event_loop.assert_called_with(loop) - - loop.close() + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + policy.get_event_loop() def test_get_event_loop_after_set_none(self): policy = asyncio.DefaultEventLoopPolicy() @@ -2599,7 +2578,8 @@ def test_new_event_loop(self): def test_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() - old_loop = policy.get_event_loop() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) self.assertRaises(TypeError, policy.set_event_loop, object()) @@ -2716,15 +2696,11 @@ def get_event_loop(self): asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2738,16 +2714,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) - + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) @@ -2766,15 +2737,11 @@ def test_get_event_loop_returns_running_loop2(self): loop = asyncio.new_event_loop() self.addCleanup(loop.close) - with self.assertWarns(DeprecationWarning) as cm: - loop2 = asyncio.get_event_loop() - self.addCleanup(loop2.close) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2788,15 +2755,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - self.assertIs(asyncio.get_event_loop(), loop) - self.assertEqual(cm.filename, __file__) + self.assertIs(asyncio.get_event_loop(), loop) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 83ea01c24525..56b0b864de2d 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -146,10 +146,8 @@ def test_initial_state(self): self.assertTrue(f.cancelled()) def test_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - self._new_future() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() def test_constructor_use_running_loop(self): async def test(): @@ -159,12 +157,10 @@ async def test(): self.assertIs(f.get_loop(), self.loop) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - f = self._new_future() - self.assertEqual(cm.filename, __file__) + f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -500,10 +496,8 @@ def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) ex.shutdown(wait=True) def test_wrap_future_use_running_loop(self): @@ -518,16 +512,14 @@ async def test(): ex.shutdown(wait=True) def test_wrap_future_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - f2 = asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) ex.shutdown(wait=True) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 01d5407a497a..7f9dc6218083 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -816,10 +816,8 @@ def test_read_all_from_pipe_reader(self): self.assertEqual(data, b'data') def test_streamreader_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() def test_streamreader_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -833,21 +831,17 @@ async def test(): def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) - with self.assertWarns(DeprecationWarning) as cm: - reader = asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) def test_streamreaderprotocol_constructor_without_loop(self): reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) def test_streamreaderprotocol_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -861,13 +855,11 @@ async def test(): def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - protocol = asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) def test_multiple_drain(self): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index d8ba2f4e2a74..bb1ffdf2df74 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -196,10 +196,8 @@ async def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -209,12 +207,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -1532,10 +1528,8 @@ async def coro(): self.addCleanup(a.close) futs = asyncio.as_completed([a]) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - list(futs) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + list(futs) def test_as_completed_coroutine_use_running_loop(self): loop = self.new_test_loop() @@ -1965,10 +1959,8 @@ async def coro(): inner = coro() self.addCleanup(inner.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.shield(inner) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) def test_shield_coroutine_use_running_loop(self): async def coro(): @@ -1982,15 +1974,13 @@ async def test(): self.assertEqual(res, 42) def test_shield_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 async def coro(): return 42 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - outer = asyncio.shield(coro()) - self.assertEqual(cm.filename, __file__) + outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) @@ -2827,7 +2817,7 @@ def test_current_task_no_running_loop(self): self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): asyncio.current_task() def test_current_task_with_implicit_loop(self): @@ -2991,10 +2981,8 @@ def _gather(self, *args, **kwargs): return asyncio.gather(*args, **kwargs) def test_constructor_empty_sequence_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() def test_constructor_empty_sequence_use_running_loop(self): async def gather(): @@ -3007,12 +2995,10 @@ async def gather(): self.assertEqual(fut.result(), []) def test_constructor_empty_sequence_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather() - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) @@ -3100,10 +3086,8 @@ async def coro(): self.addCleanup(gen1.close) gen2 = coro() self.addCleanup(gen2.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) def test_constructor_use_running_loop(self): async def coro(): @@ -3117,16 +3101,14 @@ async def gather(): self.one_loop.run_until_complete(fut) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.12 async def coro(): return 'abc' asyncio.set_event_loop(self.other_loop) self.addCleanup(asyncio.set_event_loop, None) gen1 = coro() gen2 = coro() - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather(gen1, gen2) self.assertIs(fut._loop, self.other_loop) self.other_loop.run_until_complete(fut) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 309a1cfdb4aa..600a5900da08 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1775,7 +1775,8 @@ def f(): def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() - loop = policy.get_event_loop() + loop = policy.new_event_loop() + policy.set_event_loop(loop) # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property @@ -1884,13 +1885,15 @@ async def test_fork_not_share_event_loop(self): # child try: loop = asyncio.get_event_loop_policy().get_event_loop() - os.write(w, str(id(loop)).encode()) + except RuntimeError: + os.write(w, b'NO LOOP') + except: + os.write(w, b'ERROR:' + ascii(sys.exc_info()).encode()) finally: os._exit(0) else: # parent - child_loop = int(os.read(r, 100).decode()) - self.assertNotEqual(child_loop, id(loop)) + self.assertEqual(os.read(r, 100), b'NO LOOP') wait_process(pid, exitcode=0) @hashlib_helper.requires_hashdigest('md5') diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index f91c9cc47741..43a3ff0536fe 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2418,7 +2418,8 @@ class UnawaitedWarningDuringShutdownTest(unittest.TestCase): def test_unawaited_warning_during_shutdown(self): code = ("import asyncio\n" "async def f(): pass\n" - "asyncio.gather(f())\n") + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") assert_python_ok("-c", code) code = ("import sys\n" diff --git a/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst b/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst new file mode 100644 index 000000000000..5aee4b4f0aa7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst @@ -0,0 +1,9 @@ +:func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like +:func:`asyncio.ensure_future`, :func:`asyncio.shield` or +:func:`asyncio.gather`, and also the +:meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of +:class:`asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` +if called when there is no running event loop and the current event loop was +not set. Previously they implicitly created and set a new current event +loop. :exc:`DeprecationWarning` is no longer emitted if there is no running +event loop but the current event loop was set. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index cabcaec94bec..60369d89dc39 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -357,7 +357,7 @@ set_running_loop(asyncio_state *state, PyObject *loop) static PyObject * -get_event_loop(asyncio_state *state, int stacklevel) +get_event_loop(asyncio_state *state) { PyObject *loop; PyObject *policy; @@ -369,13 +369,6 @@ get_event_loop(asyncio_state *state, int stacklevel) return loop; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "There is no current event loop", - stacklevel)) - { - return NULL; - } - policy = PyObject_CallNoArgs(state->asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; @@ -538,7 +531,7 @@ future_init(FutureObj *fut, PyObject *loop) if (loop == Py_None) { asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - loop = get_event_loop(state, 1); + loop = get_event_loop(state); if (loop == NULL) { return -1; } @@ -3229,20 +3222,7 @@ _asyncio_get_event_loop_impl(PyObject *module) /*[clinic end generated code: output=2a2d8b2f824c648b input=9364bf2916c8655d]*/ { asyncio_state *state = get_asyncio_state(module); - return get_event_loop(state, 1); -} - -/*[clinic input] -_asyncio._get_event_loop - stacklevel: int = 3 -[clinic start generated code]*/ - -static PyObject * -_asyncio__get_event_loop_impl(PyObject *module, int stacklevel) -/*[clinic end generated code: output=9c1d6d3c802e67c9 input=d17aebbd686f711d]*/ -{ - asyncio_state *state = get_asyncio_state(module); - return get_event_loop(state, stacklevel-1); + return get_event_loop(state); } /*[clinic input] @@ -3620,7 +3600,6 @@ PyDoc_STRVAR(module_doc, "Accelerator module for asyncio"); static PyMethodDef asyncio_methods[] = { _ASYNCIO_GET_EVENT_LOOP_METHODDEF - _ASYNCIO__GET_EVENT_LOOP_METHODDEF _ASYNCIO_GET_RUNNING_LOOP_METHODDEF _ASYNCIO__GET_RUNNING_LOOP_METHODDEF _ASYNCIO__SET_RUNNING_LOOP_METHODDEF diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 11db478a8b48..f2fbb352c2c6 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -987,68 +987,6 @@ _asyncio_get_event_loop(PyObject *module, PyObject *Py_UNUSED(ignored)) return _asyncio_get_event_loop_impl(module); } -PyDoc_STRVAR(_asyncio__get_event_loop__doc__, -"_get_event_loop($module, /, stacklevel=3)\n" -"--\n" -"\n"); - -#define _ASYNCIO__GET_EVENT_LOOP_METHODDEF \ - {"_get_event_loop", _PyCFunction_CAST(_asyncio__get_event_loop), METH_FASTCALL|METH_KEYWORDS, _asyncio__get_event_loop__doc__}, - -static PyObject * -_asyncio__get_event_loop_impl(PyObject *module, int stacklevel); - -static PyObject * -_asyncio__get_event_loop(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(stacklevel), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"stacklevel", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_get_event_loop", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int stacklevel = 3; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - stacklevel = _PyLong_AsInt(args[0]); - if (stacklevel == -1 && PyErr_Occurred()) { - goto exit; - } -skip_optional_pos: - return_value = _asyncio__get_event_loop_impl(module, stacklevel); - -exit: - return return_value; -} - PyDoc_STRVAR(_asyncio_get_running_loop__doc__, "get_running_loop($module, /)\n" "--\n" @@ -1304,4 +1242,4 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=550bc6603df89ed9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=83580c190031241c input=a9049054013a1b77]*/ From webhook-mailer at python.org Tue Dec 6 13:51:27 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 18:51:27 -0000 Subject: [Python-checkins] Python 3.9.16 Message-ID: <mailman.2828.1670352688.3313.python-checkins@python.org> https://github.com/python/cpython/commit/595f9ccb0c059f2fb5bf13643bfc0cdd5b55a422 commit: 595f9ccb0c059f2fb5bf13643bfc0cdd5b55a422 branch: 3.9 author: ?ukasz Langa <lukasz at langa.pl> committer: ambv <lukasz at langa.pl> date: 2022-12-06T18:59:46+01:00 summary: Python 3.9.16 files: A Misc/NEWS.d/3.9.16.rst D Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst D Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Include/patchlevel.h M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 8af690f9eded..e33d9a1d0170 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 9 -#define PY_MICRO_VERSION 15 +#define PY_MICRO_VERSION 16 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.9.15+" +#define PY_VERSION "3.9.16" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS.d/3.9.16.rst b/Misc/NEWS.d/3.9.16.rst new file mode 100644 index 000000000000..408cd9ca300c --- /dev/null +++ b/Misc/NEWS.d/3.9.16.rst @@ -0,0 +1,90 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-09-07-10-42-00 +.. gh-issue: 97514 +.. nonce: Yggdsl +.. section: Security + +On Linux the :mod:`multiprocessing` module returns to using filesystem +backed unix domain sockets for communication with the *forkserver* process +instead of the Linux abstract socket namespace. Only code that chooses to +use the :ref:`"forkserver" start method <multiprocessing-start-methods>` is +affected. + +Abstract sockets have no permissions and could allow any user on the system +in the same `network namespace +<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often +the whole system) to inject code into the multiprocessing *forkserver* +process. This was a potential privilege escalation. Filesystem based socket +permissions restrict this to the *forkserver* process user as was the +default in Python 3.8 and earlier. + +This prevents Linux `CVE-2022-42919 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. + +.. + +.. date: 2022-04-27-18-25-30 +.. gh-issue: 68966 +.. nonce: gjS8zs +.. section: Security + +The deprecated mailcap module now refuses to inject unsafe text (filenames, +MIME types, parameters) into shell commands. Instead of using such text, it +will warn and act as if a match was not found (or for test commands, as if +the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst deleted file mode 100644 index da81a1f6993d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst +++ /dev/null @@ -1,4 +0,0 @@ -The deprecated mailcap module now refuses to inject unsafe text (filenames, -MIME types, parameters) into shell commands. Instead of using such text, it -will warn and act as if a match was not found (or for test commands, as if -the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst deleted file mode 100644 index 02d95b570520..000000000000 --- a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst +++ /dev/null @@ -1,15 +0,0 @@ -On Linux the :mod:`multiprocessing` module returns to using filesystem backed -unix domain sockets for communication with the *forkserver* process instead of -the Linux abstract socket namespace. Only code that chooses to use the -:ref:`"forkserver" start method <multiprocessing-start-methods>` is affected. - -Abstract sockets have no permissions and could allow any user on the system in -the same `network namespace -<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often the -whole system) to inject code into the multiprocessing *forkserver* process. -This was a potential privilege escalation. Filesystem based socket permissions -restrict this to the *forkserver* process user as was the default in Python 3.8 -and earlier. - -This prevents Linux `CVE-2022-42919 -<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/README.rst b/README.rst index d78b6bc9e86f..4c804d677cf9 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.9.15 +This is Python version 3.9.16 ============================= .. image:: https://travis-ci.org/python/cpython.svg?branch=3.9 From webhook-mailer at python.org Tue Dec 6 14:34:15 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 06 Dec 2022 19:34:15 -0000 Subject: [Python-checkins] Python 3.8.16 Message-ID: <mailman.2829.1670355256.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1e3d2d52109c9d82ba307116e912d16bb4b0dbb7 commit: 1e3d2d52109c9d82ba307116e912d16bb4b0dbb7 branch: 3.8 author: ?ukasz Langa <lukasz at langa.pl> committer: ambv <lukasz at langa.pl> date: 2022-12-06T19:59:58+01:00 summary: Python 3.8.16 files: A Misc/NEWS.d/3.8.16.rst D Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Include/patchlevel.h M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 04b2eb8ad3fc..8b5b8d3a85d6 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 8 -#define PY_MICRO_VERSION 15 +#define PY_MICRO_VERSION 16 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.8.15+" +#define PY_VERSION "3.8.16" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS.d/3.8.16.rst b/Misc/NEWS.d/3.8.16.rst new file mode 100644 index 000000000000..c320159b73b5 --- /dev/null +++ b/Misc/NEWS.d/3.8.16.rst @@ -0,0 +1,66 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-04-27-18-25-30 +.. gh-issue: 68966 +.. nonce: gjS8zs +.. section: Security + +The deprecated mailcap module now refuses to inject unsafe text (filenames, +MIME types, parameters) into shell commands. Instead of using such text, it +will warn and act as if a match was not found (or for test commands, as if +the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst deleted file mode 100644 index da81a1f6993d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst +++ /dev/null @@ -1,4 +0,0 @@ -The deprecated mailcap module now refuses to inject unsafe text (filenames, -MIME types, parameters) into shell commands. Instead of using such text, it -will warn and act as if a match was not found (or for test commands, as if -the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/README.rst b/README.rst index f244a926fa79..3aed3a5c47d4 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.8.15 +This is Python version 3.8.16 ============================= .. image:: https://travis-ci.org/python/cpython.svg?branch=3.8 From webhook-mailer at python.org Tue Dec 6 15:36:08 2022 From: webhook-mailer at python.org (ned-deily) Date: Tue, 06 Dec 2022 20:36:08 -0000 Subject: [Python-checkins] Python 3.7.16 Message-ID: <mailman.2830.1670358969.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3f82aa744678620a811927dc4e56ad9c7c3d0c14 commit: 3f82aa744678620a811927dc4e56ad9c7c3d0c14 branch: 3.7 author: Ned Deily <nad at python.org> committer: ned-deily <nad at python.org> date: 2022-12-06T14:00:00-05:00 summary: Python 3.7.16 files: A Misc/NEWS.d/3.7.16.rst D Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index ab3fc9b392f7..f943b207b1ed 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 15 +#define PY_MICRO_VERSION 16 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.7.15+" +#define PY_VERSION "3.7.16" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 95eb9d4b6744..5f09d234bdb2 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Oct 10 05:27:13 2022 +# Autogenerated by Sphinx on Tue Dec 6 13:59:35 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' diff --git a/Misc/NEWS.d/3.7.16.rst b/Misc/NEWS.d/3.7.16.rst new file mode 100644 index 000000000000..8991a1f9c45f --- /dev/null +++ b/Misc/NEWS.d/3.7.16.rst @@ -0,0 +1,56 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-04-27-18-25-30 +.. gh-issue: 68966 +.. nonce: gjS8zs +.. section: Security + +The deprecated mailcap module now refuses to inject unsafe text (filenames, +MIME types, parameters) into shell commands. Instead of using such text, it +will warn and act as if a match was not found (or for test commands, as if +the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst deleted file mode 100644 index da81a1f6993d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst +++ /dev/null @@ -1,4 +0,0 @@ -The deprecated mailcap module now refuses to inject unsafe text (filenames, -MIME types, parameters) into shell commands. Instead of using such text, it -will warn and act as if a match was not found (or for test commands, as if -the test failed). diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/README.rst b/README.rst index 2c80b7b7383e..28fb2a398f14 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.7.15+ -============================== +This is Python version 3.7.16 +============================= .. image:: https://travis-ci.org/python/cpython.svg?branch=3.7 :alt: CPython build status on Travis CI From webhook-mailer at python.org Tue Dec 6 16:11:36 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 21:11:36 -0000 Subject: [Python-checkins] Python 3.11.1 Message-ID: <mailman.2831.1670361097.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a7a450f84a087421603170c2dad226bb881d4d9a commit: a7a450f84a087421603170c2dad226bb881d4d9a branch: 3.11 author: Pablo Galindo <pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T19:05:27Z summary: Python 3.11.1 files: A Misc/NEWS.d/3.11.1.rst D Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst D Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst D Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst D Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst D Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst D Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst D Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst D Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst D Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst D Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst D Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst D Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst D Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst D Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst D Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst D Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst D Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst D Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst D Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst D Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst D Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst D Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst D Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst D Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst D Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst D Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst D Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst D Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst D Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst D Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst D Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst D Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst D Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst D Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst D Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst D Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst D Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst D Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst D Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst D Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst D Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst D Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst D Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst D Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst D Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst D Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst D Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst D Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst D Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst D Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst D Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst D Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst D Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst D Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst D Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst D Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst D Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst D Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst D Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst D Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst D Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst D Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst D Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst D Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst D Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst D Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst D Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst D Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst D Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst D Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst D Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst D Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst D Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst D Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst D Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst D Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst D Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst D Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst D Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst D Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst D Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst D Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst D Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst D Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst D Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst D Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst D Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst D Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst D Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst D Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst D Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst D Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst D Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst D Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst D Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst D Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst D Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst D Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst D Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst D Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst D Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst D Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst D Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst D Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst D Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst D Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst D Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst D Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst D Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c37154baa22b..7f217c1db4fe 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 11 -#define PY_MICRO_VERSION 0 +#define PY_MICRO_VERSION 1 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.11.0+" +#define PY_VERSION "3.11.1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 7bba5cb88093..11a1c73bf70f 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Oct 24 18:35:07 2022 +# Autogenerated by Sphinx on Tue Dec 6 19:05:00 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -358,7 +358,7 @@ 'yield_expression)]\n' '\n' 'The difference from normal Assignment statements is that only ' - 'single\n' + 'a single\n' 'target is allowed.\n' '\n' 'For simple names as assignment targets, if in class or module ' @@ -408,12 +408,13 @@ 'analysis\n' ' tools and IDEs.\n' '\n' - 'Changed in version 3.8: Now annotated assignments allow same\n' - 'expressions in the right hand side as the regular ' - 'assignments.\n' - 'Previously, some expressions (like un-parenthesized tuple ' - 'expressions)\n' - 'caused a syntax error.\n', + 'Changed in version 3.8: Now annotated assignments allow the ' + 'same\n' + 'expressions in the right hand side as regular assignments. ' + 'Previously,\n' + 'some expressions (like un-parenthesized tuple expressions) ' + 'caused a\n' + 'syntax error.\n', 'async': 'Coroutines\n' '**********\n' '\n' @@ -2551,12 +2552,13 @@ 'that\n' 'multiple "except*" clauses can execute, each handling part of ' 'the\n' - 'exception group. Each clause executes once and handles an ' - 'exception\n' - 'group of all matching exceptions. Each exception in the group ' - 'is\n' - 'handled by at most one "except*" clause, the first that matches ' - 'it.\n' + 'exception group. Each clause executes at most once and handles ' + 'an\n' + 'exception group of all matching exceptions. Each exception in ' + 'the\n' + 'group is handled by at most one "except*" clause, the first ' + 'that\n' + 'matches it.\n' '\n' ' >>> try:\n' ' ... raise ExceptionGroup("eg",\n' @@ -2579,21 +2581,32 @@ ' | ValueError: 1\n' ' +------------------------------------\n' '\n' - ' Any remaining exceptions that were not handled by any ' - ':keyword:`!except*`\n' - ' clause are re-raised at the end, combined into an exception ' - 'group along with\n' - ' all exceptions that were raised from within ' - ':keyword:`!except*` clauses.\n' + 'Any remaining exceptions that were not handled by any "except*" ' + 'clause\n' + 'are re-raised at the end, combined into an exception group along ' + 'with\n' + 'all exceptions that were raised from within "except*" clauses.\n' + '\n' + 'If the raised exception is not an exception group and its type ' + 'matches\n' + 'one of the "except*" clauses, it is caught and wrapped by an ' + 'exception\n' + 'group with an empty message string.\n' + '\n' + ' >>> try:\n' + ' ... raise BlockingIOError\n' + ' ... except* BlockingIOError as e:\n' + ' ... print(repr(e))\n' + ' ...\n' + " ExceptionGroup('', (BlockingIOError()))\n" '\n' - ' An :keyword:`!except*` clause must have a matching type,\n' - ' and this type cannot be a subclass of ' - ':exc:`BaseExceptionGroup`.\n' - ' It is not possible to mix :keyword:`except` and ' - ':keyword:`!except*`\n' - ' in the same :keyword:`try`.\n' - ' :keyword:`break`, :keyword:`continue` and :keyword:`return`\n' - ' cannot appear in an :keyword:`!except*` clause.\n' + 'An "except*" clause must have a matching type, and this type ' + 'cannot be\n' + 'a subclass of "BaseExceptionGroup". It is not possible to mix ' + '"except"\n' + 'and "except*" in the same "try". "break", "continue" and ' + '"return"\n' + 'cannot appear in an "except*" clause.\n' '\n' '\n' '"else" clause\n' @@ -7308,7 +7321,7 @@ 'the clauses had been separated out into individual import ' 'statements.\n' '\n' - 'The details of the first step, finding and loading modules are\n' + 'The details of the first step, finding and loading modules, are\n' 'described in greater detail in the section on the import system, ' 'which\n' 'also describes the various types of packages and modules that can ' @@ -11096,8 +11109,9 @@ 'y)" is\n' 'typically invalid without special support in "MyClass". To ' 'be able to\n' - 'use that kind of patterns, the class needs to define a\n' - '*__match_args__* attribute.\n' + 'use that kind of pattern, the class needs to define a ' + '*__match_args__*\n' + 'attribute.\n' '\n' 'object.__match_args__\n' '\n' @@ -11302,6 +11316,10 @@ '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' + ' If *sub* is empty, returns the number of empty strings ' + 'between\n' + ' characters which is the length of the string plus one.\n' + '\n' "str.encode(encoding='utf-8', errors='strict')\n" '\n' ' Return an encoded version of the string as a bytes ' @@ -11808,7 +11826,7 @@ 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11849,7 +11867,7 @@ " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -12650,9 +12668,10 @@ 'the type matches some of the exceptions in the group. This means ' 'that\n' 'multiple "except*" clauses can execute, each handling part of the\n' - 'exception group. Each clause executes once and handles an exception\n' - 'group of all matching exceptions. Each exception in the group is\n' - 'handled by at most one "except*" clause, the first that matches it.\n' + 'exception group. Each clause executes at most once and handles an\n' + 'exception group of all matching exceptions. Each exception in the\n' + 'group is handled by at most one "except*" clause, the first that\n' + 'matches it.\n' '\n' ' >>> try:\n' ' ... raise ExceptionGroup("eg",\n' @@ -12673,20 +12692,31 @@ ' | ValueError: 1\n' ' +------------------------------------\n' '\n' - ' Any remaining exceptions that were not handled by any ' - ':keyword:`!except*`\n' - ' clause are re-raised at the end, combined into an exception group ' - 'along with\n' - ' all exceptions that were raised from within :keyword:`!except*` ' - 'clauses.\n' + 'Any remaining exceptions that were not handled by any "except*" ' + 'clause\n' + 'are re-raised at the end, combined into an exception group along ' + 'with\n' + 'all exceptions that were raised from within "except*" clauses.\n' '\n' - ' An :keyword:`!except*` clause must have a matching type,\n' - ' and this type cannot be a subclass of :exc:`BaseExceptionGroup`.\n' - ' It is not possible to mix :keyword:`except` and ' - ':keyword:`!except*`\n' - ' in the same :keyword:`try`.\n' - ' :keyword:`break`, :keyword:`continue` and :keyword:`return`\n' - ' cannot appear in an :keyword:`!except*` clause.\n' + 'If the raised exception is not an exception group and its type ' + 'matches\n' + 'one of the "except*" clauses, it is caught and wrapped by an ' + 'exception\n' + 'group with an empty message string.\n' + '\n' + ' >>> try:\n' + ' ... raise BlockingIOError\n' + ' ... except* BlockingIOError as e:\n' + ' ... print(repr(e))\n' + ' ...\n' + " ExceptionGroup('', (BlockingIOError()))\n" + '\n' + 'An "except*" clause must have a matching type, and this type cannot ' + 'be\n' + 'a subclass of "BaseExceptionGroup". It is not possible to mix ' + '"except"\n' + 'and "except*" in the same "try". "break", "continue" and "return"\n' + 'cannot appear in an "except*" clause.\n' '\n' '\n' '"else" clause\n' @@ -13947,17 +13977,11 @@ 'dictionaries or\n' 'other mutable types (that are compared by value rather than ' 'by object\n' - 'identity) may not be used as keys. Numeric types used for ' - 'keys obey\n' - 'the normal rules for numeric comparison: if two numbers ' - 'compare equal\n' - '(such as "1" and "1.0") then they can be used ' - 'interchangeably to index\n' - 'the same dictionary entry. (Note however, that since ' - 'computers store\n' - 'floating-point numbers as approximations it is usually ' - 'unwise to use\n' - 'them as dictionary keys.)\n' + 'identity) may not be used as keys. Values that compare equal ' + '(such as\n' + '"1", "1.0", and "True") can be used interchangeably to index ' + 'the same\n' + 'dictionary entry.\n' '\n' 'class dict(**kwargs)\n' 'class dict(mapping, **kwargs)\n' diff --git a/Misc/NEWS.d/3.11.1.rst b/Misc/NEWS.d/3.11.1.rst new file mode 100644 index 000000000000..62cf53afe507 --- /dev/null +++ b/Misc/NEWS.d/3.11.1.rst @@ -0,0 +1,1545 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-09-28-12-10-57 +.. gh-issue: 97612 +.. nonce: y6NvOQ +.. section: Security + +Fix a shell code injection vulnerability in the +``get-remote-certificate.py`` example script. The script no longer uses a +shell to run ``openssl`` commands. Issue reported and initial fix by Caleb +Shortt. Patch by Victor Stinner. + +.. + +.. date: 2022-11-30-15-29-08 +.. gh-issue: 99886 +.. nonce: feJkSv +.. section: Core and Builtins + +Fix a crash when an object which does not have a dictionary frees its +instance values. + +.. + +.. date: 2022-11-30-11-09-40 +.. gh-issue: 99891 +.. nonce: 9VomwB +.. section: Core and Builtins + +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo + +.. + +.. date: 2022-11-26-05-34-00 +.. gh-issue: 99729 +.. nonce: A3ovwQ +.. section: Core and Builtins + +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. + +.. + +.. date: 2022-11-21-11-27-14 +.. gh-issue: 99578 +.. nonce: DcKoBJ +.. section: Core and Builtins + +Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by +Victor Stinner. + +.. + +.. date: 2022-11-19-22-27-52 +.. gh-issue: 99581 +.. nonce: yKYPbf +.. section: Core and Builtins + +Fixed a bug that was causing a buffer overflow if the tokenizer copies a +line missing the newline caracter from a file that is as long as the +available tokenizer buffer. Patch by Pablo galindo + +.. + +.. date: 2022-11-18-11-24-25 +.. gh-issue: 99553 +.. nonce: F64h-n +.. section: Core and Builtins + +Fix bug where an :exc:`ExceptionGroup` subclass can wrap a +:exc:`BaseException`. + +.. + +.. date: 2022-11-12-01-39-57 +.. gh-issue: 99370 +.. nonce: _cu32j +.. section: Core and Builtins + +Fix zip path for venv created from a non-installed python on POSIX +platforms. + +.. + +.. date: 2022-11-10-02-11-23 +.. gh-issue: 99298 +.. nonce: NeArAJ +.. section: Core and Builtins + +Fix an issue that could potentially cause incorrect error handling for some +bytecode instructions. + +.. + +.. date: 2022-11-08-16-35-25 +.. gh-issue: 99205 +.. nonce: 2YOoFT +.. section: Core and Builtins + +Fix an issue that prevented :c:type:`PyThreadState` and +:c:type:`PyInterpreterState` memory from being freed properly. + +.. + +.. date: 2022-11-07-10-29-41 +.. gh-issue: 99181 +.. nonce: bfG4bI +.. section: Core and Builtins + +Fix failure in :keyword:`except* <except_star>` with unhashable exceptions. + +.. + +.. date: 2022-11-07-08-17-12 +.. gh-issue: 99204 +.. nonce: Mf4hMD +.. section: Core and Builtins + +Fix calculation of :data:`sys._base_executable` when inside a POSIX virtual +environment using copies of the python binary when the base installation +does not provide the executable name used by the venv. Calculation will fall +back to alternative names ("python<MAJOR>", "python<MAJOR>.<MINOR>"). + +.. + +.. date: 2022-11-06-22-59-02 +.. gh-issue: 96055 +.. nonce: TmQuJn +.. section: Core and Builtins + +Update :mod:`faulthandler` to emit an error message with the proper +unexpected signal number. Patch by Dong-hee Na. + +.. + +.. date: 2022-11-06-13-25-01 +.. gh-issue: 99153 +.. nonce: uE3CVL +.. section: Core and Builtins + +Fix location of :exc:`SyntaxError` for a :keyword:`try` block with both +:keyword:`except` and :keyword:`except* <except_star>`. + +.. + +.. date: 2022-11-06-00-17-58 +.. gh-issue: 99103 +.. nonce: bFA9BX +.. section: Core and Builtins + +Fix the error reporting positions of specialized traceback anchors when the +source line contains Unicode characters. + +.. + +.. date: 2022-10-31-21-01-35 +.. gh-issue: 98852 +.. nonce: MYaRN6 +.. section: Core and Builtins + +Fix subscription of type aliases containing bare generic types or types like +:class:`~typing.TypeVar`: for example ``tuple[A, T][int]`` and +``tuple[TypeVar, T][int]``, where ``A`` is a generic type, and ``T`` is a +type variable. + +.. + +.. date: 2022-10-31-18-03-10 +.. gh-issue: 98925 +.. nonce: zpdjVd +.. section: Core and Builtins + +Lower the recursion depth for marshal on WASI to support wasmtime 2.0/main. + +.. + +.. date: 2022-10-28-14-52-55 +.. gh-issue: 98783 +.. nonce: iG0kMs +.. section: Core and Builtins + +Fix multiple crashes in debug mode when ``str`` subclasses are used instead +of ``str`` itself. + +.. + +.. date: 2022-10-21-11-28-53 +.. gh-issue: 99257 +.. nonce: nmcuf- +.. section: Core and Builtins + +Fix an issue where member descriptors (such as those for +:attr:`~object.__slots__`) could behave incorrectly or crash instead of +raising a :exc:`TypeError` when accessed via an instance of an invalid type. + +.. + +.. date: 2022-10-19-23-48-46 +.. gh-issue: 98374 +.. nonce: eOBh8M +.. section: Core and Builtins + +Suppress ImportError for invalid query for help() command. Patch by Dong-hee +Na. + +.. + +.. date: 2022-10-19-01-01-08 +.. gh-issue: 98415 +.. nonce: ZS2eWh +.. section: Core and Builtins + +Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by +Chaim Sanders + +.. + +.. date: 2022-10-15-23-15-14 +.. gh-issue: 92119 +.. nonce: PMSwwG +.. section: Core and Builtins + +Print exception class name instead of its string representation when raising +errors from :mod:`ctypes` calls. + +.. + +.. date: 2022-10-06-15-45-57 +.. gh-issue: 96078 +.. nonce: fS-6mU +.. section: Core and Builtins + +:func:`os.sched_yield` now release the GIL while calling sched_yield(2). +Patch by Dong-hee Na. + +.. + +.. date: 2022-10-06-05-41-01 +.. gh-issue: 93354 +.. nonce: 6BpHl2 +.. section: Core and Builtins + +Fix an issue that could delay the specialization of :opcode:`PRECALL` +instructions. + +.. + +.. date: 2022-10-05-17-02-22 +.. gh-issue: 97943 +.. nonce: LYAWlE +.. section: Core and Builtins + +Bugfix: :func:`PyFunction_GetAnnotations` should return a borrowed +reference. It was returning a new reference. + +.. + +.. date: 2022-10-04-02-00-10 +.. gh-issue: 97779 +.. nonce: f3N1hI +.. section: Core and Builtins + +Ensure that all Python frame objects are backed by "complete" frames. + +.. + +.. date: 2022-10-01-08-55-09 +.. gh-issue: 97591 +.. nonce: pw6kkH +.. section: Core and Builtins + +Fixed a missing incref/decref pair in ``Exception.__setstate__()``. Patch by +Ofey Chan. + +.. + +.. date: 2022-09-29-15-19-29 +.. gh-issue: 94526 +.. nonce: wq5m6T +.. section: Core and Builtins + +Fix the Python path configuration used to initialized :data:`sys.path` at +Python startup. Paths are no longer encoded to UTF-8/strict to avoid +encoding errors if it contains surrogate characters (bytes paths are decoded +with the surrogateescape error handler). Patch by Victor Stinner. + +.. + +.. date: 2022-09-20-11-06-45 +.. gh-issue: 95921 +.. nonce: dkcRQn +.. section: Core and Builtins + +Fix overly-broad source position information for chained comparisons used as +branching conditions. + +.. + +.. date: 2022-09-16-16-54-35 +.. gh-issue: 96387 +.. nonce: GRzewg +.. section: Core and Builtins + +At Python exit, sometimes a thread holding the GIL can wait forever for a +thread (usually a daemon thread) which requested to drop the GIL, whereas +the thread already exited. To fix the race condition, the thread which +requested the GIL drop now resets its request before exiting. Issue +discovered and analyzed by Mingliang ZHAO. Patch by Victor Stinner. + +.. + +.. date: 2022-09-16-12-36-13 +.. gh-issue: 96864 +.. nonce: PLU3i8 +.. section: Core and Builtins + +Fix a possible assertion failure, fatal error, or :exc:`SystemError` if a +line tracing event raises an exception while opcode tracing is enabled. + +.. + +.. date: 2022-09-13-12-06-46 +.. gh-issue: 96678 +.. nonce: NqGFyb +.. section: Core and Builtins + +Fix undefined behaviour in C code of null pointer arithmetic. + +.. + +.. date: 2022-09-12-16-58-22 +.. gh-issue: 96754 +.. nonce: 0GRme5 +.. section: Core and Builtins + +Make sure that all frame objects created are created from valid interpreter +frames. Prevents the possibility of invalid frames in backtraces and signal +handlers. + +.. + +.. date: 2022-08-29-13-06-58 +.. gh-issue: 95196 +.. nonce: eGRR4b +.. section: Core and Builtins + +Disable incorrect pickling of the C implemented classmethod descriptors. + +.. + +.. date: 2022-08-15-21-08-11 +.. gh-issue: 96005 +.. nonce: 6eoc8k +.. section: Core and Builtins + +On WASI :data:`~errno.ENOTCAPABLE` is now mapped to :exc:`PermissionError`. +The :mod:`errno` modules exposes the new error number. ``getpath.py`` now +ignores :exc:`PermissionError` when it cannot open landmark files +``pybuilddir.txt`` and ``pyenv.cfg``. + +.. + +.. date: 2022-06-10-16-37-44 +.. gh-issue: 93696 +.. nonce: 65BI2R +.. section: Core and Builtins + +Allow :mod:`pdb` to locate source for frozen modules in the standard +library. + +.. + +.. bpo: 31718 +.. date: 2020-02-23-23-48-15 +.. nonce: sXko5e +.. section: Core and Builtins + +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. + +.. + +.. bpo: 38031 +.. date: 2019-09-04-19-09-49 +.. nonce: Yq4L72 +.. section: Core and Builtins + +Fix a possible assertion failure in :class:`io.FileIO` when the opener +returns an invalid file descriptor. + +.. + +.. date: 2022-12-05-13-40-15 +.. gh-issue: 100001 +.. nonce: 78ReYp +.. section: Library + +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. + +.. + +.. date: 2022-12-02-13-05-00 +.. gh-issue: 93453 +.. nonce: EFj1NN +.. section: Library + +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. + +.. + +.. date: 2022-11-21-17-56-18 +.. gh-issue: 51524 +.. nonce: nTykx8 +.. section: Library + +Fix bug when calling trace.CoverageResults with valid infile. + +.. + +.. date: 2022-11-21-13-49-03 +.. gh-issue: 99645 +.. nonce: 9w1QKq +.. section: Library + +Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now +``addClassCleanup()`` uses separate lists for different ``TestCase`` +subclasses, and ``doClassCleanups()`` only cleans up the particular class. + +.. + +.. date: 2022-11-15-10-55-24 +.. gh-issue: 97001 +.. nonce: KeQuVF +.. section: Library + +Release the GIL when calling termios APIs to avoid blocking threads. + +.. + +.. date: 2022-11-13-02-06-56 +.. gh-issue: 99341 +.. nonce: 8-OlwB +.. section: Library + +Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when +changing line numbers. + +.. + +.. date: 2022-11-12-15-45-51 +.. gh-issue: 99418 +.. nonce: FxfAXS +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes URL schemes that begin +with a digit, a plus sign, or a minus sign to be parsed incorrectly. + +.. + +.. date: 2022-11-12-12-15-30 +.. gh-issue: 99382 +.. nonce: dKg_rW +.. section: Library + +Check the number of arguments in substitution in user generics containing a +:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. + +.. + +.. date: 2022-11-12-12-10-23 +.. gh-issue: 99379 +.. nonce: bcGhxF +.. section: Library + +Fix substitution of :class:`~typing.ParamSpec` followed by +:class:`~typing.TypeVarTuple` in generic aliases. + +.. + +.. date: 2022-11-12-12-08-34 +.. gh-issue: 99344 +.. nonce: 7M_u8G +.. section: Library + +Fix substitution of :class:`~typing.TypeVarTuple` and +:class:`~typing.ParamSpec` together in user generics. + +.. + +.. date: 2022-11-09-20-48-42 +.. gh-issue: 74044 +.. nonce: zBj26K +.. section: Library + +Fixed bug where :func:`inspect.signature` reported incorrect arguments for +decorated methods. + +.. + +.. date: 2022-11-09-12-16-35 +.. gh-issue: 99275 +.. nonce: klOqoL +.. section: Library + +Fix ``SystemError`` in :mod:`ctypes` when exception was not set during +``__initsubclass__``. + +.. + +.. date: 2022-11-09-08-40-52 +.. gh-issue: 99277 +.. nonce: J1P44O +.. section: Library + +Remove older version of ``_SSLProtocolTransport.get_write_buffer_limits`` in +:mod:`!asyncio.sslproto` + +.. + +.. date: 2022-11-08-11-15-37 +.. gh-issue: 99248 +.. nonce: 1vt8xI +.. section: Library + +fix negative numbers failing in verify() + +.. + +.. date: 2022-11-06-12-44-51 +.. gh-issue: 99155 +.. nonce: vLZOzi +.. section: Library + +Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. + +.. + +.. date: 2022-11-05-23-16-15 +.. gh-issue: 93464 +.. nonce: ucd4vP +.. section: Library + +``enum.auto()`` is now correctly activated when combined with other +assignment values. E.g. ``ONE = auto(), 'some text'`` will now evaluate as +``(1, 'some text')``. + +.. + +.. date: 2022-11-05-17-16-40 +.. gh-issue: 99134 +.. nonce: Msgspf +.. section: Library + +Update the bundled copy of pip to version 22.3.1. + +.. + +.. date: 2022-11-02-05-54-02 +.. gh-issue: 83004 +.. nonce: 0v8iyw +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_zoneinfo` + +.. + +.. date: 2022-11-02-05-53-25 +.. gh-issue: 83004 +.. nonce: qc_KHr +.. section: Library + +Clean up refleaks on failed module initialisation in in :mod:`_pickle` + +.. + +.. date: 2022-11-02-05-52-36 +.. gh-issue: 83004 +.. nonce: LBl79O +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_io`. + +.. + +.. date: 2022-10-31-12-34-03 +.. gh-issue: 98897 +.. nonce: rgNn4x +.. section: Library + +Fix memory leak in :func:`math.dist` when both points don't have the same +dimension. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-30-12-22-24 +.. gh-issue: 98706 +.. nonce: v1Kuy5 +.. section: Library + +[3.11] Applied changes from importlib_metadata `4.11.4 through 4.13 +<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-13-0>`_, +including compatibility and robustness fixes for ``Distribution`` objects +without ``_normalized_name``, disallowing invalid inputs to +``Distribution.from_name``, and refined behaviors in +``PathDistribution._name_from_stem`` and +``PathDistribution._normalized_name``. + +.. + +.. date: 2022-10-29-03-40-18 +.. gh-issue: 98793 +.. nonce: WSPB4A +.. section: Library + +Fix argument typechecks in :func:`!_overlapped.WSAConnect` and +:func:`!_overlapped.Overlapped.WSASendTo` functions. + +.. + +.. date: 2022-10-28-23-44-17 +.. gh-issue: 98744 +.. nonce: sGHDWm +.. section: Library + +Prevent crashing in :mod:`traceback` when retrieving the byte-offset for +some source files that contain certain unicode characters. + +.. + +.. date: 2022-10-27-12-56-38 +.. gh-issue: 98740 +.. nonce: ZoqqGM +.. section: Library + +Fix internal error in the :mod:`re` module which in very rare circumstances +prevented compilation of a regular expression containing a :ref:`conditional +expression <re-conditional-expression>` without the "else" branch. + +.. + +.. date: 2022-10-26-07-51-55 +.. gh-issue: 98703 +.. nonce: 0hW773 +.. section: Library + +Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` +callback only once on Windows. + +.. + +.. date: 2022-10-25-20-17-34 +.. gh-issue: 98624 +.. nonce: YQUPFy +.. section: Library + +Add a mutex to unittest.mock.NonCallableMock to protect concurrent access to +mock attributes. + +.. + +.. date: 2022-10-23-18-30-39 +.. gh-issue: 89237 +.. nonce: kBui30 +.. section: Library + +Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with +:class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-19-18-31-53 +.. gh-issue: 98458 +.. nonce: vwyq7O +.. section: Library + +Fix infinite loop in unittest when a self-referencing chained exception is +raised + +.. + +.. date: 2022-10-19-09-29-12 +.. gh-issue: 97928 +.. nonce: xj3im7 +.. section: Library + +:meth:`tkinter.Text.count` raises now an exception for options starting with +"-" instead of silently ignoring them. + +.. + +.. date: 2022-10-16-18-52-00 +.. gh-issue: 97966 +.. nonce: humlhz +.. section: Library + +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. + +.. + +.. date: 2022-10-16-06-18-59 +.. gh-issue: 98307 +.. nonce: b2_CDu +.. section: Library + +A :meth:`~logging.handlers.SysLogHandler.createSocket` method was added to +:class:`~logging.handlers.SysLogHandler`. + +.. + +.. date: 2022-10-14-19-57-37 +.. gh-issue: 96035 +.. nonce: 0xcX-p +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers +containing whitespace, underscores, plus and minus signs, or non-ASCII +digits to be incorrectly accepted. + +.. + +.. date: 2022-10-14-11-46-31 +.. gh-issue: 98251 +.. nonce: Uxc9al +.. section: Library + +Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +and ``pip`` when they do not impact path resolution + +.. + +.. date: 2022-10-12-10-00-40 +.. gh-issue: 98178 +.. nonce: hspH51 +.. section: Library + +On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded +applications. On macOS, the libc ``syslog()`` function is not thread-safe, +so :func:`syslog.syslog` no longer releases the GIL to call it. Patch by +Victor Stinner. + +.. + +.. date: 2022-10-10-07-07-31 +.. gh-issue: 96151 +.. nonce: K9fwoq +.. section: Library + +Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. + +.. + +.. date: 2022-10-09-12-12-38 +.. gh-issue: 87730 +.. nonce: ClgP3f +.. section: Library + +Wrap network errors consistently in urllib FTP support, so the test suite +doesn't fail when a network is available but the public internet is not +reachable. + +.. + +.. date: 2022-10-08-19-39-27 +.. gh-issue: 98086 +.. nonce: y---WC +.. section: Library + +Make sure ``patch.dict()`` can be applied on async functions. + +.. + +.. date: 2022-10-06-23-42-00 +.. gh-issue: 90985 +.. nonce: s280JY +.. section: Library + +Earlier in 3.11 we deprecated ``asyncio.Task.cancel("message")``. We +realized we were too harsh, and have undeprecated it. + +.. + +.. date: 2022-10-04-21-21-41 +.. gh-issue: 97837 +.. nonce: 19q-eg +.. section: Library + +Change deprecate warning message in :mod:`unittest` from + +``It is deprecated to return a value!=None`` + +to + +``It is deprecated to return a value that is not None from a test case`` + +.. + +.. date: 2022-10-04-07-55-19 +.. gh-issue: 97825 +.. nonce: mNdv1l +.. section: Library + +Fixes :exc:`AttributeError` when :meth:`subprocess.check_output` is used +with argument ``input=None`` and either of the arguments *encoding* or +*errors* are used. + +.. + +.. date: 2022-10-02-12-38-22 +.. gh-issue: 82836 +.. nonce: OvYLmC +.. section: Library + +Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the +:mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would +return True from this method; now they correctly return False. + +.. + +.. date: 2022-09-30-15-56-20 +.. gh-issue: 96827 +.. nonce: lzy1iw +.. section: Library + +Avoid spurious tracebacks from :mod:`asyncio` when default executor cleanup +is delayed until after the event loop is closed (e.g. as the result of a +keyboard interrupt). + +.. + +.. date: 2022-09-29-23-22-24 +.. gh-issue: 97592 +.. nonce: tpJg_J +.. section: Library + +Avoid a crash in the C version of +:meth:`asyncio.Future.remove_done_callback` when an evil argument is passed. + +.. + +.. date: 2022-09-29-08-15-55 +.. gh-issue: 97639 +.. nonce: JSjWYW +.. section: Library + +Remove ``tokenize.NL`` check from :mod:`tabnanny`. + +.. + +.. date: 2022-09-25-20-42-33 +.. gh-issue: 73588 +.. nonce: uVtjEA +.. section: Library + +Fix generation of the default name of :class:`tkinter.Checkbutton`. +Previously, checkbuttons in different parent widgets could have the same +short name and share the same state if arguments "name" and "variable" are +not specified. Now they are globally unique. + +.. + +.. date: 2022-09-22-14-35-02 +.. gh-issue: 97005 +.. nonce: yf21Q7 +.. section: Library + +Update bundled libexpat to 2.4.9 + +.. + +.. date: 2022-09-22-11-50-29 +.. gh-issue: 85760 +.. nonce: DETTPd +.. section: Library + +Fix race condition in :mod:`asyncio` where +:meth:`~asyncio.SubprocessProtocol.process_exited` called before the +:meth:`~asyncio.SubprocessProtocol.pipe_data_received` leading to +inconsistent output. Patch by Kumar Aditya. + +.. + +.. date: 2022-09-17-13-15-10 +.. gh-issue: 96819 +.. nonce: 6RfqM7 +.. section: Library + +Fixed check in :mod:`multiprocessing.resource_tracker` that guarantees that +the length of a write to a pipe is not greater than ``PIPE_BUF``. + +.. + +.. date: 2022-09-15-00-37-33 +.. gh-issue: 96741 +.. nonce: 4b6czN +.. section: Library + +Corrected type annotation for dataclass attribute +``pstats.FunctionProfile.ncalls`` to be ``str``. + +.. + +.. date: 2022-08-30-11-46-36 +.. gh-issue: 95987 +.. nonce: CV7_u4 +.. section: Library + +Fix ``repr`` of ``Any`` subclasses. + +.. + +.. date: 2022-08-29-16-54-36 +.. gh-issue: 96388 +.. nonce: dCpJcu +.. section: Library + +Work around missing socket functions in :class:`~socket.socket`'s +``__repr__``. + +.. + +.. date: 2022-08-29-12-35-28 +.. gh-issue: 96073 +.. nonce: WaGstf +.. section: Library + +In :mod:`inspect`, fix overeager replacement of "``typing.``" in formatting +annotations. + +.. + +.. date: 2022-08-23-03-13-18 +.. gh-issue: 96192 +.. nonce: TJywOF +.. section: Library + +Fix handling of ``bytes`` :term:`path-like objects <path-like object>` in +:func:`os.ismount()`. + +.. + +.. date: 2022-08-20-10-31-01 +.. gh-issue: 96052 +.. nonce: a6FhaD +.. section: Library + +Fix handling compiler warnings (SyntaxWarning and DeprecationWarning) in +:func:`codeop.compile_command` when checking for incomplete input. +Previously it emitted warnings and raised a SyntaxError. Now it always +returns ``None`` for incomplete input without emitting any warnings. + +.. + +.. date: 2022-08-06-12-18-07 +.. gh-issue: 88863 +.. nonce: NnqsuJ +.. section: Library + +To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, +break reference cycles generated by local exception and future instances +(which has exception instance as its member var). Patch by Dong Uk, Kang. + +.. + +.. date: 2022-07-22-09-09-08 +.. gh-issue: 91212 +.. nonce: 53O8Ab +.. section: Library + +Fixed flickering of the turtle window when the tracer is turned off. Patch +by Shin-myoung-serp. + +.. + +.. date: 2022-07-08-08-39-35 +.. gh-issue: 88050 +.. nonce: 0aOC_m +.. section: Library + +Fix :mod:`asyncio` subprocess transport to kill process cleanly when process +is blocked and avoid ``RuntimeError`` when loop is closed. Patch by Kumar +Aditya. + +.. + +.. date: 2022-06-17-12-02-30 +.. gh-issue: 93858 +.. nonce: R49ARc +.. section: Library + +Prevent error when activating venv in nested fish instances. + +.. + +.. date: 2022-04-23-03-46-37 +.. gh-issue: 91078 +.. nonce: 87-hkp +.. section: Library + +:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. + +.. + +.. bpo: 47220 +.. date: 2022-04-04-22-54-11 +.. nonce: L9jYu4 +.. section: Library + +Document the optional *callback* parameter of :class:`WeakMethod`. Patch by +G?ry Ogam. + +.. + +.. bpo: 46364 +.. date: 2022-01-14-10-49-20 +.. nonce: SzhlU9 +.. section: Library + +Restrict use of sockets instead of pipes for stdin of subprocesses created +by :mod:`asyncio` to AIX platform only. + +.. + +.. bpo: 38523 +.. date: 2020-10-23-22-20-52 +.. nonce: CrkxLh +.. section: Library + +:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument +recursively. + +.. + +.. bpo: 36267 +.. date: 2019-09-03-15-45-19 +.. nonce: z42Ex7 +.. section: Library + +Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` +action is given an explicit argument. + +.. + +.. date: 2022-11-16-12-52-23 +.. gh-issue: 92892 +.. nonce: TS-P0j +.. section: Documentation + +Document that calling variadic functions with ctypes requires special care +on macOS/arm64 (and possibly other platforms). + +.. + +.. date: 2022-10-16-17-34-45 +.. gh-issue: 85525 +.. nonce: DvkD0v +.. section: Documentation + +Remove extra row + +.. + +.. date: 2022-08-12-01-12-52 +.. gh-issue: 95588 +.. nonce: PA0FI7 +.. section: Documentation + +Clarified the conflicting advice given in the :mod:`ast` documentation about +:func:`ast.literal_eval` being "safe" for use on untrusted input while at +the same time warning that it can crash the process. The latter statement is +true and is deemed unfixable without a large amount of work unsuitable for a +bugfix. So we keep the warning and no longer claim that ``literal_eval`` is +safe. + +.. + +.. bpo: 41825 +.. date: 2020-09-22-12-32-16 +.. nonce: npcaCb +.. section: Documentation + +Restructured the documentation for the :func:`os.wait* <os.wait>` family of +functions, and improved the docs for :func:`os.waitid` with more explanation +of the possible argument constants. + +.. + +.. date: 2022-12-05-16-12-56 +.. gh-issue: 99892 +.. nonce: sz_eW8 +.. section: Tests + +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. + +.. + +.. date: 2022-12-01-18-55-18 +.. gh-issue: 99934 +.. nonce: Ox3Fqf +.. section: Tests + +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. + +.. + +.. date: 2022-11-21-19-21-30 +.. gh-issue: 99659 +.. nonce: 4gP0nm +.. section: Tests + +Optional big memory tests in ``test_sqlite3`` now catch the correct +:exc:`sqlite.DataError` exception type in case of too large strings and/or +blobs passed. + +.. + +.. date: 2022-10-26-15-19-20 +.. gh-issue: 98713 +.. nonce: Lnu32R +.. section: Tests + +Fix a bug in the :mod:`typing` tests where a test relying on +CPython-specific implementation details was not decorated with +``@cpython_only`` and was not skipped on other implementations. + +.. + +.. date: 2022-10-15-07-46-48 +.. gh-issue: 87390 +.. nonce: asR-Zo +.. section: Tests + +Add tests for star-unpacking with PEP 646, and some other miscellaneous PEP +646 tests. + +.. + +.. date: 2022-10-12-14-57-06 +.. gh-issue: 96853 +.. nonce: ANe-bw +.. section: Tests + +Added explicit coverage of ``Py_Initialize`` (and hence ``Py_InitializeEx``) +back to the embedding tests (all other embedding tests migrated to +``Py_InitializeFromConfig`` in Python 3.11) + +.. + +.. bpo: 34272 +.. date: 2018-07-29-15-59-51 +.. nonce: lVX2uR +.. section: Tests + +Some C API tests were moved into the new Lib/test/test_capi/ directory. + +.. + +.. date: 2022-11-24-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and +``-Wimplicit-function-declaration`` compiler warnings in +:program:`configure` checks. + +.. + +.. date: 2022-11-15-08-40-22 +.. gh-issue: 99337 +.. nonce: 5LoQDE +.. section: Build + +Fix a compilation issue with GCC 12 on macOS. + +.. + +.. date: 2022-11-04-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for +``PTHREAD_SCOPE_SYSTEM``. + +.. + +.. date: 2022-11-03-08-10-49 +.. gh-issue: 98872 +.. nonce: gdsR8X +.. section: Build + +Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python +3.11. + +.. + +.. date: 2022-11-02-19-25-07 +.. gh-issue: 99016 +.. nonce: R05NkD +.. section: Build + +Fix build with ``PYTHON_FOR_REGEN=python3.8``. + +.. + +.. date: 2022-11-02-18-45-35 +.. gh-issue: 97731 +.. nonce: zKpTlj +.. section: Build + +Specify the full path to the source location for ``make docclean`` (needed +for cross-builds). + +.. + +.. date: 2022-10-26-12-37-52 +.. gh-issue: 98707 +.. nonce: eVXGEx +.. section: Build + +Don't use vendored ``libmpdec`` headers if :option:`--with-system-libmpdec` +is passed to :program:`configure`. Don't use vendored ``libexpat`` headers +if :option:`--with-system-expat` is passed to :program:`!configure`. + +.. + +.. date: 2022-09-20-12-43-44 +.. gh-issue: 96761 +.. nonce: IF29kR +.. section: Build + +Fix the build process of clang compiler for :program:`_bootstrap_python` if +LTO optimization is applied. Patch by Matthias G?rgens and Dong-hee Na. + +.. + +.. date: 2022-09-17-11-19-24 +.. gh-issue: 96883 +.. nonce: p_gr62 +.. section: Build + +``wasm32-emscripten`` builds for browsers now include +:mod:`concurrent.futures` for :mod:`asyncio` and :mod:`unittest.mock`. + +.. + +.. date: 2022-08-26-11-09-11 +.. gh-issue: 84461 +.. nonce: Nsdn_R +.. section: Build + +``wasm32-emscripten`` platform no longer builds :mod:`resource` module, +:func:`~os.getresuid`, :func:`~os.getresgid`, and their setters. The APIs +are stubs and not functional. + +.. + +.. date: 2022-06-25-23-25-47 +.. gh-issue: 94280 +.. nonce: YhEyW_ +.. section: Build + +Updated pegen regeneration script on Windows to find and use Python 3.9 or +higher. Prior to this, pegen regeneration already required 3.9 or higher, +but the script may have used lower versions of Python. + +.. + +.. date: 2022-11-23-17-17-16 +.. gh-issue: 99345 +.. nonce: jOa3-f +.. section: Windows + +Use faster initialization functions to detect install location for Windows +Store package + +.. + +.. date: 2022-11-21-19-50-18 +.. gh-issue: 98629 +.. nonce: tMmB_B +.. section: Windows + +Fix initialization of :data:`sys.version` and ``sys._git`` on Windows + +.. + +.. date: 2022-11-16-19-03-21 +.. gh-issue: 99442 +.. nonce: 6Dgk3Q +.. section: Windows + +Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file +extension. + +.. + +.. date: 2022-11-01-11-07-33 +.. gh-issue: 98689 +.. nonce: 0f6e_N +.. section: Windows + +Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +vulnerable ``inflateGetHeader`` API is not used by Python. + +.. + +.. date: 2022-11-01-00-37-13 +.. gh-issue: 98790 +.. nonce: fpaPAx +.. section: Windows + +Assumes that a missing ``DLLs`` directory means that standard extension +modules are in the executable's directory. + +.. + +.. date: 2022-10-27-20-30-16 +.. gh-issue: 98745 +.. nonce: v06p4r +.. section: Windows + +Update :file:`py.exe` launcher to install 3.11 by default and 3.12 on +request. + +.. + +.. date: 2022-10-26-17-43-09 +.. gh-issue: 98692 +.. nonce: bOopfZ +.. section: Windows + +Fix the :ref:`launcher` ignoring unrecognized shebang lines instead of +treating them as local paths + +.. + +.. date: 2022-10-25-10-34-17 +.. gh-issue: 94328 +.. nonce: 19NhdU +.. section: Windows + +Update Windows installer to use SQLite 3.39.4. + +.. + +.. date: 2022-10-02-11-59-23 +.. gh-issue: 97728 +.. nonce: dIdlPE +.. section: Windows + +Fix possible crashes caused by the use of uninitialized variables when pass +invalid arguments in :func:`os.system` on Windows and in Windows-specific +modules (like ``winreg``). + +.. + +.. date: 2022-09-23-15-40-04 +.. gh-issue: 96965 +.. nonce: CsnEGs +.. section: Windows + +Update libffi to 3.4.3 + +.. + +.. date: 2022-08-30-12-01-51 +.. gh-issue: 94781 +.. nonce: OxO-Gr +.. section: Windows + +Fix :file:`pcbuild.proj` to clean previous instances of ouput files in +``Python\deepfreeze`` and ``Python\frozen_modules`` directories on Windows. +Patch by Charlie Zhao. + +.. + +.. bpo: 40882 +.. date: 2020-06-06-15-10-37 +.. nonce: UvNbdj +.. section: Windows + +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. + +.. + +.. date: 2022-11-25-09-23-20 +.. gh-issue: 87235 +.. nonce: SifjCD +.. section: macOS + +On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script +longer than a couple of bytes. + +.. + +.. date: 2022-11-01-10-32-23 +.. gh-issue: 98940 +.. nonce: W3YzC_ +.. section: macOS + +Fix ``Mac/Extras.install.py`` file filter bug. + +.. + +.. date: 2022-10-25-10-32-23 +.. gh-issue: 94328 +.. nonce: W3YNC_ +.. section: macOS + +Update macOS installer to SQLite 3.39.4. + +.. + +.. date: 2022-10-15-21-20-40 +.. gh-issue: 97527 +.. nonce: otAHJM +.. section: IDLE + +Fix a bug in the previous bugfix that caused IDLE to not start when run with +3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed +without the Lib/test package. 3.11.0 was never affected. + +.. + +.. date: 2022-08-29-17-25-13 +.. gh-issue: 95853 +.. nonce: Ce17cT +.. section: Tools/Demos + +The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for +broken EMSDK versions, and warns about pkg-config env vars. + +.. + +.. date: 2022-08-10-17-08-43 +.. gh-issue: 95853 +.. nonce: HCjC2m +.. section: Tools/Demos + +The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, +and test steps for building CPython on WebAssembly platforms. + +.. + +.. date: 2022-08-05-23-25-59 +.. gh-issue: 95731 +.. nonce: N2KohU +.. section: Tools/Demos + +Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. + +.. + +.. date: 2022-12-05-17-30-13 +.. gh-issue: 98680 +.. nonce: FiMCxZ +.. section: C API + +``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. +These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for +3.11, and are necessary to use the buffer API. + +.. + +.. date: 2022-11-03-17-46-41 +.. gh-issue: 98978 +.. nonce: KJjBvv +.. section: C API + +Fix use-after-free in ``Py_SetPythonHome(NULL)``, +``Py_SetProgramName(NULL)`` and ``_Py_SetProgramFullPath(NULL)`` function +calls. Issue reported by Benedikt Reinartz. Patch by Victor Stinner. + +.. + +.. date: 2022-10-16-15-00-25 +.. gh-issue: 96853 +.. nonce: V0wiXP +.. section: C API + +``Py_InitializeEx`` now correctly calls ``PyConfig_Clear`` after +initializing the interpreter (the omission didn't cause a memory leak only +because none of the dynamically allocated config fields are populated by the +wrapper function) diff --git a/Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst b/Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst deleted file mode 100644 index 1199e842177d..000000000000 --- a/Misc/NEWS.d/next/Build/2022-06-25-23-25-47.gh-issue-94280.YhEyW_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Updated pegen regeneration script on Windows to find and use Python 3.9 or -higher. Prior to this, pegen regeneration already required 3.9 or higher, -but the script may have used lower versions of Python. diff --git a/Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst b/Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst deleted file mode 100644 index 134e9645159a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-08-26-11-09-11.gh-issue-84461.Nsdn_R.rst +++ /dev/null @@ -1,3 +0,0 @@ -``wasm32-emscripten`` platform no longer builds :mod:`resource` module, -:func:`~os.getresuid`, :func:`~os.getresgid`, and their setters. The APIs -are stubs and not functional. diff --git a/Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst b/Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst deleted file mode 100644 index 2258ce8ab9d5..000000000000 --- a/Misc/NEWS.d/next/Build/2022-09-17-11-19-24.gh-issue-96883.p_gr62.rst +++ /dev/null @@ -1,2 +0,0 @@ -``wasm32-emscripten`` builds for browsers now include -:mod:`concurrent.futures` for :mod:`asyncio` and :mod:`unittest.mock`. diff --git a/Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst b/Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst deleted file mode 100644 index 18f75ac48910..000000000000 --- a/Misc/NEWS.d/next/Build/2022-09-20-12-43-44.gh-issue-96761.IF29kR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the build process of clang compiler for :program:`_bootstrap_python` if -LTO optimization is applied. Patch by Matthias G?rgens and Dong-hee Na. diff --git a/Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst b/Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst deleted file mode 100644 index 69afa9dea307..000000000000 --- a/Misc/NEWS.d/next/Build/2022-10-26-12-37-52.gh-issue-98707.eVXGEx.rst +++ /dev/null @@ -1,4 +0,0 @@ -Don't use vendored ``libmpdec`` headers if :option:`--with-system-libmpdec` -is passed to :program:`configure`. -Don't use vendored ``libexpat`` headers if :option:`--with-system-expat` -is passed to :program:`!configure`. diff --git a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst b/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst deleted file mode 100644 index 46b1fb833d47..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specify the full path to the source location for ``make docclean`` (needed for -cross-builds). diff --git a/Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst b/Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst deleted file mode 100644 index df189daca3a2..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-02-19-25-07.gh-issue-99016.R05NkD.rst +++ /dev/null @@ -1 +0,0 @@ -Fix build with ``PYTHON_FOR_REGEN=python3.8``. diff --git a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst b/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst deleted file mode 100644 index ad4dc496ee0e..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python 3.11. diff --git a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index e320ecfdfbb7..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for ``PTHREAD_SCOPE_SYSTEM``. diff --git a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst b/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst deleted file mode 100644 index f7396abde13b..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a compilation issue with GCC 12 on macOS. diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index 2dace165ca1a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst b/Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst deleted file mode 100644 index d7e3cc423ac8..000000000000 --- a/Misc/NEWS.d/next/C API/2022-10-16-15-00-25.gh-issue-96853.V0wiXP.rst +++ /dev/null @@ -1,4 +0,0 @@ -``Py_InitializeEx`` now correctly calls ``PyConfig_Clear`` after initializing -the interpreter (the omission didn't cause a memory leak only because none -of the dynamically allocated config fields are populated by the wrapper -function) diff --git a/Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst b/Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst deleted file mode 100644 index b9672728009a..000000000000 --- a/Misc/NEWS.d/next/C API/2022-11-03-17-46-41.gh-issue-98978.KJjBvv.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix use-after-free in ``Py_SetPythonHome(NULL)``, -``Py_SetProgramName(NULL)`` and ``_Py_SetProgramFullPath(NULL)`` function -calls. Issue reported by Benedikt Reinartz. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst deleted file mode 100644 index a87090168e06..000000000000 --- a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. -These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for -3.11, and are necessary to use the buffer API. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst deleted file mode 100644 index b5964375962f..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure in :class:`io.FileIO` when the opener -returns an invalid file descriptor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst deleted file mode 100644 index dd96c9e20d87..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of -uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. -Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst deleted file mode 100644 index 8eadab0ad8fb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst +++ /dev/null @@ -1 +0,0 @@ -Allow :mod:`pdb` to locate source for frozen modules in the standard library. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst b/Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst deleted file mode 100644 index 06e414bca0f9..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-08-15-21-08-11.gh-issue-96005.6eoc8k.rst +++ /dev/null @@ -1,4 +0,0 @@ -On WASI :data:`~errno.ENOTCAPABLE` is now mapped to :exc:`PermissionError`. -The :mod:`errno` modules exposes the new error number. ``getpath.py`` now -ignores :exc:`PermissionError` when it cannot open landmark files -``pybuilddir.txt`` and ``pyenv.cfg``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst b/Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst deleted file mode 100644 index 37534fa17525..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-08-29-13-06-58.gh-issue-95196.eGRR4b.rst +++ /dev/null @@ -1 +0,0 @@ -Disable incorrect pickling of the C implemented classmethod descriptors. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst deleted file mode 100644 index beac84ee822a..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-12-16-58-22.gh-issue-96754.0GRme5.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make sure that all frame objects created are created from valid interpreter -frames. Prevents the possibility of invalid frames in backtraces and signal -handlers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst deleted file mode 100644 index bdd33c8d2ca9..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-13-12-06-46.gh-issue-96678.NqGFyb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix undefined behaviour in C code of null pointer arithmetic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst deleted file mode 100644 index c0d41ae7d21e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-12-36-13.gh-issue-96864.PLU3i8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure, fatal error, or :exc:`SystemError` if a -line tracing event raises an exception while opcode tracing is enabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst deleted file mode 100644 index 611ab94bc636..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-16-16-54-35.gh-issue-96387.GRzewg.rst +++ /dev/null @@ -1,5 +0,0 @@ -At Python exit, sometimes a thread holding the GIL can wait forever for a -thread (usually a daemon thread) which requested to drop the GIL, whereas -the thread already exited. To fix the race condition, the thread which -requested the GIL drop now resets its request before exiting. Issue -discovered and analyzed by Mingliang ZHAO. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst deleted file mode 100644 index 0c8b704c9510..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-20-11-06-45.gh-issue-95921.dkcRQn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix overly-broad source position information for chained comparisons used as -branching conditions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst deleted file mode 100644 index 59e389a64ee0..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the Python path configuration used to initialized :data:`sys.path` at -Python startup. Paths are no longer encoded to UTF-8/strict to avoid encoding -errors if it contains surrogate characters (bytes paths are decoded with the -surrogateescape error handler). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst deleted file mode 100644 index 6f07529f15bb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-01-08-55-09.gh-issue-97591.pw6kkH.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a missing incref/decref pair in ``Exception.__setstate__()``. -Patch by Ofey Chan. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst deleted file mode 100644 index 611521808865..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure that all Python frame objects are backed by "complete" frames. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst deleted file mode 100644 index 9b4a421a9d47..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-17-02-22.gh-issue-97943.LYAWlE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Bugfix: :func:`PyFunction_GetAnnotations` should return a borrowed -reference. It was returning a new reference. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst deleted file mode 100644 index 4efc10da293a..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-05-41-01.gh-issue-93354.6BpHl2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue that could delay the specialization of :opcode:`PRECALL` -instructions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst deleted file mode 100644 index d1f949c6e13a..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-15-45-57.gh-issue-96078.fS-6mU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`os.sched_yield` now release the GIL while calling sched_yield(2). -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst deleted file mode 100644 index 7142fc619765..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Print exception class name instead of its string representation when raising -errors from :mod:`ctypes` calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst deleted file mode 100644 index af2db1f9965c..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst +++ /dev/null @@ -1 +0,0 @@ -Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by Chaim Sanders diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst deleted file mode 100644 index 56a41e3883d1..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-23-48-46.gh-issue-98374.eOBh8M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Suppress ImportError for invalid query for help() command. Patch by Dong-hee -Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst deleted file mode 100644 index e8de568e8528..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-21-11-28-53.gh-issue-99257.nmcuf-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue where member descriptors (such as those for -:attr:`~object.__slots__`) could behave incorrectly or crash instead of -raising a :exc:`TypeError` when accessed via an instance of an invalid type. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst deleted file mode 100644 index da1e61ea8504..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-28-14-52-55.gh-issue-98783.iG0kMs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix multiple crashes in debug mode when ``str`` subclasses -are used instead of ``str`` itself. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst deleted file mode 100644 index 6f23969f4ddc..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Lower the recursion depth for marshal on WASI to support -wasmtime 2.0/main. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst deleted file mode 100644 index 0e15819a862b..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix subscription of type aliases containing bare generic types or types like -:class:`~typing.TypeVar`: for example ``tuple[A, T][int]`` and -``tuple[TypeVar, T][int]``, where ``A`` is a generic type, and ``T`` is a -type variable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst deleted file mode 100644 index f5378eb837d1..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-17-58.gh-issue-99103.bFA9BX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the error reporting positions of specialized traceback anchors when the -source line contains Unicode characters. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst deleted file mode 100644 index 0445afbbc4fb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-13-25-01.gh-issue-99153.uE3CVL.rst +++ /dev/null @@ -1 +0,0 @@ -Fix location of :exc:`SyntaxError` for a :keyword:`try` block with both :keyword:`except` and :keyword:`except* <except_star>`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst deleted file mode 100644 index c72fb21942e6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`faulthandler` to emit an error message with the proper -unexpected signal number. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst deleted file mode 100644 index 571cdd02cd55..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-08-17-12.gh-issue-99204.Mf4hMD.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix calculation of :data:`sys._base_executable` when inside a POSIX virtual -environment using copies of the python binary when the base installation does -not provide the executable name used by the venv. Calculation will fall back to -alternative names ("python<MAJOR>", "python<MAJOR>.<MINOR>"). diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst deleted file mode 100644 index aa6160dd5a5e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-07-10-29-41.gh-issue-99181.bfG4bI.rst +++ /dev/null @@ -1 +0,0 @@ -Fix failure in :keyword:`except* <except_star>` with unhashable exceptions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst deleted file mode 100644 index 8ad0e147c203..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-16-35-25.gh-issue-99205.2YOoFT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue that prevented :c:type:`PyThreadState` and -:c:type:`PyInterpreterState` memory from being freed properly. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst deleted file mode 100644 index 8908bfaa8e25..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-02-11-23.gh-issue-99298.NeArAJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue that could potentially cause incorrect error handling for some -bytecode instructions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst deleted file mode 100644 index 142f91ccd92e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix zip path for venv created from a non-installed python on POSIX -platforms. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst deleted file mode 100644 index 8d9f55d6d9d0..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where an :exc:`ExceptionGroup` subclass can wrap a -:exc:`BaseException`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst deleted file mode 100644 index 8071fd130dd6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the -available tokenizer buffer. Patch by Pablo galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst deleted file mode 100644 index 9321cef77eed..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of the -first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst deleted file mode 100644 index 3fe21a8a21bf..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-05-34-00.gh-issue-99729.A3ovwQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue that could cause frames to be visible to Python code as they -are being torn down, possibly leading to memory corruption or hard crashes -of the interpreter. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst deleted file mode 100644 index 20cd361affea..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the tokenizer that could cause infinite recursion when showing -syntax warnings that happen in the first line of the source. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst deleted file mode 100644 index 8bdaa9462750..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-15-29-08.gh-issue-99886.feJkSv.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash when an object which does not have a dictionary frees its instance values. diff --git a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst b/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst deleted file mode 100644 index 390b4a9824c7..000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Restructured the documentation for the :func:`os.wait* <os.wait>` family of functions, -and improved the docs for :func:`os.waitid` with more explanation of the -possible argument constants. diff --git a/Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst b/Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst deleted file mode 100644 index c070bbc19517..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-08-12-01-12-52.gh-issue-95588.PA0FI7.rst +++ /dev/null @@ -1,6 +0,0 @@ -Clarified the conflicting advice given in the :mod:`ast` documentation about -:func:`ast.literal_eval` being "safe" for use on untrusted input while at -the same time warning that it can crash the process. The latter statement is -true and is deemed unfixable without a large amount of work unsuitable for a -bugfix. So we keep the warning and no longer claim that ``literal_eval`` is -safe. diff --git a/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst b/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst deleted file mode 100644 index 292e16998acd..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst +++ /dev/null @@ -1 +0,0 @@ -Remove extra row diff --git a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst b/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst deleted file mode 100644 index 54e421d19d9d..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst +++ /dev/null @@ -1 +0,0 @@ -Document that calling variadic functions with ctypes requires special care on macOS/arm64 (and possibly other platforms). diff --git a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst b/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst deleted file mode 100644 index e7fda8974194..000000000000 --- a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the previous bugfix that caused IDLE to not start when run with -3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed -without the Lib/test package. 3.11.0 was never affected. diff --git a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst b/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst deleted file mode 100644 index 7c9b592d6ecd..000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst +++ /dev/null @@ -1 +0,0 @@ -Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` action is given an explicit argument. diff --git a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst b/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst deleted file mode 100644 index 3810e299c78b..000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument -recursively. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst deleted file mode 100644 index d547ffc6f97e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst +++ /dev/null @@ -1 +0,0 @@ -Restrict use of sockets instead of pipes for stdin of subprocesses created by :mod:`asyncio` to AIX platform only. diff --git a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst b/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst deleted file mode 100644 index 6e2af088640b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document the optional *callback* parameter of :class:`WeakMethod`. Patch by -G?ry Ogam. diff --git a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst b/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst deleted file mode 100644 index e05d5e2a1314..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. diff --git a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst b/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst deleted file mode 100644 index 508ba626bab4..000000000000 --- a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent error when activating venv in nested fish instances. diff --git a/Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst b/Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst deleted file mode 100644 index 43c0765d940d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-08-08-39-35.gh-issue-88050.0aOC_m.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :mod:`asyncio` subprocess transport to kill process cleanly when process is blocked and avoid ``RuntimeError`` when loop is closed. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst b/Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst deleted file mode 100644 index 8552f51196b5..000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-22-09-09-08.gh-issue-91212.53O8Ab.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed flickering of the turtle window when the tracer is turned off. Patch by Shin-myoung-serp. diff --git a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst b/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst deleted file mode 100644 index 23f8cb01cf0a..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, -break reference cycles generated by local exception and future instances -(which has exception instance as its member var). Patch by Dong Uk, Kang. diff --git a/Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst b/Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst deleted file mode 100644 index c190fb7dbcb9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-20-10-31-01.gh-issue-96052.a6FhaD.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix handling compiler warnings (SyntaxWarning and DeprecationWarning) in -:func:`codeop.compile_command` when checking for incomplete input. -Previously it emitted warnings and raised a SyntaxError. Now it always -returns ``None`` for incomplete input without emitting any warnings. diff --git a/Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst b/Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst deleted file mode 100644 index 58e51da6ba39..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-23-03-13-18.gh-issue-96192.TJywOF.rst +++ /dev/null @@ -1 +0,0 @@ -Fix handling of ``bytes`` :term:`path-like objects <path-like object>` in :func:`os.ismount()`. diff --git a/Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst b/Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst deleted file mode 100644 index 8f20588c4c58..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-29-12-35-28.gh-issue-96073.WaGstf.rst +++ /dev/null @@ -1 +0,0 @@ -In :mod:`inspect`, fix overeager replacement of "``typing.``" in formatting annotations. diff --git a/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst b/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst deleted file mode 100644 index 3a35c4734871..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Work around missing socket functions in :class:`~socket.socket`'s -``__repr__``. diff --git a/Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst b/Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst deleted file mode 100644 index 232bba1b9244..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-30-11-46-36.gh-issue-95987.CV7_u4.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``repr`` of ``Any`` subclasses. diff --git a/Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst b/Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst deleted file mode 100644 index e7f53311e589..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-15-00-37-33.gh-issue-96741.4b6czN.rst +++ /dev/null @@ -1 +0,0 @@ -Corrected type annotation for dataclass attribute ``pstats.FunctionProfile.ncalls`` to be ``str``. diff --git a/Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst b/Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst deleted file mode 100644 index 07b62a883b85..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-17-13-15-10.gh-issue-96819.6RfqM7.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed check in :mod:`multiprocessing.resource_tracker` that guarantees that the length of a write to a pipe is not greater than ``PIPE_BUF``. diff --git a/Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst b/Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst deleted file mode 100644 index af8ae2026f16..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-22-11-50-29.gh-issue-85760.DETTPd.rst +++ /dev/null @@ -1 +0,0 @@ -Fix race condition in :mod:`asyncio` where :meth:`~asyncio.SubprocessProtocol.process_exited` called before the :meth:`~asyncio.SubprocessProtocol.pipe_data_received` leading to inconsistent output. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst b/Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst deleted file mode 100644 index d57999aa29b7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-22-14-35-02.gh-issue-97005.yf21Q7.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.4.9 diff --git a/Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst b/Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst deleted file mode 100644 index d8a0e690e291..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-25-20-42-33.gh-issue-73588.uVtjEA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix generation of the default name of :class:`tkinter.Checkbutton`. -Previously, checkbuttons in different parent widgets could have the same -short name and share the same state if arguments "name" and "variable" are -not specified. Now they are globally unique. diff --git a/Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst b/Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst deleted file mode 100644 index 65c3105f3bc3..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-29-08-15-55.gh-issue-97639.JSjWYW.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``tokenize.NL`` check from :mod:`tabnanny`. diff --git a/Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst b/Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst deleted file mode 100644 index aa245cf94400..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-29-23-22-24.gh-issue-97592.tpJg_J.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid a crash in the C version of :meth:`asyncio.Future.remove_done_callback` when an evil argument is passed. diff --git a/Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst b/Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst deleted file mode 100644 index 159ab32ffbfc..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-30-15-56-20.gh-issue-96827.lzy1iw.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid spurious tracebacks from :mod:`asyncio` when default executor cleanup is delayed until after the event loop is closed (e.g. as the result of a keyboard interrupt). diff --git a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst deleted file mode 100644 index dcbea66d66bf..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would return True from this method; now they correctly return False. diff --git a/Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst b/Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst deleted file mode 100644 index 4633dce7b663..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-04-07-55-19.gh-issue-97825.mNdv1l.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes :exc:`AttributeError` when :meth:`subprocess.check_output` is used with argument ``input=None`` and either of the arguments *encoding* or *errors* are used. diff --git a/Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst b/Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst deleted file mode 100644 index b1350c959e2b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-04-21-21-41.gh-issue-97837.19q-eg.rst +++ /dev/null @@ -1,7 +0,0 @@ -Change deprecate warning message in :mod:`unittest` from - -``It is deprecated to return a value!=None`` - -to - -``It is deprecated to return a value that is not None from a test case`` diff --git a/Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst b/Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst deleted file mode 100644 index 964aa3986331..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-06-23-42-00.gh-issue-90985.s280JY.rst +++ /dev/null @@ -1 +0,0 @@ -Earlier in 3.11 we deprecated ``asyncio.Task.cancel("message")``. We realized we were too harsh, and have undeprecated it. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst b/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst deleted file mode 100644 index f4a1d272e13b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst +++ /dev/null @@ -1 +0,0 @@ -Make sure ``patch.dict()`` can be applied on async functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst b/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst deleted file mode 100644 index 6c63fa4928c6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst +++ /dev/null @@ -1,3 +0,0 @@ -Wrap network errors consistently in urllib FTP support, so the test suite -doesn't fail when a network is available but the public internet is not -reachable. diff --git a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst b/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst deleted file mode 100644 index 700c9748735f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. diff --git a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst b/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst deleted file mode 100644 index 833a6e6bb3f7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst +++ /dev/null @@ -1,4 +0,0 @@ -On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded applications. -On macOS, the libc ``syslog()`` function is not thread-safe, so -:func:`syslog.syslog` no longer releases the GIL to call it. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst b/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst deleted file mode 100644 index 1a2b6a2537b9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` and ``pip`` when -they do not impact path resolution diff --git a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst b/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst deleted file mode 100644 index f04a0fd0915e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers -containing whitespace, underscores, plus and minus signs, or non-ASCII digits to be -incorrectly accepted. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst b/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst deleted file mode 100644 index 3fe41d53c980..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst +++ /dev/null @@ -1,2 +0,0 @@ -A :meth:`~logging.handlers.SysLogHandler.createSocket` method was added to -:class:`~logging.handlers.SysLogHandler`. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst deleted file mode 100644 index b725465ae4f0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst +++ /dev/null @@ -1,2 +0,0 @@ -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst b/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst deleted file mode 100644 index cf33db7548f6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`tkinter.Text.count` raises now an exception for options starting with -"-" instead of silently ignoring them. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst deleted file mode 100644 index f74195cc8e7d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst +++ /dev/null @@ -1 +0,0 @@ -Fix infinite loop in unittest when a self-referencing chained exception is raised diff --git a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst b/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst deleted file mode 100644 index 668ea4c7a4ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with :class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst b/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst deleted file mode 100644 index fb3a2b837fc3..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a mutex to unittest.mock.NonCallableMock to protect concurrent access -to mock attributes. diff --git a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst b/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst deleted file mode 100644 index 3107519a8714..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` -callback only once on Windows. diff --git a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst b/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst deleted file mode 100644 index 887d506d4bdd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix internal error in the :mod:`re` module which in very rare circumstances -prevented compilation of a regular expression containing a :ref:`conditional -expression <re-conditional-expression>` without the "else" branch. diff --git a/Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst b/Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst deleted file mode 100644 index cf99ea51bf7c..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prevent crashing in :mod:`traceback` when retrieving the byte-offset for -some source files that contain certain unicode characters. diff --git a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst b/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst deleted file mode 100644 index 7b67af06cf3d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst +++ /dev/null @@ -1 +0,0 @@ -Fix argument typechecks in :func:`!_overlapped.WSAConnect` and :func:`!_overlapped.Overlapped.WSASendTo` functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst b/Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst deleted file mode 100644 index 9b5deb0372f2..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-30-12-22-24.gh-issue-98706.v1Kuy5.rst +++ /dev/null @@ -1,7 +0,0 @@ -[3.11] Applied changes from importlib_metadata `4.11.4 through 4.13 -<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-13-0>`_, -including compatibility and robustness fixes for ``Distribution`` objects -without ``_normalized_name``, disallowing invalid inputs to -``Distribution.from_name``, and refined behaviors in -``PathDistribution._name_from_stem`` and -``PathDistribution._normalized_name``. diff --git a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst b/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst deleted file mode 100644 index f61af2543c7f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in :func:`math.dist` when both points don't have the same dimension. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst deleted file mode 100644 index 4de17abd0634..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_io`. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst deleted file mode 100644 index de0006342063..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleaks on failed module initialisation in in :mod:`_pickle` diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst deleted file mode 100644 index bd54d3eae8c9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` diff --git a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst b/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst deleted file mode 100644 index d9f12a6775fe..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 22.3.1. diff --git a/Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst b/Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst deleted file mode 100644 index 6393ec614353..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-23-16-15.gh-issue-93464.ucd4vP.rst +++ /dev/null @@ -1 +0,0 @@ -``enum.auto()`` is now correctly activated when combined with other assignment values. E.g. ``ONE = auto(), 'some text'`` will now evaluate as ``(1, 'some text')``. diff --git a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst b/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst deleted file mode 100644 index a84caa6ac2ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. diff --git a/Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst b/Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst deleted file mode 100644 index 99bf1d5d08ba..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-08-11-15-37.gh-issue-99248.1vt8xI.rst +++ /dev/null @@ -1 +0,0 @@ -fix negative numbers failing in verify() diff --git a/Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst b/Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst deleted file mode 100644 index f0a5390b03a7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-08-40-52.gh-issue-99277.J1P44O.rst +++ /dev/null @@ -1 +0,0 @@ -Remove older version of ``_SSLProtocolTransport.get_write_buffer_limits`` in :mod:`!asyncio.sslproto` diff --git a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst b/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst deleted file mode 100644 index 2bf05a3bdfbd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``SystemError`` in :mod:`ctypes` when exception was not set during -``__initsubclass__``. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst b/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst deleted file mode 100644 index 3102ef41f162..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :func:`inspect.signature` reported incorrect arguments for decorated methods. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst deleted file mode 100644 index 412c8c793435..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.TypeVarTuple` and -:class:`~typing.ParamSpec` together in user generics. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst deleted file mode 100644 index 1950680b1df8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.ParamSpec` followed by -:class:`~typing.TypeVarTuple` in generic aliases. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst deleted file mode 100644 index f153f2fceac8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Check the number of arguments in substitution in user generics containing a -:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst b/Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst deleted file mode 100644 index 0a06e7c5c6ac..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-15-45-51.gh-issue-99418.FxfAXS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug in :func:`urllib.parse.urlparse` that causes URL schemes that begin -with a digit, a plus sign, or a minus sign to be parsed incorrectly. diff --git a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst b/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst deleted file mode 100644 index 451561c579da..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when -changing line numbers. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst b/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst deleted file mode 100644 index 014161cf7b1d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst +++ /dev/null @@ -1 +0,0 @@ -Release the GIL when calling termios APIs to avoid blocking threads. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst b/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst deleted file mode 100644 index f6ee449891d9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now -``addClassCleanup()`` uses separate lists for different ``TestCase`` -subclasses, and ``doClassCleanups()`` only cleans up the particular class. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst deleted file mode 100644 index 63fe7b8a3a32..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug when calling trace.CoverageResults with valid infile. diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst deleted file mode 100644 index dd4f43351ac7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`asyncio.get_event_loop` now only emits a deprecation warning when a -new event loop was created implicitly. It no longer emits a deprecation -warning if the current event loop was set. diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst deleted file mode 100644 index e305352c7a55..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst +++ /dev/null @@ -1,5 +0,0 @@ -Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so -that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex -replacement we put in or the characters r"\x" came through in the original -request line. diff --git a/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst b/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst deleted file mode 100644 index 2f113492d42d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a shell code injection vulnerability in the ``get-remote-certificate.py`` -example script. The script no longer uses a shell to run ``openssl`` commands. -Issue reported and initial fix by Caleb Shortt. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst deleted file mode 100644 index 479299e54547..000000000000 --- a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst +++ /dev/null @@ -1 +0,0 @@ -Some C API tests were moved into the new Lib/test/test_capi/ directory. diff --git a/Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst b/Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst deleted file mode 100644 index 89958c577219..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-10-12-14-57-06.gh-issue-96853.ANe-bw.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added explicit coverage of ``Py_Initialize`` (and hence ``Py_InitializeEx``) -back to the embedding tests (all other embedding tests migrated to -``Py_InitializeFromConfig`` in Python 3.11) diff --git a/Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst b/Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst deleted file mode 100644 index 181e12c7430b..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-10-15-07-46-48.gh-issue-87390.asR-Zo.rst +++ /dev/null @@ -1 +0,0 @@ -Add tests for star-unpacking with PEP 646, and some other miscellaneous PEP 646 tests. diff --git a/Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst b/Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst deleted file mode 100644 index 57e58943df49..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-10-26-15-19-20.gh-issue-98713.Lnu32R.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the :mod:`typing` tests where a test relying on CPython-specific -implementation details was not decorated with ``@cpython_only`` and was not -skipped on other implementations. diff --git a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst b/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst deleted file mode 100644 index 3db1ec12b520..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optional big memory tests in ``test_sqlite3`` now catch the correct -:exc:`sqlite.DataError` exception type in case of too large strings and/or -blobs passed. diff --git a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst deleted file mode 100644 index 1b1287c33dbd..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst +++ /dev/null @@ -1 +0,0 @@ -Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst deleted file mode 100644 index eded0361fbeb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip test_normalization() of test_unicodedata if it fails to download -NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst deleted file mode 100644 index 6b214616c0a9..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst deleted file mode 100644 index c38db3af425e..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst +++ /dev/null @@ -1,2 +0,0 @@ -The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and -test steps for building CPython on WebAssembly platforms. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst deleted file mode 100644 index 1cd1ce14fac0..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for -broken EMSDK versions, and warns about pkg-config env vars. diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst deleted file mode 100644 index 2670aeef9a25..000000000000 --- a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on -Windows. diff --git a/Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst b/Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst deleted file mode 100644 index d343173d40d6..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-08-30-12-01-51.gh-issue-94781.OxO-Gr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :file:`pcbuild.proj` to clean previous instances of ouput files in ``Python\deepfreeze`` and -``Python\frozen_modules`` directories on Windows. Patch by Charlie Zhao. diff --git a/Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst b/Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst deleted file mode 100644 index de7aff0563e5..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-09-23-15-40-04.gh-issue-96965.CsnEGs.rst +++ /dev/null @@ -1 +0,0 @@ -Update libffi to 3.4.3 diff --git a/Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst b/Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst deleted file mode 100644 index 2a6a253a52ae..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-02-11-59-23.gh-issue-97728.dIdlPE.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix possible crashes caused by the use of uninitialized variables when pass -invalid arguments in :func:`os.system` on Windows and in Windows-specific -modules (like ``winreg``). diff --git a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst b/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst deleted file mode 100644 index eb48ff9b6ec6..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.39.4. diff --git a/Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst b/Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst deleted file mode 100644 index 3a5efd9a1cfa..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-26-17-43-09.gh-issue-98692.bOopfZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :ref:`launcher` ignoring unrecognized shebang lines instead of -treating them as local paths diff --git a/Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst b/Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst deleted file mode 100644 index caf73db280f3..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-27-20-30-16.gh-issue-98745.v06p4r.rst +++ /dev/null @@ -1 +0,0 @@ -Update :file:`py.exe` launcher to install 3.11 by default and 3.12 on request. diff --git a/Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst b/Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst deleted file mode 100644 index e8c5ac452858..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-01-00-37-13.gh-issue-98790.fpaPAx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Assumes that a missing ``DLLs`` directory means that standard extension -modules are in the executable's directory. diff --git a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst b/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst deleted file mode 100644 index 295debb81369..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but -the vulnerable ``inflateGetHeader`` API is not used by Python. diff --git a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst b/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst deleted file mode 100644 index 8e19366c4297..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file -extension. diff --git a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst b/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst deleted file mode 100644 index 46cbf998eb20..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst +++ /dev/null @@ -1 +0,0 @@ -Fix initialization of :data:`sys.version` and ``sys._git`` on Windows diff --git a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst b/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst deleted file mode 100644 index 99db0c55a67e..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use faster initialization functions to detect install location for Windows -Store package diff --git a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst b/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst deleted file mode 100644 index cbef54d17a82..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.39.4. diff --git a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst b/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst deleted file mode 100644 index 18ef0b0e2523..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``Mac/Extras.install.py`` file filter bug. diff --git a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst b/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst deleted file mode 100644 index 3111e4975e87..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst +++ /dev/null @@ -1 +0,0 @@ -On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script longer than a couple of bytes. diff --git a/README.rst b/README.rst index 335041972059..8d9ace93b6c9 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.11.0 +This is Python version 3.11.1 ============================= .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From webhook-mailer at python.org Tue Dec 6 16:19:03 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 06 Dec 2022 21:19:03 -0000 Subject: [Python-checkins] Python 3.10.9 Message-ID: <mailman.2832.1670361544.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1dd9be6584413fbfa823f37a224f101b819505d1 commit: 1dd9be6584413fbfa823f37a224f101b819505d1 branch: 3.10 author: Pablo Galindo <pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-06T18:31:21Z summary: Python 3.10.9 files: A Misc/NEWS.d/3.10.9.rst D Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst D Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst D Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst D Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst D Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst D Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst D Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst D Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst D Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst D Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst D Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst D Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst D Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst D Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst D Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst D Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst D Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst D Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst D Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst D Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst D Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst D Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst D Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst D Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst D Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst D Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst D Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst D Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst D Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst D Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst D Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst D Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst D Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst D Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst D Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst D Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst D Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst D Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst D Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst D Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst D Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst D Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst D Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst D Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst D Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst D Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst D Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst D Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst D Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst D Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst D Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst D Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 6db63f934c16..48a34d0ef5fc 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 10 -#define PY_MICRO_VERSION 8 +#define PY_MICRO_VERSION 9 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.10.8+" +#define PY_VERSION "3.10.9" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 34bb19eaf50c..e8bedaae60be 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Oct 11 12:21:26 2022 +# Autogenerated by Sphinx on Tue Dec 6 18:31:02 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -358,7 +358,7 @@ 'yield_expression)]\n' '\n' 'The difference from normal Assignment statements is that only ' - 'single\n' + 'a single\n' 'target is allowed.\n' '\n' 'For simple names as assignment targets, if in class or module ' @@ -408,12 +408,13 @@ 'analysis\n' ' tools and IDEs.\n' '\n' - 'Changed in version 3.8: Now annotated assignments allow same\n' - 'expressions in the right hand side as the regular ' - 'assignments.\n' - 'Previously, some expressions (like un-parenthesized tuple ' - 'expressions)\n' - 'caused a syntax error.\n', + 'Changed in version 3.8: Now annotated assignments allow the ' + 'same\n' + 'expressions in the right hand side as regular assignments. ' + 'Previously,\n' + 'some expressions (like un-parenthesized tuple expressions) ' + 'caused a\n' + 'syntax error.\n', 'async': 'Coroutines\n' '**********\n' '\n' @@ -2041,7 +2042,7 @@ '\n' '* Mappings (instances of "dict") compare equal if and only if ' 'they\n' - ' have equal *(key, value)* pairs. Equality comparison of the ' + ' have equal "(key, value)" pairs. Equality comparison of the ' 'keys and\n' ' values enforces reflexivity.\n' '\n' @@ -7225,7 +7226,7 @@ 'the clauses had been separated out into individual import ' 'statements.\n' '\n' - 'The details of the first step, finding and loading modules are\n' + 'The details of the first step, finding and loading modules, are\n' 'described in greater detail in the section on the import system, ' 'which\n' 'also describes the various types of packages and modules that can ' @@ -10999,8 +11000,9 @@ 'y)" is\n' 'typically invalid without special support in "MyClass". To ' 'be able to\n' - 'use that kind of patterns, the class needs to define a\n' - '*__match_args__* attribute.\n' + 'use that kind of pattern, the class needs to define a ' + '*__match_args__*\n' + 'attribute.\n' '\n' 'object.__match_args__\n' '\n' @@ -11205,6 +11207,10 @@ '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' + ' If *sub* is empty, returns the number of empty strings ' + 'between\n' + ' characters which is the length of the string plus one.\n' + '\n' "str.encode(encoding='utf-8', errors='strict')\n" '\n' ' Return an encoded version of the string as a bytes ' @@ -11711,7 +11717,7 @@ 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11752,7 +11758,7 @@ " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -13733,17 +13739,11 @@ 'dictionaries or\n' 'other mutable types (that are compared by value rather than ' 'by object\n' - 'identity) may not be used as keys. Numeric types used for ' - 'keys obey\n' - 'the normal rules for numeric comparison: if two numbers ' - 'compare equal\n' - '(such as "1" and "1.0") then they can be used ' - 'interchangeably to index\n' - 'the same dictionary entry. (Note however, that since ' - 'computers store\n' - 'floating-point numbers as approximations it is usually ' - 'unwise to use\n' - 'them as dictionary keys.)\n' + 'identity) may not be used as keys. Values that compare equal ' + '(such as\n' + '"1", "1.0", and "True") can be used interchangeably to index ' + 'the same\n' + 'dictionary entry.\n' '\n' 'class dict(**kwargs)\n' 'class dict(mapping, **kwargs)\n' diff --git a/Misc/NEWS.d/3.10.9.rst b/Misc/NEWS.d/3.10.9.rst new file mode 100644 index 000000000000..7dbc291d857a --- /dev/null +++ b/Misc/NEWS.d/3.10.9.rst @@ -0,0 +1,656 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-10-21-13-31-47 +.. gh-issue: 98517 +.. nonce: SXXGfV +.. section: Security + +Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). + +.. + +.. date: 2022-09-07-10-42-00 +.. gh-issue: 97514 +.. nonce: Yggdsl +.. section: Security + +On Linux the :mod:`multiprocessing` module returns to using filesystem +backed unix domain sockets for communication with the *forkserver* process +instead of the Linux abstract socket namespace. Only code that chooses to +use the :ref:`"forkserver" start method <multiprocessing-start-methods>` is +affected. + +Abstract sockets have no permissions and could allow any user on the system +in the same `network namespace +<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often +the whole system) to inject code into the multiprocessing *forkserver* +process. This was a potential privilege escalation. Filesystem based socket +permissions restrict this to the *forkserver* process user as was the +default in Python 3.8 and earlier. + +This prevents Linux `CVE-2022-42919 +<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. + +.. + +.. date: 2022-11-21-11-27-14 +.. gh-issue: 99578 +.. nonce: DcKoBJ +.. section: Core and Builtins + +Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by +Victor Stinner. + +.. + +.. date: 2022-11-19-22-27-52 +.. gh-issue: 99581 +.. nonce: yKYPbf +.. section: Core and Builtins + +Fixed a bug that was causing a buffer overflow if the tokenizer copies a +line missing the newline caracter from a file that is as long as the +available tokenizer buffer. Patch by Pablo galindo + +.. + +.. date: 2022-11-06-22-59-02 +.. gh-issue: 96055 +.. nonce: TmQuJn +.. section: Core and Builtins + +Update :mod:`faulthandler` to emit an error message with the proper +unexpected signal number. Patch by Dong-hee Na. + +.. + +.. date: 2022-10-31-21-01-35 +.. gh-issue: 98852 +.. nonce: MYaRN6 +.. section: Core and Builtins + +Fix subscription of :class:`types.GenericAlias` instances containing bare +generic types: for example ``tuple[A, T][int]``, where ``A`` is a generic +type, and ``T`` is a type variable. + +.. + +.. date: 2022-10-19-01-01-08 +.. gh-issue: 98415 +.. nonce: ZS2eWh +.. section: Core and Builtins + +Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by +Chaim Sanders + +.. + +.. date: 2022-10-15-23-15-14 +.. gh-issue: 92119 +.. nonce: PMSwwG +.. section: Core and Builtins + +Print exception class name instead of its string representation when raising +errors from :mod:`ctypes` calls. + +.. + +.. date: 2022-06-10-16-37-44 +.. gh-issue: 93696 +.. nonce: 65BI2R +.. section: Core and Builtins + +Allow :mod:`pdb` to locate source for frozen modules in the standard +library. + +.. + +.. bpo: 31718 +.. date: 2020-02-23-23-48-15 +.. nonce: sXko5e +.. section: Core and Builtins + +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. + +.. + +.. bpo: 38031 +.. date: 2019-09-04-19-09-49 +.. nonce: Yq4L72 +.. section: Core and Builtins + +Fix a possible assertion failure in :class:`io.FileIO` when the opener +returns an invalid file descriptor. + +.. + +.. date: 2022-12-05-13-40-15 +.. gh-issue: 100001 +.. nonce: 78ReYp +.. section: Library + +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. + +.. + +.. date: 2022-12-02-13-05-00 +.. gh-issue: 93453 +.. nonce: EFj1NN +.. section: Library + +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. + +.. + +.. date: 2022-11-21-17-56-18 +.. gh-issue: 51524 +.. nonce: nTykx8 +.. section: Library + +Fix bug when calling trace.CoverageResults with valid infile. + +.. + +.. date: 2022-11-21-13-49-03 +.. gh-issue: 99645 +.. nonce: 9w1QKq +.. section: Library + +Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now +``addClassCleanup()`` uses separate lists for different ``TestCase`` +subclasses, and ``doClassCleanups()`` only cleans up the particular class. + +.. + +.. date: 2022-11-15-10-55-24 +.. gh-issue: 97001 +.. nonce: KeQuVF +.. section: Library + +Release the GIL when calling termios APIs to avoid blocking threads. + +.. + +.. date: 2022-11-13-02-06-56 +.. gh-issue: 99341 +.. nonce: 8-OlwB +.. section: Library + +Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when +changing line numbers. + +.. + +.. date: 2022-11-09-20-48-42 +.. gh-issue: 74044 +.. nonce: zBj26K +.. section: Library + +Fixed bug where :func:`inspect.signature` reported incorrect arguments for +decorated methods. + +.. + +.. date: 2022-11-09-12-16-35 +.. gh-issue: 99275 +.. nonce: klOqoL +.. section: Library + +Fix ``SystemError`` in :mod:`ctypes` when exception was not set during +``__initsubclass__``. + +.. + +.. date: 2022-11-06-12-44-51 +.. gh-issue: 99155 +.. nonce: vLZOzi +.. section: Library + +Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. + +.. + +.. date: 2022-11-05-17-16-40 +.. gh-issue: 99134 +.. nonce: Msgspf +.. section: Library + +Update the bundled copy of pip to version 22.3.1. + +.. + +.. date: 2022-11-05-11-42-15 +.. gh-issue: 99130 +.. nonce: 91MMXu +.. section: Library + +Apply bugfixes from `importlib_metadata 4.11.4 +<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-11-4>`_, +namely: In ``PathDistribution._name_from_stem``, avoid including parts of +the extension in the result. In ``PathDistribution._normalized_name``, +ensure names loaded from the stem of the filename are also normalized, +ensuring duplicate entry points by packages varying only by non-normalized +name are hidden. + +.. + +.. date: 2022-11-02-05-54-02 +.. gh-issue: 83004 +.. nonce: 0v8iyw +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_zoneinfo` + +.. + +.. date: 2022-11-02-05-53-25 +.. gh-issue: 83004 +.. nonce: qc_KHr +.. section: Library + +Clean up refleaks on failed module initialisation in in :mod:`_pickle` + +.. + +.. date: 2022-11-02-05-52-36 +.. gh-issue: 83004 +.. nonce: LBl79O +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_io`. + +.. + +.. date: 2022-10-31-12-34-03 +.. gh-issue: 98897 +.. nonce: rgNn4x +.. section: Library + +Fix memory leak in :func:`math.dist` when both points don't have the same +dimension. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-29-03-40-18 +.. gh-issue: 98793 +.. nonce: WSPB4A +.. section: Library + +Fix argument typechecks in :func:`!_overlapped.WSAConnect` and +:func:`!_overlapped.Overlapped.WSASendTo` functions. + +.. + +.. date: 2022-10-27-12-56-38 +.. gh-issue: 98740 +.. nonce: ZoqqGM +.. section: Library + +Fix internal error in the :mod:`re` module which in very rare circumstances +prevented compilation of a regular expression containing a :ref:`conditional +expression <re-conditional-expression>` without the "else" branch. + +.. + +.. date: 2022-10-26-07-51-55 +.. gh-issue: 98703 +.. nonce: 0hW773 +.. section: Library + +Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` +callback only once on Windows. + +.. + +.. date: 2022-10-25-20-17-34 +.. gh-issue: 98624 +.. nonce: YQUPFy +.. section: Library + +Add a mutex to unittest.mock.NonCallableMock to protect concurrent access to +mock attributes. + +.. + +.. date: 2022-10-23-18-30-39 +.. gh-issue: 89237 +.. nonce: kBui30 +.. section: Library + +Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with +:class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-19-18-31-53 +.. gh-issue: 98458 +.. nonce: vwyq7O +.. section: Library + +Fix infinite loop in unittest when a self-referencing chained exception is +raised + +.. + +.. date: 2022-10-19-09-29-12 +.. gh-issue: 97928 +.. nonce: xj3im7 +.. section: Library + +:meth:`tkinter.Text.count` raises now an exception for options starting with +"-" instead of silently ignoring them. + +.. + +.. date: 2022-10-16-18-52-00 +.. gh-issue: 97966 +.. nonce: humlhz +.. section: Library + +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. + +.. + +.. date: 2022-10-16-15-31-50 +.. gh-issue: 98331 +.. nonce: Y5kPOX +.. section: Library + +Update the bundled copies of pip and setuptools to versions 22.3 and 65.5.0 +respectively. + +.. + +.. date: 2022-10-14-19-57-37 +.. gh-issue: 96035 +.. nonce: 0xcX-p +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers +containing whitespace, underscores, plus and minus signs, or non-ASCII +digits to be incorrectly accepted. + +.. + +.. date: 2022-10-14-11-46-31 +.. gh-issue: 98251 +.. nonce: Uxc9al +.. section: Library + +Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +and ``pip`` when they do not impact path resolution + +.. + +.. date: 2022-10-12-10-00-40 +.. gh-issue: 98178 +.. nonce: hspH51 +.. section: Library + +On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded +applications. On macOS, the libc ``syslog()`` function is not thread-safe, +so :func:`syslog.syslog` no longer releases the GIL to call it. Patch by +Victor Stinner. + +.. + +.. date: 2022-10-10-07-07-31 +.. gh-issue: 96151 +.. nonce: K9fwoq +.. section: Library + +Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. + +.. + +.. date: 2022-10-08-19-39-27 +.. gh-issue: 98086 +.. nonce: y---WC +.. section: Library + +Make sure ``patch.dict()`` can be applied on async functions. + +.. + +.. date: 2022-08-06-12-18-07 +.. gh-issue: 88863 +.. nonce: NnqsuJ +.. section: Library + +To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, +break reference cycles generated by local exception and future instances +(which has exception instance as its member var). Patch by Dong Uk, Kang. + +.. + +.. date: 2022-06-17-12-02-30 +.. gh-issue: 93858 +.. nonce: R49ARc +.. section: Library + +Prevent error when activating venv in nested fish instances. + +.. + +.. bpo: 46364 +.. date: 2022-01-14-10-49-20 +.. nonce: SzhlU9 +.. section: Library + +Restrict use of sockets instead of pipes for stdin of subprocesses created +by :mod:`asyncio` to AIX platform only. + +.. + +.. bpo: 38523 +.. date: 2020-10-23-22-20-52 +.. nonce: CrkxLh +.. section: Library + +:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument +recursively. + +.. + +.. bpo: 36267 +.. date: 2019-09-03-15-45-19 +.. nonce: z42Ex7 +.. section: Library + +Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` +action is given an explicit argument. + +.. + +.. date: 2022-11-16-12-52-23 +.. gh-issue: 92892 +.. nonce: TS-P0j +.. section: Documentation + +Document that calling variadic functions with ctypes requires special care +on macOS/arm64 (and possibly other platforms). + +.. + +.. date: 2022-12-05-16-12-56 +.. gh-issue: 99892 +.. nonce: sz_eW8 +.. section: Tests + +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. + +.. + +.. bpo: 34272 +.. date: 2018-07-29-15-59-51 +.. nonce: lVX2uR +.. section: Tests + +Some C API tests were moved into the new Lib/test/test_capi/ directory. + +.. + +.. date: 2022-11-24-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and +``-Wimplicit-function-declaration`` compiler warnings in +:program:`configure` checks. + +.. + +.. date: 2022-11-04-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for +``PTHREAD_SCOPE_SYSTEM``. + +.. + +.. date: 2022-11-02-18-45-35 +.. gh-issue: 97731 +.. nonce: zKpTlj +.. section: Build + +Specify the full path to the source location for ``make docclean`` (needed +for cross-builds). + +.. + +.. date: 2022-10-25-14-43-00 +.. gh-issue: 98671 +.. nonce: a42a6d +.. section: Build + +Fix ``NO_MISALIGNED_ACCESSES`` being not defined for the SHA3 extension when +``HAVE_ALIGNED_REQUIRED`` is set. Allowing builds on hardware that unaligned +memory accesses are not allowed. + +.. + +.. date: 2022-11-23-17-17-16 +.. gh-issue: 99345 +.. nonce: jOa3-f +.. section: Windows + +Use faster initialization functions to detect install location for Windows +Store package + +.. + +.. date: 2022-11-01-11-07-33 +.. gh-issue: 98689 +.. nonce: 0f6e_N +.. section: Windows + +Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +vulnerable ``inflateGetHeader`` API is not used by Python. + +.. + +.. date: 2022-10-25-10-34-17 +.. gh-issue: 94328 +.. nonce: 19NhdU +.. section: Windows + +Update Windows installer to use SQLite 3.39.4. + +.. + +.. bpo: 40882 +.. date: 2020-06-06-15-10-37 +.. nonce: UvNbdj +.. section: Windows + +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. + +.. + +.. date: 2022-10-25-10-32-23 +.. gh-issue: 94328 +.. nonce: W3YNC_ +.. section: macOS + +Update macOS installer to SQLite 3.39.4. + +.. + +.. date: 2022-10-15-21-20-40 +.. gh-issue: 97527 +.. nonce: otAHJM +.. section: IDLE + +Fix a bug in the previous bugfix that caused IDLE to not start when run with +3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed +without the Lib/test package. 3.11.0 was never affected. + +.. + +.. date: 2022-08-05-23-25-59 +.. gh-issue: 95731 +.. nonce: N2KohU +.. section: Tools/Demos + +Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. diff --git a/Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst b/Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst deleted file mode 100644 index 4602d53fcdb7..000000000000 --- a/Misc/NEWS.d/next/Build/2022-10-25-14-43-00.gh-issue-98671.a42a6d.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``NO_MISALIGNED_ACCESSES`` being not defined for the SHA3 extension -when ``HAVE_ALIGNED_REQUIRED`` is set. Allowing builds on hardware that -unaligned memory accesses are not allowed. diff --git a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst b/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst deleted file mode 100644 index 46b1fb833d47..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-02-18-45-35.gh-issue-97731.zKpTlj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specify the full path to the source location for ``make docclean`` (needed for -cross-builds). diff --git a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index e320ecfdfbb7..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-04-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for ``PTHREAD_SCOPE_SYSTEM``. diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index 2dace165ca1a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst deleted file mode 100644 index b5964375962f..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure in :class:`io.FileIO` when the opener -returns an invalid file descriptor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst deleted file mode 100644 index dd96c9e20d87..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of -uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. -Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst deleted file mode 100644 index 8eadab0ad8fb..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-06-10-16-37-44.gh-issue-93696.65BI2R.rst +++ /dev/null @@ -1 +0,0 @@ -Allow :mod:`pdb` to locate source for frozen modules in the standard library. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst deleted file mode 100644 index 7142fc619765..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-15-23-15-14.gh-issue-92119.PMSwwG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Print exception class name instead of its string representation when raising -errors from :mod:`ctypes` calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst deleted file mode 100644 index af2db1f9965c..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-01-01-08.gh-issue-98415.ZS2eWh.rst +++ /dev/null @@ -1 +0,0 @@ -Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by Chaim Sanders diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst deleted file mode 100644 index 25c473717ca2..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-21-01-35.gh-issue-98852.MYaRN6.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix subscription of :class:`types.GenericAlias` instances containing bare -generic types: for example ``tuple[A, T][int]``, -where ``A`` is a generic type, and ``T`` is a type variable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst deleted file mode 100644 index c72fb21942e6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-22-59-02.gh-issue-96055.TmQuJn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`faulthandler` to emit an error message with the proper -unexpected signal number. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst deleted file mode 100644 index 8071fd130dd6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the -available tokenizer buffer. Patch by Pablo galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst deleted file mode 100644 index 9321cef77eed..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of the -first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst b/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst deleted file mode 100644 index 54e421d19d9d..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst +++ /dev/null @@ -1 +0,0 @@ -Document that calling variadic functions with ctypes requires special care on macOS/arm64 (and possibly other platforms). diff --git a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst b/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst deleted file mode 100644 index e7fda8974194..000000000000 --- a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the previous bugfix that caused IDLE to not start when run with -3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed -without the Lib/test package. 3.11.0 was never affected. diff --git a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst b/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst deleted file mode 100644 index 7c9b592d6ecd..000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-03-15-45-19.bpo-36267.z42Ex7.rst +++ /dev/null @@ -1 +0,0 @@ -Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` action is given an explicit argument. diff --git a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst b/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst deleted file mode 100644 index 3810e299c78b..000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-23-22-20-52.bpo-38523.CrkxLh.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument -recursively. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst deleted file mode 100644 index d547ffc6f97e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-01-14-10-49-20.bpo-46364.SzhlU9.rst +++ /dev/null @@ -1 +0,0 @@ -Restrict use of sockets instead of pipes for stdin of subprocesses created by :mod:`asyncio` to AIX platform only. diff --git a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst b/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst deleted file mode 100644 index 508ba626bab4..000000000000 --- a/Misc/NEWS.d/next/Library/2022-06-17-12-02-30.gh-issue-93858.R49ARc.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent error when activating venv in nested fish instances. diff --git a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst b/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst deleted file mode 100644 index 23f8cb01cf0a..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, -break reference cycles generated by local exception and future instances -(which has exception instance as its member var). Patch by Dong Uk, Kang. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst b/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst deleted file mode 100644 index f4a1d272e13b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-19-39-27.gh-issue-98086.y---WC.rst +++ /dev/null @@ -1 +0,0 @@ -Make sure ``patch.dict()`` can be applied on async functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst b/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst deleted file mode 100644 index 700c9748735f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-10-07-07-31.gh-issue-96151.K9fwoq.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. diff --git a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst b/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst deleted file mode 100644 index 833a6e6bb3f7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-12-10-00-40.gh-issue-98178.hspH51.rst +++ /dev/null @@ -1,4 +0,0 @@ -On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded applications. -On macOS, the libc ``syslog()`` function is not thread-safe, so -:func:`syslog.syslog` no longer releases the GIL to call it. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst b/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst deleted file mode 100644 index 1a2b6a2537b9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-11-46-31.gh-issue-98251.Uxc9al.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` and ``pip`` when -they do not impact path resolution diff --git a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst b/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst deleted file mode 100644 index f04a0fd0915e..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-14-19-57-37.gh-issue-96035.0xcX-p.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers -containing whitespace, underscores, plus and minus signs, or non-ASCII digits to be -incorrectly accepted. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst b/Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst deleted file mode 100644 index b4cf94310af6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-15-31-50.gh-issue-98331.Y5kPOX.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copies of pip and setuptools to versions 22.3 and 65.5.0 respectively. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst deleted file mode 100644 index b725465ae4f0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst +++ /dev/null @@ -1,2 +0,0 @@ -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst b/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst deleted file mode 100644 index cf33db7548f6..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-09-29-12.gh-issue-97928.xj3im7.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`tkinter.Text.count` raises now an exception for options starting with -"-" instead of silently ignoring them. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst deleted file mode 100644 index f74195cc8e7d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst +++ /dev/null @@ -1 +0,0 @@ -Fix infinite loop in unittest when a self-referencing chained exception is raised diff --git a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst b/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst deleted file mode 100644 index 668ea4c7a4ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-23-18-30-39.gh-issue-89237.kBui30.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with :class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst b/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst deleted file mode 100644 index fb3a2b837fc3..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-25-20-17-34.gh-issue-98624.YQUPFy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a mutex to unittest.mock.NonCallableMock to protect concurrent access -to mock attributes. diff --git a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst b/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst deleted file mode 100644 index 3107519a8714..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-26-07-51-55.gh-issue-98703.0hW773.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` -callback only once on Windows. diff --git a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst b/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst deleted file mode 100644 index 887d506d4bdd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-27-12-56-38.gh-issue-98740.ZoqqGM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix internal error in the :mod:`re` module which in very rare circumstances -prevented compilation of a regular expression containing a :ref:`conditional -expression <re-conditional-expression>` without the "else" branch. diff --git a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst b/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst deleted file mode 100644 index 7b67af06cf3d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-29-03-40-18.gh-issue-98793.WSPB4A.rst +++ /dev/null @@ -1 +0,0 @@ -Fix argument typechecks in :func:`!_overlapped.WSAConnect` and :func:`!_overlapped.Overlapped.WSASendTo` functions. diff --git a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst b/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst deleted file mode 100644 index f61af2543c7f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-31-12-34-03.gh-issue-98897.rgNn4x.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in :func:`math.dist` when both points don't have the same dimension. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst deleted file mode 100644 index 4de17abd0634..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-52-36.gh-issue-83004.LBl79O.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_io`. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst deleted file mode 100644 index de0006342063..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleaks on failed module initialisation in in :mod:`_pickle` diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst deleted file mode 100644 index bd54d3eae8c9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-05-54-02.gh-issue-83004.0v8iyw.rst +++ /dev/null @@ -1 +0,0 @@ -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` diff --git a/Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst b/Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst deleted file mode 100644 index 124d5e478f8f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-11-42-15.gh-issue-99130.91MMXu.rst +++ /dev/null @@ -1,7 +0,0 @@ -Apply bugfixes from `importlib_metadata 4.11.4 -<https://importlib-metadata.readthedocs.io/en/latest/history.html#v4-11-4>`_, -namely: In ``PathDistribution._name_from_stem``, avoid including parts of -the extension in the result. In ``PathDistribution._normalized_name``, -ensure names loaded from the stem of the filename are also normalized, -ensuring duplicate entry points by packages varying only by non-normalized -name are hidden. diff --git a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst b/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst deleted file mode 100644 index d9f12a6775fe..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-05-17-16-40.gh-issue-99134.Msgspf.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 22.3.1. diff --git a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst b/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst deleted file mode 100644 index a84caa6ac2ea..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-06-12-44-51.gh-issue-99155.vLZOzi.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst b/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst deleted file mode 100644 index 2bf05a3bdfbd..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-12-16-35.gh-issue-99275.klOqoL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``SystemError`` in :mod:`ctypes` when exception was not set during -``__initsubclass__``. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst b/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst deleted file mode 100644 index 3102ef41f162..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-20-48-42.gh-issue-74044.zBj26K.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :func:`inspect.signature` reported incorrect arguments for decorated methods. diff --git a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst b/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst deleted file mode 100644 index 451561c579da..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when -changing line numbers. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst b/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst deleted file mode 100644 index 014161cf7b1d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst +++ /dev/null @@ -1 +0,0 @@ -Release the GIL when calling termios APIs to avoid blocking threads. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst b/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst deleted file mode 100644 index f6ee449891d9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now -``addClassCleanup()`` uses separate lists for different ``TestCase`` -subclasses, and ``doClassCleanups()`` only cleans up the particular class. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst deleted file mode 100644 index 63fe7b8a3a32..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug when calling trace.CoverageResults with valid infile. diff --git a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst b/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst deleted file mode 100644 index dd4f43351ac7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-02-13-05-00.gh-issue-93453.EFj1NN.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`asyncio.get_event_loop` now only emits a deprecation warning when a -new event loop was created implicitly. It no longer emits a deprecation -warning if the current event loop was set. diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst deleted file mode 100644 index e305352c7a55..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst +++ /dev/null @@ -1,5 +0,0 @@ -Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so -that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex -replacement we put in or the characters r"\x" came through in the original -request line. diff --git a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst deleted file mode 100644 index 02d95b570520..000000000000 --- a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst +++ /dev/null @@ -1,15 +0,0 @@ -On Linux the :mod:`multiprocessing` module returns to using filesystem backed -unix domain sockets for communication with the *forkserver* process instead of -the Linux abstract socket namespace. Only code that chooses to use the -:ref:`"forkserver" start method <multiprocessing-start-methods>` is affected. - -Abstract sockets have no permissions and could allow any user on the system in -the same `network namespace -<https://man7.org/linux/man-pages/man7/network_namespaces.7.html>`_ (often the -whole system) to inject code into the multiprocessing *forkserver* process. -This was a potential privilege escalation. Filesystem based socket permissions -restrict this to the *forkserver* process user as was the default in Python 3.8 -and earlier. - -This prevents Linux `CVE-2022-42919 -<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-42919>`_. diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst deleted file mode 100644 index 2d23a6ad93c7..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst +++ /dev/null @@ -1 +0,0 @@ -Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454). diff --git a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst b/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst deleted file mode 100644 index b63a54b3676c..000000000000 --- a/Misc/NEWS.d/next/Security/2022-10-26-21-04-23.gh-issue-98739.keBWcY.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.5.0 diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst deleted file mode 100644 index 5185fac2e29d..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst +++ /dev/null @@ -1,6 +0,0 @@ -The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio` -related name resolution functions no longer involves a quadratic algorithm. -This prevents a potential CPU denial of service if an out-of-spec excessive -length hostname involving bidirectional characters were decoded. Some protocols -such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker -to supply such a name. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst deleted file mode 100644 index 479299e54547..000000000000 --- a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst +++ /dev/null @@ -1 +0,0 @@ -Some C API tests were moved into the new Lib/test/test_capi/ directory. diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst deleted file mode 100644 index eded0361fbeb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip test_normalization() of test_unicodedata if it fails to download -NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst deleted file mode 100644 index 6b214616c0a9..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst deleted file mode 100644 index 2670aeef9a25..000000000000 --- a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on -Windows. diff --git a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst b/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst deleted file mode 100644 index eb48ff9b6ec6..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-10-25-10-34-17.gh-issue-94328.19NhdU.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.39.4. diff --git a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst b/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst deleted file mode 100644 index 295debb81369..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-01-11-07-33.gh-issue-98689.0f6e_N.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but -the vulnerable ``inflateGetHeader`` API is not used by Python. diff --git a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst b/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst deleted file mode 100644 index 99db0c55a67e..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use faster initialization functions to detect install location for Windows -Store package diff --git a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst b/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst deleted file mode 100644 index cbef54d17a82..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-10-25-10-32-23.gh-issue-94328.W3YNC_.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.39.4. diff --git a/README.rst b/README.rst index 986940dbb5d8..4e40478cf9af 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.10.8 +This is Python version 3.10.9 ============================= .. image:: https://travis-ci.com/python/cpython.svg?branch=master From webhook-mailer at python.org Tue Dec 6 16:24:09 2022 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 06 Dec 2022 21:24:09 -0000 Subject: [Python-checkins] Python 3.12.0a3 Message-ID: <mailman.2833.1670361850.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b6bd7ffcbc1ffaa68e3423e7415ef8ba0f8a188d commit: b6bd7ffcbc1ffaa68e3423e7415ef8ba0f8a188d branch: main author: Thomas Wouters <thomas at python.org> committer: Yhg1s <thomas at python.org> date: 2022-12-06T19:33:02+01:00 summary: Python 3.12.0a3 files: A Misc/NEWS.d/3.12.0a3.rst D Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst D Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst D Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst D Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst D Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst D Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst D Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst D Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst D Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst D Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst D Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst D Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst D Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst D Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst D Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst D Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst D Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst D Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst D Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst D Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst D Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst D Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst D Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst D Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst D Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst D Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst D Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst D Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst D Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst D Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst D Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst D Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst D Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst D Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst D Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst D Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst D Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst D Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst D Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst D Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst D Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst D Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst D Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst D Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst D Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst D Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst D Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst D Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst D Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst D Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst D Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst D Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst D Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst D Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst D Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst D Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst D Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst D Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst D Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst D Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst D Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst D Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst D Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst D Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst D Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst D Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst D Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index a16b8d7104e3..1ae4889c7573 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.12.0a2+" +#define PY_VERSION "3.12.0a3" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index a817dc3547fa..4ba680cceda9 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Nov 14 12:13:19 2022 +# Autogenerated by Sphinx on Tue Dec 6 19:31:49 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -11109,8 +11109,9 @@ 'y)" is\n' 'typically invalid without special support in "MyClass". To ' 'be able to\n' - 'use that kind of patterns, the class needs to define a\n' - '*__match_args__* attribute.\n' + 'use that kind of pattern, the class needs to define a ' + '*__match_args__*\n' + 'attribute.\n' '\n' 'object.__match_args__\n' '\n' @@ -11510,7 +11511,7 @@ 'property\n' ' being one of ?Lm?, ?Lt?, ?Lu?, ?Ll?, or ?Lo?. Note ' 'that this is\n' - ' different from the ?Alphabetic? property defined in the ' + ' different from the Alphabetic property defined in the ' 'Unicode\n' ' Standard.\n' '\n' @@ -11559,9 +11560,9 @@ 'according to the\n' ' language definition, section Identifiers and keywords.\n' '\n' - ' Call "keyword.iskeyword()" to test whether string "s" ' - 'is a reserved\n' - ' identifier, such as "def" and "class".\n' + ' "keyword.iskeyword()" can be used to test whether ' + 'string "s" is a\n' + ' reserved identifier, such as "def" and "class".\n' '\n' ' Example:\n' '\n' diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst new file mode 100644 index 000000000000..3d1e43350d13 --- /dev/null +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -0,0 +1,836 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-30-11-09-40 +.. gh-issue: 99891 +.. nonce: 9VomwB +.. section: Core and Builtins + +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo + +.. + +.. date: 2022-11-27-13-50-13 +.. gh-issue: 91054 +.. nonce: oox_kW +.. section: Core and Builtins + +Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to +register callbacks to receive notification on creation and destruction of +code objects. + +.. + +.. date: 2022-11-26-04-00-41 +.. gh-issue: 99729 +.. nonce: A3ovwQ +.. section: Core and Builtins + +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. + +.. + +.. date: 2022-11-23-18-16-18 +.. gh-issue: 99708 +.. nonce: 7MuaiR +.. section: Core and Builtins + +Fix bug where compiler crashes on an if expression with an empty body block. + +.. + +.. date: 2022-11-21-11-27-14 +.. gh-issue: 99578 +.. nonce: DcKoBJ +.. section: Core and Builtins + +Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by +Victor Stinner. + +.. + +.. date: 2022-11-19-22-27-52 +.. gh-issue: 99581 +.. nonce: yKYPbf +.. section: Core and Builtins + +Fixed a bug that was causing a buffer overflow if the tokenizer copies a +line missing the newline caracter from a file that is as long as the +available tokenizer buffer. Patch by Pablo galindo + +.. + +.. date: 2022-11-18-11-24-25 +.. gh-issue: 99553 +.. nonce: F64h-n +.. section: Core and Builtins + +Fix bug where an :exc:`ExceptionGroup` subclass can wrap a +:exc:`BaseException`. + +.. + +.. date: 2022-11-16-21-35-30 +.. gh-issue: 99547 +.. nonce: p_c_bp +.. section: Core and Builtins + +Add a function to os.path to check if a path is a junction: isjunction. Add +similar functionality to pathlib.Path as is_junction. + +.. + +.. date: 2022-11-12-01-39-57 +.. gh-issue: 99370 +.. nonce: _cu32j +.. section: Core and Builtins + +Fix zip path for venv created from a non-installed python on POSIX +platforms. + +.. + +.. date: 2022-11-11-14-04-01 +.. gh-issue: 99377 +.. nonce: -CJvWn +.. section: Core and Builtins + +Add audit events for thread creation and clear operations. + +.. + +.. date: 2022-11-10-17-09-16 +.. gh-issue: 98686 +.. nonce: bmAKwr +.. section: Core and Builtins + +Remove the ``BINARY_OP_GENERIC`` and ``COMPARE_OP_GENERIC`` +"specializations". + +.. + +.. date: 2022-11-10-16-53-40 +.. gh-issue: 99298 +.. nonce: HqRJES +.. section: Core and Builtins + +Remove the remaining error paths for attribute specializations, and refuse +to specialize attribute accesses on types that haven't had +:c:func:`PyType_Ready` called on them yet. + +.. + +.. date: 2022-11-05-22-26-35 +.. gh-issue: 99127 +.. nonce: Btk7ih +.. section: Core and Builtins + +Allow some features of :mod:`syslog` to the main interpreter only. Patch by +Dong-hee Na. + +.. + +.. date: 2022-10-05-11-44-52 +.. gh-issue: 91053 +.. nonce: f5Bo3p +.. section: Core and Builtins + +Optimizing interpreters and JIT compilers may need to invalidate internal +metadata when functions are modified. This change adds the ability to +provide a callback that will be invoked each time a function is created, +modified, or destroyed. + +.. + +.. date: 2022-09-17-17-08-01 +.. gh-issue: 90994 +.. nonce: f0H2Yd +.. section: Core and Builtins + +Improve error messages when there's a syntax error with call arguments. The +following three cases are covered: - No value is assigned to a named +argument, eg ``foo(a=)``. - A value is assigned to a star argument, eg +``foo(*args=[0])``. - A value is assigned to a double-star keyword argument, +eg ``foo(**kwarg={'a': 0})``. + +.. + +.. bpo: 45026 +.. date: 2021-08-29-15-55-19 +.. nonce: z7nTA3 +.. section: Core and Builtins + +Optimize the :class:`range` object iterator. It is now smaller, faster +iteration of ranges containing large numbers. Smaller pickles, faster +unpickling. + +.. + +.. bpo: 31718 +.. date: 2020-02-23-23-48-15 +.. nonce: sXko5e +.. section: Core and Builtins + +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. + +.. + +.. bpo: 38031 +.. date: 2019-09-04-19-09-49 +.. nonce: Yq4L72 +.. section: Core and Builtins + +Fix a possible assertion failure in :class:`io.FileIO` when the opener +returns an invalid file descriptor. + +.. + +.. date: 2022-12-05-13-40-15 +.. gh-issue: 100001 +.. nonce: 78ReYp +.. section: Library + +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. + +.. + +.. date: 2022-12-03-05-58-48 +.. gh-issue: 99957 +.. nonce: jLYYgN +.. section: Library + +Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. + +.. + +.. date: 2022-11-22-19-31-26 +.. gh-issue: 79033 +.. nonce: MW6kHq +.. section: Library + +Fix :func:`asyncio.Server.wait_closed` to actually do what the docs promise +-- wait for all existing connections to complete, after closing the server. + +.. + +.. date: 2022-11-21-17-56-18 +.. gh-issue: 51524 +.. nonce: nTykx8 +.. section: Library + +Fix bug when calling trace.CoverageResults with valid infile. + +.. + +.. date: 2022-11-21-13-49-03 +.. gh-issue: 99645 +.. nonce: 9w1QKq +.. section: Library + +Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now +``addClassCleanup()`` uses separate lists for different ``TestCase`` +subclasses, and ``doClassCleanups()`` only cleans up the particular class. + +.. + +.. date: 2022-11-21-10-45-54 +.. gh-issue: 99508 +.. nonce: QqVbby +.. section: Library + +Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling +``_imp.source_hash()``. + +.. + +.. date: 2022-11-17-10-56-47 +.. gh-issue: 66285 +.. nonce: KvjlaB +.. section: Library + +Fix :mod:`asyncio` to not share event loop and signal wakeupfd in forked +processes. Patch by Kumar Aditya. + +.. + +.. date: 2022-11-15-10-55-24 +.. gh-issue: 97001 +.. nonce: KeQuVF +.. section: Library + +Release the GIL when calling termios APIs to avoid blocking threads. + +.. + +.. date: 2022-11-15-04-08-25 +.. gh-issue: 92647 +.. nonce: cZcjnJ +.. section: Library + +Use final status of an enum to determine lookup or creation branch of +functional API. + +.. + +.. date: 2022-11-14-08-21-56 +.. gh-issue: 99388 +.. nonce: UWSlwp +.. section: Library + +Add *loop_factory* parameter to :func:`asyncio.run` to allow specifying a +custom event loop factory. Patch by Kumar Aditya. + +.. + +.. date: 2022-11-13-02-06-56 +.. gh-issue: 99341 +.. nonce: 8-OlwB +.. section: Library + +Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when +changing line numbers. + +.. + +.. date: 2022-11-12-12-15-30 +.. gh-issue: 99382 +.. nonce: dKg_rW +.. section: Library + +Check the number of arguments in substitution in user generics containing a +:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. + +.. + +.. date: 2022-11-12-12-10-23 +.. gh-issue: 99379 +.. nonce: bcGhxF +.. section: Library + +Fix substitution of :class:`~typing.ParamSpec` followed by +:class:`~typing.TypeVarTuple` in generic aliases. + +.. + +.. date: 2022-11-12-12-08-34 +.. gh-issue: 99344 +.. nonce: 7M_u8G +.. section: Library + +Fix substitution of :class:`~typing.TypeVarTuple` and +:class:`~typing.ParamSpec` together in user generics. + +.. + +.. date: 2022-11-09-12-36-12 +.. gh-issue: 99284 +.. nonce: 9p4J2l +.. section: Library + +Remove ``_use_broken_old_ctypes_structure_semantics_`` old untested and +undocumented hack from :mod:`ctypes`. + +.. + +.. date: 2022-11-09-03-34-29 +.. gh-issue: 99201 +.. nonce: lDJ7xI +.. section: Library + +Fix :exc:`IndexError` when initializing the config variables on Windows if +``HAVE_DYNAMIC_LOADING`` is not set. + +.. + +.. date: 2022-11-08-15-54-43 +.. gh-issue: 99240 +.. nonce: MhYwcz +.. section: Library + +Fix double-free bug in Argument Clinic ``str_converter`` by extracting +memory clean up to a new ``post_parsing`` section. + +.. + +.. date: 2022-11-08-11-18-51 +.. gh-issue: 64490 +.. nonce: VcBgrN +.. section: Library + +Fix refcount error when arguments are packed to tuple in Argument Clinic. + +.. + +.. date: 2022-11-02-23-47-07 +.. gh-issue: 99029 +.. nonce: 7uCiIB +.. section: Library + +:meth:`pathlib.PurePath.relative_to()` now treats naked Windows drive paths +as relative. This brings its behaviour in line with other parts of pathlib. + +.. + +.. date: 2022-10-24-11-01-05 +.. gh-issue: 98253 +.. nonce: HVd5v4 +.. section: Library + +The implementation of the typing module is now more resilient to reference +leaks in binary extension modules. + +Previously, a reference leak in a typed C API-based extension module could +leak internals of the typing module, which could in turn introduce leaks in +essentially any other package with typed function signatures. Although the +typing package is not the original source of the problem, such non-local +dependences exacerbate debugging of large-scale projects, and the +implementation was therefore changed to reduce harm by providing better +isolation. + +.. + +.. date: 2022-10-19-18-31-53 +.. gh-issue: 98458 +.. nonce: vwyq7O +.. section: Library + +Fix infinite loop in unittest when a self-referencing chained exception is +raised + +.. + +.. date: 2022-10-19-13-37-23 +.. gh-issue: 93453 +.. nonce: wTB_sH +.. section: Library + +:func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like +:func:`asyncio.ensure_future`, :func:`asyncio.shield` or +:func:`asyncio.gather`, and also the +:meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of +:class:`asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` +if called when there is no running event loop and the current event loop was +not set. Previously they implicitly created and set a new current event +loop. :exc:`DeprecationWarning` is no longer emitted if there is no running +event loop but the current event loop was set. + +.. + +.. date: 2022-10-16-18-52-00 +.. gh-issue: 97966 +.. nonce: humlhz +.. section: Library + +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. + +.. + +.. date: 2022-10-13-22-13-54 +.. gh-issue: 98248 +.. nonce: lwyygy +.. section: Library + +Provide informative error messages in :func:`struct.pack` when its integral +arguments are not in range. + +.. + +.. date: 2022-10-08-19-20-33 +.. gh-issue: 98108 +.. nonce: WUObqM +.. section: Library + +``zipfile.Path`` is now pickleable if its initialization parameters were +pickleable (e.g. for file system paths). + +.. + +.. date: 2022-10-08-15-41-00 +.. gh-issue: 98098 +.. nonce: DugpWi +.. section: Library + +Created packages from zipfile and test_zipfile modules, separating +``zipfile.Path`` functionality. + +.. + +.. date: 2022-10-02-12-38-22 +.. gh-issue: 82836 +.. nonce: OvYLmC +.. section: Library + +Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the +:mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would +return True from this method; now they correctly return False. + +.. + +.. date: 2022-09-14-21-56-15 +.. gh-issue: 96828 +.. nonce: ZoOY5G +.. section: Library + +Add an :data:`~ssl.OP_ENABLE_KTLS` option for enabling the use of the kernel +TLS (kTLS). Patch by Illia Volochii. + +.. + +.. date: 2022-08-06-12-18-07 +.. gh-issue: 88863 +.. nonce: NnqsuJ +.. section: Library + +To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, +break reference cycles generated by local exception and future instances +(which has exception instance as its member var). Patch by Dong Uk, Kang. + +.. + +.. date: 2022-04-23-03-46-37 +.. gh-issue: 91078 +.. nonce: 87-hkp +.. section: Library + +:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. + +.. + +.. bpo: 47220 +.. date: 2022-04-04-22-54-11 +.. nonce: L9jYu4 +.. section: Library + +Document the optional *callback* parameter of :class:`WeakMethod`. Patch by +G?ry Ogam. + +.. + +.. bpo: 44817 +.. date: 2021-08-03-05-31-00 +.. nonce: wOW_Qn +.. section: Library + +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) and +161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). + +.. + +.. bpo: 41260 +.. date: 2020-08-02-23-46-22 +.. nonce: Q2BNzY +.. section: Library + +Rename the *fmt* parameter of the pure Python implementation of +:meth:`datetime.date.strftime` to *format*. + +.. + +.. bpo: 15999 +.. date: 2019-08-30-10-48-53 +.. nonce: QqsRRi +.. section: Library + +All built-in functions now accept arguments of any type instead of just +``bool`` and ``int`` for boolean parameters. + +.. + +.. date: 2022-12-02-17-08-08 +.. gh-issue: 99931 +.. nonce: wC46hE +.. section: Documentation + +Use `sphinxext-opengraph <https://sphinxext-opengraph.readthedocs.io/>`__ to +generate `OpenGraph metadata <https://ogp.me/>`__. + +.. + +.. date: 2022-11-26-21-43-05 +.. gh-issue: 89682 +.. nonce: DhKoTM +.. section: Documentation + +Reworded docstring of the default ``__contains__`` to clarify that it +returns a :class:`bool`. + +.. + +.. date: 2022-11-26-15-51-23 +.. gh-issue: 88330 +.. nonce: B_wFq8 +.. section: Documentation + +Improved the description of what a resource is in importlib.resources docs. + +.. + +.. date: 2022-11-16-12-52-23 +.. gh-issue: 92892 +.. nonce: TS-P0j +.. section: Documentation + +Document that calling variadic functions with ctypes requires special care +on macOS/arm64 (and possibly other platforms). + +.. + +.. bpo: 41825 +.. date: 2020-09-22-12-32-16 +.. nonce: npcaCb +.. section: Documentation + +Restructured the documentation for the :func:`os.wait* <os.wait>` family of +functions, and improved the docs for :func:`os.waitid` with more explanation +of the possible argument constants. + +.. + +.. date: 2022-12-05-16-12-56 +.. gh-issue: 99892 +.. nonce: sz_eW8 +.. section: Tests + +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. + +.. + +.. date: 2022-12-01-18-55-18 +.. gh-issue: 99934 +.. nonce: Ox3Fqf +.. section: Tests + +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. + +.. + +.. date: 2022-11-23-18-32-16 +.. gh-issue: 99741 +.. nonce: q4R7NH +.. section: Tests + +We've implemented multi-phase init (PEP 489/630/687) for the internal (for +testing) _xxsubinterpreters module. + +.. + +.. date: 2022-11-21-19-21-30 +.. gh-issue: 99659 +.. nonce: 4gP0nm +.. section: Tests + +Optional big memory tests in ``test_sqlite3`` now catch the correct +:exc:`sqlite.DataError` exception type in case of too large strings and/or +blobs passed. + +.. + +.. date: 2022-11-19-13-34-28 +.. gh-issue: 99593 +.. nonce: 8ZfCkj +.. section: Tests + +Cover the Unicode C API with tests. + +.. + +.. date: 2022-08-22-15-49-14 +.. gh-issue: 96002 +.. nonce: 4UE9UE +.. section: Tests + +Add functional test for Argument Clinic. + +.. + +.. date: 2022-11-24-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and +``-Wimplicit-function-declaration`` compiler warnings in +:program:`configure` checks. + +.. + +.. date: 2022-11-15-08-40-22 +.. gh-issue: 99337 +.. nonce: 5LoQDE +.. section: Build + +Fix a compilation issue with GCC 12 on macOS. + +.. + +.. date: 2022-11-09-14-42-48 +.. gh-issue: 99289 +.. nonce: X7wFE1 +.. section: Build + +Add a ``COMPILEALL_OPTS`` variable in Makefile to override :mod:`compileall` +options (default: ``-j0``) in ``make install``. Also merged the +``compileall`` commands into a single command building .pyc files for the +all optimization levels (0, 1, 2) at once. Patch by Victor Stinner. + +.. + +.. date: 2022-11-03-08-10-49 +.. gh-issue: 98872 +.. nonce: gdsR8X +.. section: Build + +Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python +3.11. + +.. + +.. date: 2022-10-16-12-49-24 +.. gh-issue: 88226 +.. nonce: BsnQ4k +.. section: Build + +Always define ``TARGET_*`` labels in ``Python/ceval.c``, even if +``USE_COMPUTED_GOTOS`` is disabled. This allows breakpoints to be set at +those labels in (for instance) ``gdb``. + +.. + +.. date: 2022-11-23-17-17-16 +.. gh-issue: 99345 +.. nonce: jOa3-f +.. section: Windows + +Use faster initialization functions to detect install location for Windows +Store package + +.. + +.. date: 2022-11-21-19-50-18 +.. gh-issue: 98629 +.. nonce: tMmB_B +.. section: Windows + +Fix initialization of :data:`sys.version` and ``sys._git`` on Windows + +.. + +.. date: 2022-11-16-19-03-21 +.. gh-issue: 99442 +.. nonce: 6Dgk3Q +.. section: Windows + +Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file +extension. + +.. + +.. bpo: 40882 +.. date: 2020-06-06-15-10-37 +.. nonce: UvNbdj +.. section: Windows + +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. + +.. + +.. date: 2022-11-25-09-23-20 +.. gh-issue: 87235 +.. nonce: SifjCD +.. section: macOS + +On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script +longer than a couple of bytes. + +.. + +.. date: 2022-11-01-10-32-23 +.. gh-issue: 98940 +.. nonce: W3YzC_ +.. section: macOS + +Fix ``Mac/Extras.install.py`` file filter bug. + +.. + +.. date: 2022-08-11-09-58-15 +.. gh-issue: 64490 +.. nonce: PjwhM4 +.. section: Tools/Demos + +Argument Clinic varargs bugfixes + +* Fix out-of-bounds error in :c:func:`!_PyArg_UnpackKeywordsWithVararg`. +* Fix incorrect check which allowed more than one varargs in clinic.py. +* Fix miscalculation of ``noptargs`` in generated code. +* Do not generate ``noptargs`` when there is a vararg argument and no optional argument. + +.. + +.. date: 2022-12-05-17-30-13 +.. gh-issue: 98680 +.. nonce: FiMCxZ +.. section: C API + +``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. +These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for +3.11, and are necessary to use the buffer API. + +.. + +.. date: 2022-11-20-09-52-50 +.. gh-issue: 99612 +.. nonce: eBHksg +.. section: C API + +Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: +``*consumed`` was not set. + +.. + +.. date: 2022-11-02-16-51-24 +.. gh-issue: 47146 +.. nonce: dsYDtI +.. section: C API + +The ``structmember.h`` header is deprecated. Its non-deprecated contents are +now available just by including ``Python.h``, with a ``Py_`` prefix added if +it was missing. (Deprecated contents are :c:macro:`T_OBJECT`, +:c:macro:`T_NONE`, and no-op flags.) Patch by Petr Viktorin, based on +earlier work by Alexander Belopolsky and Matthias Braun. diff --git a/Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst b/Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst deleted file mode 100644 index 5f32091739a2..000000000000 --- a/Misc/NEWS.d/next/Build/2022-10-16-12-49-24.gh-issue-88226.BsnQ4k.rst +++ /dev/null @@ -1,3 +0,0 @@ -Always define ``TARGET_*`` labels in ``Python/ceval.c``, even if -``USE_COMPUTED_GOTOS`` is disabled. This allows breakpoints to be -set at those labels in (for instance) ``gdb``. diff --git a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst b/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst deleted file mode 100644 index ad4dc496ee0e..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-03-08-10-49.gh-issue-98872.gdsR8X.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python 3.11. diff --git a/Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst b/Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst deleted file mode 100644 index 86ce4c79d17c..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-09-14-42-48.gh-issue-99289.X7wFE1.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add a ``COMPILEALL_OPTS`` variable in Makefile to override :mod:`compileall` -options (default: ``-j0``) in ``make install``. Also merged the ``compileall`` -commands into a single command building .pyc files for the all optimization levels -(0, 1, 2) at once. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst b/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst deleted file mode 100644 index f7396abde13b..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-15-08-40-22.gh-issue-99337.5LoQDE.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a compilation issue with GCC 12 on macOS. diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst deleted file mode 100644 index 2dace165ca1a..000000000000 --- a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst b/Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst deleted file mode 100644 index 0f419427925d..000000000000 --- a/Misc/NEWS.d/next/C API/2022-11-02-16-51-24.gh-issue-47146.dsYDtI.rst +++ /dev/null @@ -1,5 +0,0 @@ -The ``structmember.h`` header is deprecated. Its non-deprecated contents are -now available just by including ``Python.h``, with a ``Py_`` prefix added if -it was missing. (Deprecated contents are :c:macro:`T_OBJECT`, -:c:macro:`T_NONE`, and no-op flags.) Patch by Petr Viktorin, based on -earlier work by Alexander Belopolsky and Matthias Braun. diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst deleted file mode 100644 index 40e3c8db5403..000000000000 --- a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: -``*consumed`` was not set. diff --git a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst deleted file mode 100644 index a87090168e06..000000000000 --- a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. -These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for -3.11, and are necessary to use the buffer API. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst b/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst deleted file mode 100644 index b5964375962f..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-09-04-19-09-49.bpo-38031.Yq4L72.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a possible assertion failure in :class:`io.FileIO` when the opener -returns an invalid file descriptor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst deleted file mode 100644 index dd96c9e20d87..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of -uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. -Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst deleted file mode 100644 index 481ab53e4f51..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optimize the :class:`range` object iterator. It is now smaller, faster -iteration of ranges containing large numbers. Smaller pickles, faster -unpickling. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst deleted file mode 100644 index 5edc1ea65d8b..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-17-17-08-01.gh-issue-90994.f0H2Yd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve error messages when there's a syntax error with call arguments. The following three cases are covered: -- No value is assigned to a named argument, eg ``foo(a=)``. -- A value is assigned to a star argument, eg ``foo(*args=[0])``. -- A value is assigned to a double-star keyword argument, eg ``foo(**kwarg={'a': 0})``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst deleted file mode 100644 index 59bb12caef74..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst +++ /dev/null @@ -1,4 +0,0 @@ -Optimizing interpreters and JIT compilers may need to invalidate internal -metadata when functions are modified. This change adds the ability to -provide a callback that will be invoked each time a function is created, -modified, or destroyed. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst deleted file mode 100644 index e93ae4e7b127..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst +++ /dev/null @@ -1 +0,0 @@ -Allow some features of :mod:`syslog` to the main interpreter only. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst deleted file mode 100644 index 79a5b3ba98cf..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-16-53-40.gh-issue-99298.HqRJES.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove the remaining error paths for attribute specializations, and refuse -to specialize attribute accesses on types that haven't had -:c:func:`PyType_Ready` called on them yet. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst deleted file mode 100644 index b4a40d168429..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-17-09-16.gh-issue-98686.bmAKwr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove the ``BINARY_OP_GENERIC`` and ``COMPARE_OP_GENERIC`` -"specializations". diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst deleted file mode 100644 index 631b9ca23044..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-11-14-04-01.gh-issue-99377.-CJvWn.rst +++ /dev/null @@ -1 +0,0 @@ -Add audit events for thread creation and clear operations. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst deleted file mode 100644 index 142f91ccd92e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix zip path for venv created from a non-installed python on POSIX -platforms. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst deleted file mode 100644 index 7e3c52924213..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-16-21-35-30.gh-issue-99547.p_c_bp.rst +++ /dev/null @@ -1 +0,0 @@ -Add a function to os.path to check if a path is a junction: isjunction. Add similar functionality to pathlib.Path as is_junction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst deleted file mode 100644 index 8d9f55d6d9d0..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-18-11-24-25.gh-issue-99553.F64h-n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where an :exc:`ExceptionGroup` subclass can wrap a -:exc:`BaseException`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst deleted file mode 100644 index 8071fd130dd6..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the -available tokenizer buffer. Patch by Pablo galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst deleted file mode 100644 index 9321cef77eed..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-21-11-27-14.gh-issue-99578.DcKoBJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of the -first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst deleted file mode 100644 index 47f385c2eefe..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-23-18-16-18.gh-issue-99708.7MuaiR.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug where compiler crashes on an if expression with an empty body block. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst deleted file mode 100644 index 3fe21a8a21bf..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-26-04-00-41.gh-issue-99729.A3ovwQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue that could cause frames to be visible to Python code as they -are being torn down, possibly leading to memory corruption or hard crashes -of the interpreter. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst deleted file mode 100644 index c46459c15b9e..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to -register callbacks to receive notification on creation and destruction of -code objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst deleted file mode 100644 index 20cd361affea..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the tokenizer that could cause infinite recursion when showing -syntax warnings that happen in the first line of the source. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst b/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst deleted file mode 100644 index 390b4a9824c7..000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Restructured the documentation for the :func:`os.wait* <os.wait>` family of functions, -and improved the docs for :func:`os.waitid` with more explanation of the -possible argument constants. diff --git a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst b/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst deleted file mode 100644 index 54e421d19d9d..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-16-12-52-23.gh-issue-92892.TS-P0j.rst +++ /dev/null @@ -1 +0,0 @@ -Document that calling variadic functions with ctypes requires special care on macOS/arm64 (and possibly other platforms). diff --git a/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst b/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst deleted file mode 100644 index 0f242eecc312..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst +++ /dev/null @@ -1 +0,0 @@ -Improved the description of what a resource is in importlib.resources docs. diff --git a/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst b/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst deleted file mode 100644 index 46be065b6539..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst +++ /dev/null @@ -1 +0,0 @@ -Reworded docstring of the default ``__contains__`` to clarify that it returns a :class:`bool`. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst deleted file mode 100644 index 0c01a2cb2cfa..000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use `sphinxext-opengraph <https://sphinxext-opengraph.readthedocs.io/>`__ -to generate `OpenGraph metadata <https://ogp.me/>`__. diff --git a/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst b/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst deleted file mode 100644 index 9650c694cc67..000000000000 --- a/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst +++ /dev/null @@ -1,2 +0,0 @@ -All built-in functions now accept arguments of any type instead of just -``bool`` and ``int`` for boolean parameters. diff --git a/Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst b/Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst deleted file mode 100644 index ae2fdd9b84a0..000000000000 --- a/Misc/NEWS.d/next/Library/2020-08-02-23-46-22.bpo-41260.Q2BNzY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Rename the *fmt* parameter of the pure Python implementation of -:meth:`datetime.date.strftime` to *format*. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst deleted file mode 100644 index 79f8c506b54f..000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) -and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). diff --git a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst b/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst deleted file mode 100644 index 6e2af088640b..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document the optional *callback* parameter of :class:`WeakMethod`. Patch by -G?ry Ogam. diff --git a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst b/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst deleted file mode 100644 index e05d5e2a1314..000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. diff --git a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst b/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst deleted file mode 100644 index 23f8cb01cf0a..000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-06-12-18-07.gh-issue-88863.NnqsuJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, -break reference cycles generated by local exception and future instances -(which has exception instance as its member var). Patch by Dong Uk, Kang. diff --git a/Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst b/Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst deleted file mode 100644 index d8a448851f47..000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-14-21-56-15.gh-issue-96828.ZoOY5G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add an :data:`~ssl.OP_ENABLE_KTLS` option for enabling the use of the kernel -TLS (kTLS). Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst deleted file mode 100644 index dcbea66d66bf..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would return True from this method; now they correctly return False. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst b/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst deleted file mode 100644 index 202275e16ea0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Created packages from zipfile and test_zipfile modules, separating -``zipfile.Path`` functionality. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst b/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst deleted file mode 100644 index 7e962580dda2..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst +++ /dev/null @@ -1,2 +0,0 @@ -``zipfile.Path`` is now pickleable if its initialization parameters were -pickleable (e.g. for file system paths). diff --git a/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst b/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst deleted file mode 100644 index 347f6e160335..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst +++ /dev/null @@ -1 +0,0 @@ -Provide informative error messages in :func:`struct.pack` when its integral arguments are not in range. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst deleted file mode 100644 index b725465ae4f0..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst +++ /dev/null @@ -1,2 +0,0 @@ -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst b/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst deleted file mode 100644 index 5aee4b4f0aa7..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-13-37-23.gh-issue-93453.wTB_sH.rst +++ /dev/null @@ -1,9 +0,0 @@ -:func:`asyncio.get_event_loop` and many other :mod:`asyncio` functions like -:func:`asyncio.ensure_future`, :func:`asyncio.shield` or -:func:`asyncio.gather`, and also the -:meth:`~asyncio.BaseDefaultEventLoopPolicy.get_event_loop` method of -:class:`asyncio.BaseDefaultEventLoopPolicy` now raise a :exc:`RuntimeError` -if called when there is no running event loop and the current event loop was -not set. Previously they implicitly created and set a new current event -loop. :exc:`DeprecationWarning` is no longer emitted if there is no running -event loop but the current event loop was set. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst deleted file mode 100644 index f74195cc8e7d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst +++ /dev/null @@ -1 +0,0 @@ -Fix infinite loop in unittest when a self-referencing chained exception is raised diff --git a/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst b/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst deleted file mode 100644 index 00df0070f3b9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst +++ /dev/null @@ -1,10 +0,0 @@ -The implementation of the typing module is now more resilient to reference -leaks in binary extension modules. - -Previously, a reference leak in a typed C API-based extension module could leak -internals of the typing module, which could in turn introduce leaks in -essentially any other package with typed function signatures. Although the -typing package is not the original source of the problem, such non-local -dependences exacerbate debugging of large-scale projects, and the -implementation was therefore changed to reduce harm by providing better -isolation. diff --git a/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst b/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst deleted file mode 100644 index 0bfba5e1e326..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`pathlib.PurePath.relative_to()` now treats naked Windows drive paths -as relative. This brings its behaviour in line with other parts of pathlib. diff --git a/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst b/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst deleted file mode 100644 index f98c181cc9c5..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst +++ /dev/null @@ -1 +0,0 @@ -Fix refcount error when arguments are packed to tuple in Argument Clinic. diff --git a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst deleted file mode 100644 index 0a4db052755f..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix double-free bug in Argument Clinic ``str_converter`` by -extracting memory clean up to a new ``post_parsing`` section. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst b/Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst deleted file mode 100644 index 6d03574fdaf5..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-03-34-29.gh-issue-99201.lDJ7xI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :exc:`IndexError` when initializing the config variables on Windows if -``HAVE_DYNAMIC_LOADING`` is not set. diff --git a/Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst b/Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst deleted file mode 100644 index 48576bd457aa..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-09-12-36-12.gh-issue-99284.9p4J2l.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove ``_use_broken_old_ctypes_structure_semantics_`` -old untested and undocumented hack from :mod:`ctypes`. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst deleted file mode 100644 index 412c8c793435..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.TypeVarTuple` and -:class:`~typing.ParamSpec` together in user generics. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst deleted file mode 100644 index 1950680b1df8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix substitution of :class:`~typing.ParamSpec` followed by -:class:`~typing.TypeVarTuple` in generic aliases. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst deleted file mode 100644 index f153f2fceac8..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Check the number of arguments in substitution in user generics containing a -:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. diff --git a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst b/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst deleted file mode 100644 index 451561c579da..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-13-02-06-56.gh-issue-99341.8-OlwB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when -changing line numbers. diff --git a/Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst b/Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst deleted file mode 100644 index f35799d45457..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-14-08-21-56.gh-issue-99388.UWSlwp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add *loop_factory* parameter to :func:`asyncio.run` to allow specifying a custom event loop factory. -Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst b/Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst deleted file mode 100644 index c6e2a0ca25ff..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-04-08-25.gh-issue-92647.cZcjnJ.rst +++ /dev/null @@ -1 +0,0 @@ -Use final status of an enum to determine lookup or creation branch of functional API. diff --git a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst b/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst deleted file mode 100644 index 014161cf7b1d..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-15-10-55-24.gh-issue-97001.KeQuVF.rst +++ /dev/null @@ -1 +0,0 @@ -Release the GIL when calling termios APIs to avoid blocking threads. diff --git a/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst b/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst deleted file mode 100644 index ebd821738827..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :mod:`asyncio` to not share event loop and signal wakeupfd in forked processes. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst deleted file mode 100644 index 82720d17bcaf..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling -``_imp.source_hash()``. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst b/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst deleted file mode 100644 index f6ee449891d9..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-13-49-03.gh-issue-99645.9w1QKq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now -``addClassCleanup()`` uses separate lists for different ``TestCase`` -subclasses, and ``doClassCleanups()`` only cleans up the particular class. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst deleted file mode 100644 index 63fe7b8a3a32..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug when calling trace.CoverageResults with valid infile. diff --git a/Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst b/Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst deleted file mode 100644 index 4b12fd9c8d79..000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-22-19-31-26.gh-issue-79033.MW6kHq.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`asyncio.Server.wait_closed` to actually do what the docs promise -- wait for all existing connections to complete, after closing the server. diff --git a/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst b/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst deleted file mode 100644 index 4fd7b6b85cef..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-03-05-58-48.gh-issue-99957.jLYYgN.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. diff --git a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst b/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst deleted file mode 100644 index e305352c7a55..000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-05-13-40-15.gh-issue-100001.78ReYp.rst +++ /dev/null @@ -1,5 +0,0 @@ -Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so -that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex -replacement we put in or the characters r"\x" came through in the original -request line. diff --git a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst b/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst deleted file mode 100644 index c931409b8171..000000000000 --- a/Misc/NEWS.d/next/Security/2022-11-11-12-50-28.gh-issue-87604.OtwH5L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid publishing list of active per-interpreter audit hooks via the -:mod:`gc` module diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst deleted file mode 100644 index a396e95cd83f..000000000000 --- a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst +++ /dev/null @@ -1,6 +0,0 @@ -``python -m http.server`` no longer allows terminal control characters sent -within a garbage request to be printed to the stderr server log. - -This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` -``.log_message`` method to replace control characters with a ``\xHH`` hex escape -before printing. diff --git a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst deleted file mode 100644 index dc86e1d70f12..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst +++ /dev/null @@ -1 +0,0 @@ -Add functional test for Argument Clinic. diff --git a/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst b/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst deleted file mode 100644 index ec4cda208032..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst +++ /dev/null @@ -1 +0,0 @@ -Cover the Unicode C API with tests. diff --git a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst b/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst deleted file mode 100644 index 3db1ec12b520..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-21-19-21-30.gh-issue-99659.4gP0nm.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optional big memory tests in ``test_sqlite3`` now catch the correct -:exc:`sqlite.DataError` exception type in case of too large strings and/or -blobs passed. diff --git a/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst deleted file mode 100644 index 791726b52bfb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst +++ /dev/null @@ -1,2 +0,0 @@ -We've implemented multi-phase init (PEP 489/630/687) -for the internal (for testing) _xxsubinterpreters module. diff --git a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst deleted file mode 100644 index 1b1287c33dbd..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst +++ /dev/null @@ -1 +0,0 @@ -Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst deleted file mode 100644 index eded0361fbeb..000000000000 --- a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip test_normalization() of test_unicodedata if it fails to download -NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst deleted file mode 100644 index 4a308a930605..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst +++ /dev/null @@ -1,7 +0,0 @@ -Argument Clinic varargs bugfixes - -* Fix out-of-bounds error in :c:func:`!_PyArg_UnpackKeywordsWithVararg`. -* Fix incorrect check which allowed more than one varargs in clinic.py. -* Fix miscalculation of ``noptargs`` in generated code. -* Do not generate ``noptargs`` when there is a vararg argument and no optional argument. - diff --git a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst b/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst deleted file mode 100644 index 2670aeef9a25..000000000000 --- a/Misc/NEWS.d/next/Windows/2020-06-06-15-10-37.bpo-40882.UvNbdj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on -Windows. diff --git a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst b/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst deleted file mode 100644 index 8e19366c4297..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-16-19-03-21.gh-issue-99442.6Dgk3Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file -extension. diff --git a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst b/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst deleted file mode 100644 index 46cbf998eb20..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-21-19-50-18.gh-issue-98629.tMmB_B.rst +++ /dev/null @@ -1 +0,0 @@ -Fix initialization of :data:`sys.version` and ``sys._git`` on Windows diff --git a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst b/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst deleted file mode 100644 index 99db0c55a67e..000000000000 --- a/Misc/NEWS.d/next/Windows/2022-11-23-17-17-16.gh-issue-99345.jOa3-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use faster initialization functions to detect install location for Windows -Store package diff --git a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst b/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst deleted file mode 100644 index 18ef0b0e2523..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-01-10-32-23.gh-issue-98940.W3YzC_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``Mac/Extras.install.py`` file filter bug. diff --git a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst b/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst deleted file mode 100644 index 3111e4975e87..000000000000 --- a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst +++ /dev/null @@ -1 +0,0 @@ -On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script longer than a couple of bytes. diff --git a/README.rst b/README.rst index 1fa019473643..ab9d3a6ea71c 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 2 +This is Python version 3.12.0 alpha 3 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From webhook-mailer at python.org Tue Dec 6 16:44:03 2022 From: webhook-mailer at python.org (ethanfurman) Date: Tue, 06 Dec 2022 21:44:03 -0000 Subject: [Python-checkins] gh-94943: [Enum] improve repr() when inheriting from a dataclass (GH-99740) Message-ID: <mailman.2834.1670363044.3313.python-checkins@python.org> https://github.com/python/cpython/commit/679efbb080242fc5be63ad873968f05faeef889f commit: 679efbb080242fc5be63ad873968f05faeef889f branch: main author: Ethan Furman <ethan at stoneleaf.us> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-06T13:43:41-08:00 summary: gh-94943: [Enum] improve repr() when inheriting from a dataclass (GH-99740) Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: A Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst M Doc/howto/enum.rst M Doc/library/enum.rst M Lib/enum.py M Lib/test/test_enum.py diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 98d9f4febe2d..3155c6cb977b 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -459,6 +459,31 @@ sense to allow sharing some common behavior between a group of enumerations. (See `OrderedEnum`_ for an example.) +.. _enum-dataclass-support: + +Dataclass support +----------------- + +When inheriting from a :class:`~dataclasses.dataclass`, +the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: + + >>> @dataclass + ... class CreatureDataMixin: + ... size: str + ... legs: int + ... tail: bool = field(repr=False, default=True) + ... + >>> class Creature(CreatureDataMixin, Enum): + ... BEETLE = 'small', 6 + ... DOG = 'medium', 4 + ... + >>> Creature.DOG + <Creature.DOG: size='medium', legs=4> + +Use the :func:`!dataclass` argument ``repr=False`` +to use the standard :func:`repr`. + + Pickling -------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 208aecf11c80..f8875440baf5 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -389,6 +389,8 @@ Data Types Using :class:`auto` with :class:`Enum` results in integers of increasing value, starting with ``1``. + .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. class:: IntEnum diff --git a/Lib/enum.py b/Lib/enum.py index 1b683c702d59..f07b821f1a60 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -955,7 +955,15 @@ def _find_data_repr_(mcls, class_name, bases): return base._value_repr_ elif '__repr__' in base.__dict__: # this is our data repr - return base.__dict__['__repr__'] + # double-check if a dataclass with a default __repr__ + if ( + '__dataclass_fields__' in base.__dict__ + and '__dataclass_params__' in base.__dict__ + and base.__dict__['__dataclass_params__'].repr + ): + return _dataclass_repr + else: + return base.__dict__['__repr__'] return None @classmethod @@ -1551,6 +1559,14 @@ def _power_of_two(value): return False return value == 2 ** _high_bit(value) +def _dataclass_repr(self): + dcf = self.__dataclass_fields__ + return ', '.join( + '%s=%r' % (k, getattr(self, k)) + for k in dcf.keys() + if dcf[k].repr + ) + def global_enum_repr(self): """ use module.enum_name instead of class.enum_name diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index b6082cf02b18..146ef39ca384 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2717,17 +2717,67 @@ def upper(self): def test_repr_with_dataclass(self): "ensure dataclass-mixin has correct repr()" - from dataclasses import dataclass - @dataclass + # + # check overridden dataclass __repr__ is used + # + from dataclasses import dataclass, field + @dataclass(repr=False) class Foo: __qualname__ = 'Foo' a: int + def __repr__(self): + return 'ha hah!' class Entries(Foo, Enum): ENTRY1 = 1 self.assertTrue(isinstance(Entries.ENTRY1, Foo)) self.assertTrue(Entries._member_type_ is Foo, Entries._member_type_) self.assertTrue(Entries.ENTRY1.value == Foo(1), Entries.ENTRY1.value) - self.assertEqual(repr(Entries.ENTRY1), '<Entries.ENTRY1: Foo(a=1)>') + self.assertEqual(repr(Entries.ENTRY1), '<Entries.ENTRY1: ha hah!>') + # + # check auto-generated dataclass __repr__ is not used + # + @dataclass + class CreatureDataMixin: + __qualname__ = 'CreatureDataMixin' + size: str + legs: int + tail: bool = field(repr=False, default=True) + class Creature(CreatureDataMixin, Enum): + __qualname__ = 'Creature' + BEETLE = ('small', 6) + DOG = ('medium', 4) + self.assertEqual(repr(Creature.DOG), "<Creature.DOG: size='medium', legs=4>") + # + # check inherited repr used + # + class Huh: + def __repr__(self): + return 'inherited' + @dataclass(repr=False) + class CreatureDataMixin(Huh): + __qualname__ = 'CreatureDataMixin' + size: str + legs: int + tail: bool = field(repr=False, default=True) + class Creature(CreatureDataMixin, Enum): + __qualname__ = 'Creature' + BEETLE = ('small', 6) + DOG = ('medium', 4) + self.assertEqual(repr(Creature.DOG), "<Creature.DOG: inherited>") + # + # check default object.__repr__ used if nothing provided + # + @dataclass(repr=False) + class CreatureDataMixin: + __qualname__ = 'CreatureDataMixin' + size: str + legs: int + tail: bool = field(repr=False, default=True) + class Creature(CreatureDataMixin, Enum): + __qualname__ = 'Creature' + BEETLE = ('small', 6) + DOG = ('medium', 4) + self.assertRegex(repr(Creature.DOG), "<Creature.DOG: .*CreatureDataMixin object at .*>") def test_repr_with_init_data_type_mixin(self): # non-data_type is a mixin that doesn't define __new__ diff --git a/Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst b/Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst new file mode 100644 index 000000000000..ed4754e49bd2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-23-23-58-45.gh-issue-94943.Oog0Zo.rst @@ -0,0 +1,5 @@ +Add :ref:`enum-dataclass-support` to the +:class:`~enum.Enum` :meth:`~enum.Enum.__repr__`. +When inheriting from a :class:`~dataclasses.dataclass`, +only show the field names in the value section of the member :func:`repr`, +and not the dataclass' class name. From webhook-mailer at python.org Tue Dec 6 17:40:10 2022 From: webhook-mailer at python.org (vstinner) Date: Tue, 06 Dec 2022 22:40:10 -0000 Subject: [Python-checkins] PyUnicode_KIND() uses _Py_RVALUE() (#100060) Message-ID: <mailman.2835.1670366411.3313.python-checkins@python.org> https://github.com/python/cpython/commit/abbe4482ab914d6da8b1678ad46174cb875ed7f6 commit: abbe4482ab914d6da8b1678ad46174cb875ed7f6 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-06T23:40:05+01:00 summary: PyUnicode_KIND() uses _Py_RVALUE() (#100060) The PyUnicode_KIND() macro is modified to use _Py_RVALUE(), so it can no longer be used as a l-value. files: M Include/cpython/unicodeobject.h diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index a75336f590e8..75a74ffa2f9d 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -231,7 +231,7 @@ enum PyUnicode_Kind { // new compiler warnings on "kind < PyUnicode_KIND(str)" (compare signed and // unsigned numbers) where kind type is an int or on // "unsigned int kind = PyUnicode_KIND(str)" (cast signed to unsigned). -#define PyUnicode_KIND(op) (_PyASCIIObject_CAST(op)->state.kind) +#define PyUnicode_KIND(op) _Py_RVALUE(_PyASCIIObject_CAST(op)->state.kind) /* Return a void pointer to the raw unicode buffer. */ static inline void* _PyUnicode_COMPACT_DATA(PyObject *op) { From webhook-mailer at python.org Tue Dec 6 18:10:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 06 Dec 2022 23:10:02 -0000 Subject: [Python-checkins] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) Message-ID: <mailman.2836.1670368203.3313.python-checkins@python.org> https://github.com/python/cpython/commit/97e7004cfe48305bcd642c653b406dc7470e196d commit: 97e7004cfe48305bcd642c653b406dc7470e196d branch: main author: Pablo Galindo Salgado <Pablogsal at gmail.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-06T15:09:56-08:00 summary: gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) Automerge-Triggered-By: GH:pablogsal files: A Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst M Lib/test/test_syntax.py M Parser/pegen_errors.c diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 78cac231929a..cb284195d976 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2145,6 +2145,22 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + def test_error_string_literal(self): + + self._check_error("'blech", "unterminated string literal") + self._check_error('"blech', "unterminated string literal") + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst new file mode 100644 index 000000000000..8e7c72d804f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst @@ -0,0 +1,2 @@ +Honor existing errors obtained when searching for mismatching parentheses in +the tokenizer. Patch by Pablo Galindo diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 7738cbaf9ef3..6ea7600119b6 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -169,6 +169,10 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { for (;;) { switch (_PyTokenizer_Get(p->tok, &new_token)) { case ERRORTOKEN: + if (PyErr_Occurred()) { + ret = -1; + goto exit; + } if (p->tok->level != 0) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; if (current_err_line > error_lineno) { From webhook-mailer at python.org Tue Dec 6 21:44:58 2022 From: webhook-mailer at python.org (ethanfurman) Date: Wed, 07 Dec 2022 02:44:58 -0000 Subject: [Python-checkins] [Enum] Remove unused code from `test_enum.py` (GH-96986) Message-ID: <mailman.2837.1670381100.3313.python-checkins@python.org> https://github.com/python/cpython/commit/889b0b9bf95651fc05ad532cc4a66c0f8ff29fc2 commit: 889b0b9bf95651fc05ad532cc4a66c0f8ff29fc2 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-06T18:44:47-08:00 summary: [Enum] Remove unused code from `test_enum.py` (GH-96986) files: M Lib/test/test_enum.py diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 146ef39ca384..376347e4c111 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4553,11 +4553,6 @@ class Quadruple(Enum): COMPLEX_A = 2j COMPLEX_B = 3j -class _ModuleWrapper: - """We use this class as a namespace for swapping modules.""" - def __init__(self, module): - self.__dict__.update(module.__dict__) - class TestConvert(unittest.TestCase): def tearDown(self): # Reset the module-level test variables to their original integer @@ -4597,12 +4592,6 @@ def test_convert_int(self): self.assertEqual(test_type.CONVERT_TEST_NAME_D, 5) self.assertEqual(test_type.CONVERT_TEST_NAME_E, 5) # Ensure that test_type only picked up names matching the filter. - int_dir = dir(int) + [ - 'CONVERT_TEST_NAME_A', 'CONVERT_TEST_NAME_B', 'CONVERT_TEST_NAME_C', - 'CONVERT_TEST_NAME_D', 'CONVERT_TEST_NAME_E', 'CONVERT_TEST_NAME_F', - 'CONVERT_TEST_SIGABRT', 'CONVERT_TEST_SIGIOT', - 'CONVERT_TEST_EIO', 'CONVERT_TEST_EBUS', - ] extra = [name for name in dir(test_type) if name not in enum_dir(test_type)] missing = [name for name in enum_dir(test_type) if name not in dir(test_type)] self.assertEqual( @@ -4644,7 +4633,6 @@ def test_convert_str(self): self.assertEqual(test_type.CONVERT_STR_TEST_1, 'hello') self.assertEqual(test_type.CONVERT_STR_TEST_2, 'goodbye') # Ensure that test_type only picked up names matching the filter. - str_dir = dir(str) + ['CONVERT_STR_TEST_1', 'CONVERT_STR_TEST_2'] extra = [name for name in dir(test_type) if name not in enum_dir(test_type)] missing = [name for name in enum_dir(test_type) if name not in dir(test_type)] self.assertEqual( @@ -4712,8 +4700,6 @@ def member_dir(member): allowed.add(name) return sorted(allowed) -missing = object() - if __name__ == '__main__': unittest.main() From webhook-mailer at python.org Wed Dec 7 01:55:59 2022 From: webhook-mailer at python.org (corona10) Date: Wed, 07 Dec 2022 06:55:59 -0000 Subject: [Python-checkins] gh-93018: Fix for the compatibility problems with expat (gh-93900) Message-ID: <mailman.2838.1670396160.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7031275776f43c76231318c2158a7a2753bc1fba commit: 7031275776f43c76231318c2158a7a2753bc1fba branch: main author: Mat?j Cepl <mcepl at cepl.eu> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-07T15:55:49+09:00 summary: gh-93018: Fix for the compatibility problems with expat (gh-93900) files: A Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst M Lib/test/test_minidom.py diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index ef38c362103f..2ca3908bd1ca 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -1163,14 +1163,10 @@ def testEncodings(self): # Verify that character decoding errors raise exceptions instead # of crashing - if pyexpat.version_info >= (2, 4, 5): - self.assertRaises(ExpatError, parseString, - b'<fran\xe7ais></fran\xe7ais>') - self.assertRaises(ExpatError, parseString, - b'<franais>Comment \xe7a va ? Tr\xe8s bien ?</franais>') - else: - self.assertRaises(UnicodeDecodeError, parseString, - b'<fran\xe7ais>Comment \xe7a va ? Tr\xe8s bien ?</fran\xe7ais>') + with self.assertRaises((UnicodeDecodeError, ExpatError)): + parseString( + b'<fran\xe7ais>Comment \xe7a va ? Tr\xe8s bien ?</fran\xe7ais>' + ) doc.unlink() @@ -1631,13 +1627,11 @@ def testEmptyXMLNSValue(self): self.confirm(doc2.namespaceURI == xml.dom.EMPTY_NAMESPACE) def testExceptionOnSpacesInXMLNSValue(self): - if pyexpat.version_info >= (2, 4, 5): - context = self.assertRaisesRegex(ExpatError, 'syntax error') - else: - context = self.assertRaisesRegex(ValueError, 'Unsupported syntax') - - with context: - parseString('<element xmlns:abc="http:abc.com/de f g/hi/j k"><abc:foo /></element>') + with self.assertRaises((ValueError, ExpatError)): + parseString( + '<element xmlns:abc="http:abc.com/de f g/hi/j k">' + + '<abc:foo /></element>' + ) def testDocRemoveChild(self): doc = parse(tstfile) diff --git a/Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst b/Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst new file mode 100644 index 000000000000..a8fb98048e40 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-06-16-13-26-31.gh-issue-93018.wvNx76.rst @@ -0,0 +1 @@ +Make two tests forgiving towards host system libexpat with backported security fixes applied. From webhook-mailer at python.org Wed Dec 7 04:18:41 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 07 Dec 2022 09:18:41 -0000 Subject: [Python-checkins] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) Message-ID: <mailman.2839.1670404722.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b97ddd5122c016bd090ed187479606205b6b1a4 commit: 2b97ddd5122c016bd090ed187479606205b6b1a4 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T01:18:00-08:00 summary: gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) (cherry picked from commit 97e7004cfe48305bcd642c653b406dc7470e196d) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> Automerge-Triggered-By: GH:pablogsal files: A Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst M Lib/test/test_syntax.py M Parser/pegen_errors.c diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 400092ee2c89..42d36e0b9d81 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2095,6 +2095,22 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + def test_error_string_literal(self): + + self._check_error("'blech", "unterminated string literal") + self._check_error('"blech', "unterminated string literal") + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst new file mode 100644 index 000000000000..8e7c72d804f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst @@ -0,0 +1,2 @@ +Honor existing errors obtained when searching for mismatching parentheses in +the tokenizer. Patch by Pablo Galindo diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index a0f4b9809e21..3d8cccb0a974 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -170,6 +170,10 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { const char *end; switch (_PyTokenizer_Get(p->tok, &start, &end)) { case ERRORTOKEN: + if (PyErr_Occurred()) { + ret = -1; + goto exit; + } if (p->tok->level != 0) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; if (current_err_line > error_lineno) { From webhook-mailer at python.org Wed Dec 7 08:52:30 2022 From: webhook-mailer at python.org (pablogsal) Date: Wed, 07 Dec 2022 13:52:30 -0000 Subject: [Python-checkins] [3.10] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) (#100073) Message-ID: <mailman.2840.1670421151.3313.python-checkins@python.org> https://github.com/python/cpython/commit/72cfe5b1b9ad1bbe9da38a5af605e5d03ad88147 commit: 72cfe5b1b9ad1bbe9da38a5af605e5d03ad88147 branch: 3.10 author: Pablo Galindo Salgado <Pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-07T13:52:05Z summary: [3.10] gh-100050: Fix an assertion error when raising unclosed parenthesis errors in the tokenizer (GH-100065) (#100073) Automerge-Triggered-By: GH:pablogsal. (cherry picked from commit 97e7004cfe48305bcd642c653b406dc7470e196d) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst M Lib/test/test_syntax.py M Parser/pegen.c diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 5f2528b904b5..cd09d1eabdf5 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1587,6 +1587,22 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + def test_error_string_literal(self): + + self._check_error("'blech", "unterminated string literal") + self._check_error('"blech', "unterminated string literal") + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + def test_match_call_does_not_raise_syntax_error(self): code = """ def match(x): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst new file mode 100644 index 000000000000..8e7c72d804f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-06-22-24-01.gh-issue-100050.lcrPqQ.rst @@ -0,0 +1,2 @@ +Honor existing errors obtained when searching for mismatching parentheses in +the tokenizer. Patch by Pablo Galindo diff --git a/Parser/pegen.c b/Parser/pegen.c index c12adedebba3..95c2273e4b42 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1328,6 +1328,10 @@ _PyPegen_check_tokenizer_errors(Parser *p) { const char *end; switch (PyTokenizer_Get(p->tok, &start, &end)) { case ERRORTOKEN: + if (PyErr_Occurred()) { + ret = -1; + goto exit; + } if (p->tok->level != 0) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; if (current_err_line > error_lineno) { From webhook-mailer at python.org Wed Dec 7 09:22:44 2022 From: webhook-mailer at python.org (vstinner) Date: Wed, 07 Dec 2022 14:22:44 -0000 Subject: [Python-checkins] gh-98724: Fix Py_CLEAR() macro side effects (#99100) (#100070) Message-ID: <mailman.2841.1670422965.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b11a384dc7471ffc16de4b86e8f5fdeef151f348 commit: b11a384dc7471ffc16de4b86e8f5fdeef151f348 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-07T15:22:38+01:00 summary: gh-98724: Fix Py_CLEAR() macro side effects (#99100) (#100070) The Py_CLEAR(), Py_SETREF() and Py_XSETREF() macros now only evaluate their arguments once. If an argument has side effects, these side effects are no longer duplicated. Use temporary variables to avoid duplicating side effects of macro arguments. If available, use _Py_TYPEOF() to avoid type punning. Otherwise, use memcpy() for the assignment to prevent a miscompilation with strict aliasing caused by type punning. Add _Py_TYPEOF() macro: __typeof__() on GCC and clang. Add test_py_clear() and test_py_setref() unit tests to _testcapi. files: A Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst M Doc/c-api/refcounting.rst M Doc/whatsnew/3.12.rst M Include/cpython/object.h M Include/object.h M Include/pyport.h M Modules/_testcapimodule.c diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index cd1f2ef70768..d8e9c2da6f3f 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -7,8 +7,8 @@ Reference Counting ****************** -The macros in this section are used for managing reference counts of Python -objects. +The functions and macros in this section are used for managing reference counts +of Python objects. .. c:function:: Py_ssize_t Py_REFCNT(PyObject *o) @@ -129,6 +129,11 @@ objects. It is a good idea to use this macro whenever decrementing the reference count of an object that might be traversed during garbage collection. + .. versionchanged:: 3.12 + The macro argument is now only evaluated once. If the argument has side + effects, these are no longer duplicated. + + .. c:function:: void Py_IncRef(PyObject *o) Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. @@ -139,3 +144,40 @@ objects. Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. + + +.. c:macro:: Py_SETREF(dst, src) + + Macro safely decrementing the `dst` reference count and setting `dst` to + `src`. + + As in case of :c:func:`Py_CLEAR`, "the obvious" code can be deadly:: + + Py_DECREF(dst); + dst = src; + + The safe way is:: + + Py_SETREF(dst, src); + + That arranges to set `dst` to `src` _before_ decrementing reference count of + *dst* old value, so that any code triggered as a side-effect of `dst` + getting torn down no longer believes `dst` points to a valid object. + + .. versionadded:: 3.6 + + .. versionchanged:: 3.12 + The macro arguments are now only evaluated once. If an argument has side + effects, these are no longer duplicated. + + +.. c:macro:: Py_XSETREF(dst, src) + + Variant of :c:macro:`Py_SETREF` macro that uses :c:func:`Py_XDECREF` instead + of :c:func:`Py_DECREF`. + + .. versionadded:: 3.6 + + .. versionchanged:: 3.12 + The macro arguments are now only evaluated once. If an argument has side + effects, these are no longer duplicated. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index e9e25b9dc466..73dc462f0b33 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -851,6 +851,11 @@ Porting to Python 3.12 :class:`bytes` type is accepted for bytes strings. (Contributed by Victor Stinner in :gh:`98393`.) +* The :c:macro:`Py_CLEAR`, :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` + macros now only evaluate their arguments once. If an argument has side + effects, these side effects are no longer duplicated. + (Contributed by Victor Stinner in :gh:`98724`.) + Deprecated ---------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3abfcb7d44f0..426337086130 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -305,38 +305,69 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *, PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); -/* Safely decref `op` and set `op` to `op2`. +/* Safely decref `dst` and set `dst` to `src`. * * As in case of Py_CLEAR "the obvious" code can be deadly: * - * Py_DECREF(op); - * op = op2; + * Py_DECREF(dst); + * dst = src; * * The safe way is: * - * Py_SETREF(op, op2); + * Py_SETREF(dst, src); * - * That arranges to set `op` to `op2` _before_ decref'ing, so that any code - * triggered as a side-effect of `op` getting torn down no longer believes - * `op` points to a valid object. + * That arranges to set `dst` to `src` _before_ decref'ing, so that any code + * triggered as a side-effect of `dst` getting torn down no longer believes + * `dst` points to a valid object. * - * Py_XSETREF is a variant of Py_SETREF that uses Py_XDECREF instead of - * Py_DECREF. + * Temporary variables are used to only evalutate macro arguments once and so + * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to + * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for + * implementation details about type punning. + * + * The memcpy() implementation does not emit a compiler warning if 'src' has + * not the same type than 'src': any pointer type is accepted for 'src'. */ - -#define Py_SETREF(op, op2) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - (op) = (op2); \ - Py_DECREF(_py_tmp); \ +#ifdef _Py_TYPEOF +#define Py_SETREF(dst, src) \ + do { \ + _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \ + _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_DECREF(_tmp_old_dst); \ } while (0) +#else +#define Py_SETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ + PyObject *_tmp_src = _PyObject_CAST(src); \ + memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_dst); \ + } while (0) +#endif -#define Py_XSETREF(op, op2) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - (op) = (op2); \ - Py_XDECREF(_py_tmp); \ +/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of + * Py_DECREF(). + */ +#ifdef _Py_TYPEOF +#define Py_XSETREF(dst, src) \ + do { \ + _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \ + _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_XDECREF(_tmp_old_dst); \ + } while (0) +#else +#define Py_XSETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ + PyObject *_tmp_src = _PyObject_CAST(src); \ + memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ + Py_XDECREF(_tmp_old_dst); \ } while (0) +#endif PyAPI_DATA(PyTypeObject) _PyNone_Type; diff --git a/Include/object.h b/Include/object.h index 75624fe8c77a..3774f1267300 100644 --- a/Include/object.h +++ b/Include/object.h @@ -598,15 +598,44 @@ static inline void Py_DECREF(PyObject *op) * one of those can't cause problems -- but in part that relies on that * Python integers aren't currently weakly referencable. Best practice is * to use Py_CLEAR() even if you can't think of a reason for why you need to. + * + * gh-98724: Use a temporary variable to only evaluate the macro argument once, + * to avoid the duplication of side effects if the argument has side effects. + * + * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, + * the code can be miscompiled with strict aliasing because of type punning. + * With strict aliasing, a compiler considers that two pointers of different + * types cannot read or write the same memory which enables optimization + * opportunities. + * + * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, + * and so avoid type punning. Otherwise, use memcpy() which causes type erasure + * and so prevents the compiler to reuse an old cached 'op' value after + * Py_CLEAR(). */ -#define Py_CLEAR(op) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - if (_py_tmp != NULL) { \ - (op) = NULL; \ - Py_DECREF(_py_tmp); \ - } \ +#ifdef _Py_TYPEOF +#define Py_CLEAR(op) \ + do { \ + _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ + _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + *_tmp_op_ptr = _Py_NULL; \ + Py_DECREF(_tmp_old_op); \ + } \ } while (0) +#else +#define Py_CLEAR(op) \ + do { \ + PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ + PyObject *_tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + PyObject *_null_ptr = _Py_NULL; \ + memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#endif + /* Function to use in case the object pointer can be NULL: */ static inline void Py_XINCREF(PyObject *op) diff --git a/Include/pyport.h b/Include/pyport.h index b3ff2f4882e9..b1b2a7477969 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -698,6 +698,15 @@ extern char * _getpty(int *, int, mode_t, int); # define _Py__has_builtin(x) 0 #endif +// _Py_TYPEOF(expr) gets the type of an expression. +// +// Example: _Py_TYPEOF(x) x_copy = (x); +// +// The macro is only defined if GCC or clang compiler is used. +#if defined(__GNUC__) || defined(__clang__) +# define _Py_TYPEOF(expr) __typeof__(expr) +#endif + /* A convenient way for code to know if sanitizers are enabled. */ #if defined(__has_feature) diff --git a/Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst b/Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst new file mode 100644 index 000000000000..e6c26cde168d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-04-16-13-35.gh-issue-98724.p0urWO.rst @@ -0,0 +1,3 @@ +The :c:macro:`Py_CLEAR`, :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros +now only evaluate their arguments once. If an argument has side effects, these +side effects are no longer duplicated. Patch by Victor Stinner. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3617fafe9b4f..83eef73a875d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2589,6 +2589,91 @@ test_set_type_size(PyObject *self, PyObject *Py_UNUSED(ignored)) } +// Test Py_CLEAR() macro +static PyObject* +test_py_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // simple case with a variable + PyObject *obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + Py_CLEAR(obj); + assert(obj == NULL); + + // gh-98724: complex case, Py_CLEAR() argument has a side effect + PyObject* array[1]; + array[0] = PyList_New(0); + if (array[0] == NULL) { + return NULL; + } + + PyObject **p = array; + Py_CLEAR(*p++); + assert(array[0] == NULL); + assert(p == array + 1); + + Py_RETURN_NONE; +} + + +// Test Py_SETREF() and Py_XSETREF() macros, similar to test_py_clear() +static PyObject* +test_py_setref(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // Py_SETREF() simple case with a variable + PyObject *obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + Py_SETREF(obj, NULL); + assert(obj == NULL); + + // Py_XSETREF() simple case with a variable + PyObject *obj2 = PyList_New(0); + if (obj2 == NULL) { + return NULL; + } + Py_XSETREF(obj2, NULL); + assert(obj2 == NULL); + // test Py_XSETREF() when the argument is NULL + Py_XSETREF(obj2, NULL); + assert(obj2 == NULL); + + // gh-98724: complex case, Py_SETREF() argument has a side effect + PyObject* array[1]; + array[0] = PyList_New(0); + if (array[0] == NULL) { + return NULL; + } + + PyObject **p = array; + Py_SETREF(*p++, NULL); + assert(array[0] == NULL); + assert(p == array + 1); + + // gh-98724: complex case, Py_XSETREF() argument has a side effect + PyObject* array2[1]; + array2[0] = PyList_New(0); + if (array2[0] == NULL) { + return NULL; + } + + PyObject **p2 = array2; + Py_XSETREF(*p2++, NULL); + assert(array2[0] == NULL); + assert(p2 == array2 + 1); + + // test Py_XSETREF() when the argument is NULL + p2 = array2; + Py_XSETREF(*p2++, NULL); + assert(array2[0] == NULL); + assert(p2 == array2 + 1); + + Py_RETURN_NONE; +} + + #define TEST_REFCOUNT() \ do { \ PyObject *obj = PyList_New(0); \ @@ -3252,6 +3337,8 @@ static PyMethodDef TestMethods[] = { {"pynumber_tobase", pynumber_tobase, METH_VARARGS}, {"without_gc", without_gc, METH_O}, {"test_set_type_size", test_set_type_size, METH_NOARGS}, + {"test_py_clear", test_py_clear, METH_NOARGS}, + {"test_py_setref", test_py_setref, METH_NOARGS}, {"test_refcount_macros", test_refcount_macros, METH_NOARGS}, {"test_refcount_funcs", test_refcount_funcs, METH_NOARGS}, {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, From webhook-mailer at python.org Wed Dec 7 11:55:43 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 07 Dec 2022 16:55:43 -0000 Subject: [Python-checkins] gh-83035: handle decorator with nested parens in inspect.getsource (#99654) Message-ID: <mailman.2842.1670432145.3313.python-checkins@python.org> https://github.com/python/cpython/commit/68e41295b8611a990de68f15c89f1eb3dea51867 commit: 68e41295b8611a990de68f15c89f1eb3dea51867 branch: main author: Carl Meyer <carl at oddbird.net> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-07T08:55:12-08:00 summary: gh-83035: handle decorator with nested parens in inspect.getsource (#99654) files: A Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst M Lib/inspect.py M Lib/test/inspect_fodder2.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 31ac888126b5..e165937e448a 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1160,7 +1160,6 @@ def __init__(self): self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 self.body_col0 = None @@ -1175,13 +1174,6 @@ def tokeneater(self, type, token, srowcol, erowcol, line): self.islambda = True self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] @@ -1189,7 +1181,7 @@ def tokeneater(self, type, token, srowcol, erowcol, line): raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass diff --git a/Lib/test/inspect_fodder2.py b/Lib/test/inspect_fodder2.py index e7d4b53ebefc..2dc49817087c 100644 --- a/Lib/test/inspect_fodder2.py +++ b/Lib/test/inspect_fodder2.py @@ -259,3 +259,17 @@ def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): #line 259 def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + + at deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 3f5c299ce681..2b7977b1648f 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -886,6 +886,12 @@ def test_class(self): self.assertSourceEqual(self.fodderModule.X, 1, 2) +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + class _BrokenDataDescriptor(object): """ A broken data descriptor. See bug #1785. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst new file mode 100644 index 000000000000..629d9aefb2d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst @@ -0,0 +1 @@ +Fix :func:`inspect.getsource` handling of decorator calls with nested parentheses. From webhook-mailer at python.org Wed Dec 7 12:25:03 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 07 Dec 2022 17:25:03 -0000 Subject: [Python-checkins] gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) Message-ID: <mailman.2843.1670433904.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fe7c309a8c5d8abedd97dd271e94eb2c0f858c12 commit: fe7c309a8c5d8abedd97dd271e94eb2c0f858c12 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T09:24:41-08:00 summary: gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) (cherry picked from commit 68e41295b8611a990de68f15c89f1eb3dea51867) Co-authored-by: Carl Meyer <carl at oddbird.net> files: A Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst M Lib/inspect.py M Lib/test/inspect_fodder2.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index ad16f5c46e3b..417371f5cf14 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1052,7 +1052,6 @@ def __init__(self): self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 self.body_col0 = None @@ -1067,13 +1066,6 @@ def tokeneater(self, type, token, srowcol, erowcol, line): self.islambda = True self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] @@ -1081,7 +1073,7 @@ def tokeneater(self, type, token, srowcol, erowcol, line): raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass diff --git a/Lib/test/inspect_fodder2.py b/Lib/test/inspect_fodder2.py index e7d4b53ebefc..2dc49817087c 100644 --- a/Lib/test/inspect_fodder2.py +++ b/Lib/test/inspect_fodder2.py @@ -259,3 +259,17 @@ def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): #line 259 def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + + at deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 03cb3bdddc2c..5aa4a60de0bf 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -825,6 +825,12 @@ def test_class(self): self.assertSourceEqual(self.fodderModule.X, 1, 2) +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + class _BrokenDataDescriptor(object): """ A broken data descriptor. See bug #1785. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst new file mode 100644 index 000000000000..629d9aefb2d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst @@ -0,0 +1 @@ +Fix :func:`inspect.getsource` handling of decorator calls with nested parentheses. From webhook-mailer at python.org Wed Dec 7 12:26:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 07 Dec 2022 17:26:17 -0000 Subject: [Python-checkins] gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) Message-ID: <mailman.2844.1670433979.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2997f3913a7f644352a1d4013588451837d2ac58 commit: 2997f3913a7f644352a1d4013588451837d2ac58 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T09:26:11-08:00 summary: gh-83035: handle decorator with nested parens in inspect.getsource (GH-99654) (cherry picked from commit 68e41295b8611a990de68f15c89f1eb3dea51867) Co-authored-by: Carl Meyer <carl at oddbird.net> files: A Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst M Lib/inspect.py M Lib/test/inspect_fodder2.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index d1a9daf28913..cab92b6b8fb8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1175,7 +1175,6 @@ def __init__(self): self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 self.body_col0 = None @@ -1190,13 +1189,6 @@ def tokeneater(self, type, token, srowcol, erowcol, line): self.islambda = True self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] @@ -1204,7 +1196,7 @@ def tokeneater(self, type, token, srowcol, erowcol, line): raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass diff --git a/Lib/test/inspect_fodder2.py b/Lib/test/inspect_fodder2.py index e7d4b53ebefc..2dc49817087c 100644 --- a/Lib/test/inspect_fodder2.py +++ b/Lib/test/inspect_fodder2.py @@ -259,3 +259,17 @@ def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): #line 259 def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + + at deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index a54e6eb53e68..6c5e674aa69f 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -886,6 +886,12 @@ def test_class(self): self.assertSourceEqual(self.fodderModule.X, 1, 2) +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + class _BrokenDataDescriptor(object): """ A broken data descriptor. See bug #1785. diff --git a/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst new file mode 100644 index 000000000000..629d9aefb2d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-21-16-24-01.gh-issue-83035.qZIujU.rst @@ -0,0 +1 @@ +Fix :func:`inspect.getsource` handling of decorator calls with nested parentheses. From webhook-mailer at python.org Wed Dec 7 13:09:34 2022 From: webhook-mailer at python.org (iritkatriel) Date: Wed, 07 Dec 2022 18:09:34 -0000 Subject: [Python-checkins] gh-100077: make test_code.test_invalid_bytecode more robust and maintainable (#100078) Message-ID: <mailman.2845.1670436575.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f3e97c90ed6f82fce67b0e8757eec54908ba49ce commit: f3e97c90ed6f82fce67b0e8757eec54908ba49ce branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-07T18:09:05Z summary: gh-100077: make test_code.test_invalid_bytecode more robust and maintainable (#100078) files: M Lib/test/test_code.py diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 4e4d82314a9f..02ab8fbcdb07 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -143,7 +143,7 @@ gc_collect) from test.support.script_helper import assert_python_ok from test.support import threading_helper -from opcode import opmap +from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -339,15 +339,19 @@ def func(): self.assertEqual(list(new_code.co_lines()), []) def test_invalid_bytecode(self): - def foo(): pass - foo.__code__ = co = foo.__code__.replace(co_code=b'\xee\x00d\x00S\x00') + def foo(): + pass - with self.assertRaises(SystemError) as se: - foo() - self.assertEqual( - f"{co.co_filename}:{co.co_firstlineno}: unknown opcode 238", - str(se.exception)) + # assert that opcode 238 is invalid + self.assertEqual(opname[238], '<238>') + # change first opcode to 0xee (=238) + foo.__code__ = foo.__code__.replace( + co_code=b'\xee' + foo.__code__.co_code[1:]) + + msg = f"unknown opcode 238" + with self.assertRaisesRegex(SystemError, msg): + foo() @requires_debug_ranges() def test_co_positions_artificial_instructions(self): From webhook-mailer at python.org Wed Dec 7 14:08:01 2022 From: webhook-mailer at python.org (hugovk) Date: Wed, 07 Dec 2022 19:08:01 -0000 Subject: [Python-checkins] gh-100072: only trigger netlify builds for doc changes (#100074) Message-ID: <mailman.2846.1670440082.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d92407ed497e3fc5acacb0294ab6095013e600f4 commit: d92407ed497e3fc5acacb0294ab6095013e600f4 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-07T21:07:30+02:00 summary: gh-100072: only trigger netlify builds for doc changes (#100074) Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> files: M netlify.toml diff --git a/netlify.toml b/netlify.toml index 52675b3d701e..f5790fc5fec7 100644 --- a/netlify.toml +++ b/netlify.toml @@ -2,7 +2,10 @@ base = "Doc/" command = "make html" publish = "build/html" + # Do not trigger netlify builds if docs were not changed. + # Changed files should be in sync with `.github/workflows/doc.yml` + ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF . ../netlify.toml" [build.environment] PYTHON_VERSION = "3.8" - IS_DEPLOYMENT_PREVIEW = "true" \ No newline at end of file + IS_DEPLOYMENT_PREVIEW = "true" From webhook-mailer at python.org Wed Dec 7 17:02:58 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Wed, 07 Dec 2022 22:02:58 -0000 Subject: [Python-checkins] gh-90110: Clean Up the C-analyzer Globals Lists (gh-100091) Message-ID: <mailman.2847.1670450580.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d47ffeb9e35dbc7264ffa12fddaa6e0d3ba767a4 commit: d47ffeb9e35dbc7264ffa12fddaa6e0d3ba767a4 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-07T15:02:47-07:00 summary: gh-90110: Clean Up the C-analyzer Globals Lists (gh-100091) https://github.com/python/cpython/issues/90110 files: M Parser/action_helpers.c M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv M Tools/c-analyzer/table-file.py diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 27c093332f67..f12dad095aca 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -13,6 +13,7 @@ void * _PyPegen_dummy_name(Parser *p, ...) { // XXX This leaks memory from the initial arena. + // Use a statically allocated variable instead of a pointer? static void *cache = NULL; if (cache != NULL) { @@ -1287,4 +1288,4 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq _PyPegen_get_last_comprehension_item(last_comprehension), "Generator expression must be parenthesized" ); -} \ No newline at end of file +} diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index adbb319a1b6f..af5b7eb8eaf0 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -4,10 +4,10 @@ filename funcname name reason # These are all variables that we will be making non-global. ################################## -# global objects to fix in core code +## global objects to fix in core code -#----------------------- -# exported builtin types (C-API) +##----------------------- +## exported builtin types (C-API) Objects/boolobject.c - PyBool_Type - Objects/bytearrayobject.c - PyByteArrayIter_Type - @@ -102,8 +102,8 @@ Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - Python/traceback.c - PyTraceBack_Type - -#----------------------- -# other exported builtin types +##----------------------- +## other exported builtin types # Not in a .h file: Objects/codeobject.c - _PyLineIterator - @@ -126,8 +126,8 @@ Python/hamt.c - _PyHamt_CollisionNode_Type - Python/hamt.c - _PyHamt_Type - Python/symtable.c - PySTEntry_Type - -#----------------------- -# private static builtin types +##----------------------- +## private static builtin types Objects/setobject.c - _PySetDummy_Type - Objects/stringlib/unicode_format.h - PyFormatterIter_Type - @@ -136,8 +136,8 @@ Objects/unicodeobject.c - EncodingMapType - #Objects/unicodeobject.c - PyFieldNameIter_Type - #Objects/unicodeobject.c - PyFormatterIter_Type - -#----------------------- -# static builtin structseq +##----------------------- +## static builtin structseq Objects/floatobject.c - FloatInfoType - Objects/longobject.c - Int_InfoType - @@ -148,8 +148,8 @@ Python/sysmodule.c - Hash_InfoType - Python/sysmodule.c - VersionInfoType - Python/thread.c - ThreadInfoType - -#----------------------- -# builtin exception types +##----------------------- +## builtin exception types Objects/exceptions.c - _PyExc_BaseException - Objects/exceptions.c - _PyExc_BaseExceptionGroup - @@ -286,8 +286,8 @@ Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - -#----------------------- -# singletons +##----------------------- +## singletons Objects/boolobject.c - _Py_FalseStruct - Objects/boolobject.c - _Py_TrueStruct - @@ -300,26 +300,26 @@ Objects/sliceobject.c - _Py_EllipsisObject - ################################## -# global non-objects to fix in core code +## global non-objects to fix in core code -#----------------------- -# effectively-const but initialized lazily +##----------------------- +## effectively-const but initialized lazily -# idempotent +## idempotent Python/dtoa.c - p5s - Objects/obmalloc.c new_arena debug_stats - -# others +## others Python/perf_trampoline.c - perf_map_file - Objects/unicodeobject.c - ucnhash_capi - -#----------------------- -# state +##----------------------- +## state -# local buffer +## local buffer Python/suggestions.c levenshtein_distance buffer - -# other +## other Objects/object.c - _Py_RefTotal - Python/perf_trampoline.c - perf_status - Python/perf_trampoline.c - extra_code_index - @@ -329,10 +329,10 @@ Python/thread_pthread_stubs.h - py_tls_entries - ################################## -# global objects to fix in builtin modules +## global objects to fix in builtin modules -#----------------------- -# static types +##----------------------- +## static types Modules/_collectionsmodule.c - defdict_type - Modules/_collectionsmodule.c - deque_type - @@ -381,10 +381,10 @@ Modules/itertoolsmodule.c - tee_type - Modules/itertoolsmodule.c - teedataobject_type - Modules/itertoolsmodule.c - ziplongest_type - -#----------------------- -# other +##----------------------- +## other -# state +## state Modules/faulthandler.c - fatal_error - Modules/faulthandler.c - thread - Modules/faulthandler.c - user_signals - @@ -393,10 +393,10 @@ Modules/faulthandler.c - old_stack - ################################## -# global non-objects to fix in builtin modules +## global non-objects to fix in builtin modules -#----------------------- -# initialized once +##----------------------- +## initialized once Modules/_io/bufferedio.c _PyIO_trap_eintr eintr_int - Modules/posixmodule.c os_dup2_impl dup3_works - @@ -405,8 +405,8 @@ Modules/posixmodule.c - ticks_per_second - Modules/timemodule.c _PyTime_GetClockWithInfo initialized - Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - -#----------------------- -# state +##----------------------- +## state Modules/_tracemalloc.c - allocators - Modules/_tracemalloc.c - tables_lock - @@ -420,7 +420,6 @@ Modules/_tracemalloc.c - tracemalloc_traces - Modules/_tracemalloc.c - tracemalloc_domains - Modules/_tracemalloc.c - tracemalloc_reentrant_key - Modules/faulthandler.c faulthandler_dump_traceback reentrant - -Modules/posixmodule.c - environ - Modules/signalmodule.c - is_tripped - Modules/signalmodule.c - signal_global_state - Modules/signalmodule.c - wakeup - @@ -428,10 +427,10 @@ Modules/signalmodule.c - Handlers - ################################## -# global objects to fix in extension modules +## global objects to fix in extension modules -#----------------------- -# static types +##----------------------- +## static types Modules/_asynciomodule.c - FutureIterType - Modules/_asynciomodule.c - FutureType - @@ -506,10 +505,10 @@ Modules/xxmodule.c - Xxo_Type - Modules/xxsubtype.c - spamdict_type - Modules/xxsubtype.c - spamlist_type - -#----------------------- -# non-static types - initialized once +##----------------------- +## non-static types - initialized once -# heap types +## heap types Modules/_decimal/_decimal.c - DecimalTuple - Modules/_decimal/_decimal.c - PyDecSignalDict_Type - Modules/_tkinter.c - PyTclObject_Type - @@ -517,7 +516,7 @@ Modules/_tkinter.c - Tkapp_Type - Modules/_tkinter.c - Tktt_Type - Modules/xxlimited_35.c - Xxo_Type - -# exception types +## exception types Modules/_ctypes/_ctypes.c - PyExc_ArgError - Modules/_cursesmodule.c - PyCursesError - Modules/_decimal/_decimal.c - DecimalException - @@ -528,15 +527,15 @@ Modules/socketmodule.c - socket_gaierror - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - -#----------------------- -# cached - initialized once +##----------------------- +## cached - initialized once -# manually cached PyUnicodeOjbect +## manually cached PyUnicodeOjbect Modules/_asynciomodule.c - context_kwname - Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - Modules/_ctypes/_ctypes.c CreateSwappedType suffix - -# other - during module init +## other - during module init Modules/_asynciomodule.c - asyncio_mod - Modules/_asynciomodule.c - traceback_extract_stack - Modules/_asynciomodule.c - asyncio_future_repr_func - @@ -551,10 +550,10 @@ Modules/_zoneinfo.c - io_open - Modules/_zoneinfo.c - _tzpath_find_tzfile - Modules/_zoneinfo.c - _common_mod - -#----------------------- -# other +##----------------------- +## other -# initialized once +## initialized once Modules/_ctypes/_ctypes.c - _unpickle - Modules/_ctypes/_ctypes.c PyCArrayType_from_ctype cache - Modules/_cursesmodule.c - ModDict - @@ -577,7 +576,7 @@ Modules/_decimal/_decimal.c - Rational - Modules/_decimal/_decimal.c - SignalTuple - Modules/arraymodule.c array_array___reduce_ex___impl array_reconstructor - -# state +## state Modules/_asynciomodule.c - cached_running_holder - Modules/_asynciomodule.c - fi_freelist - Modules/_asynciomodule.c - fi_freelist_len - @@ -592,20 +591,19 @@ Modules/_tkinter.c - valInCmd - Modules/_tkinter.c - trbInCmd - Modules/_zoneinfo.c - TIMEDELTA_CACHE - Modules/_zoneinfo.c - ZONEINFO_WEAK_CACHE - -Modules/syslogmodule.c - S_ident_o - ################################## -# global non-objects to fix in extension modules +## global non-objects to fix in extension modules -#----------------------- -# initialized once +##----------------------- +## initialized once -# pre-allocated buffer +## pre-allocated buffer Modules/nismodule.c nisproc_maplist_2 res - Modules/pyexpat.c PyUnknownEncodingHandler template_buffer - -# other +## other Include/datetime.h - PyDateTimeAPI - Modules/_asynciomodule.c - module_initialized - Modules/_ctypes/cfield.c _ctypes_get_fielddesc initialized - @@ -650,8 +648,8 @@ Modules/readline.c - libedit_history_start - Modules/socketmodule.c - accept4_works - Modules/socketmodule.c - sock_cloexec_works - -#----------------------- -# state +##----------------------- +## state Modules/_asynciomodule.c - cached_running_holder_tsid - Modules/_asynciomodule.c - task_name_counter - @@ -681,4 +679,3 @@ Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - Modules/socketmodule.c - defaulttimeout - -Modules/syslogmodule.c - S_log_open - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 242deace8c94..128336a997eb 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -5,52 +5,61 @@ filename funcname name reason ################################## -# process-global resources +## process-global values - set once -# Initialization for these should be idempotent. +# These will never re-initialize (but would be idempotent). +# These are effectively const. -#----------------------- -# effectively const, set once before/during first init +##----------------------- +## process-global resources - set during first init -Modules/getbuildinfo.c - buildinfo - -Modules/getbuildinfo.c - initialized - -Python/getversion.c - initialized - -Python/getversion.c - version - - -#----------------------- -# effectively const, set once during first init +## indicators for resource availability/capability +Python/bootstrap_hash.c py_getrandom getrandom_works - +Python/fileutils.c - _Py_open_cloexec_works - +Python/fileutils.c set_inheritable ioctl_works - -Python/bootstrap_hash.c - _Py_HashSecret_Initialized - -Python/pyhash.c - _Py_HashSecret - +## resource init Python/thread.c - initialized - Python/thread_pthread.h - condattr_monotonic - - # safe static buffer used during one-time initialization Python/thread_pthread.h init_condattr ca - -# indicators for process-global resource availability/capability -Python/bootstrap_hash.c py_getrandom getrandom_works - -Python/fileutils.c - _Py_open_cloexec_works - -Python/fileutils.c set_inheritable ioctl_works - - -#----------------------- -# effectively const but set once lazily (*after* first init) +##----------------------- +## other values (not Python-specific) +## cached computed data - set lazily (*after* first init) +# XXX Are these safe relative to write races? Objects/longobject.c long_from_non_binary_base log_base_BASE - Objects/longobject.c long_from_non_binary_base convwidth_base - Objects/longobject.c long_from_non_binary_base convmultmax_base - Objects/unicodeobject.c - bloom_linebreak - +# This is safe: Objects/unicodeobject.c _init_global_state initialized - -# XXX Move to _PyRuntimeState? +##----------------------- +## other values (Python-specific) + +## internal state - set before/during first init +Modules/getbuildinfo.c - buildinfo - +Modules/getbuildinfo.c - initialized - +Python/getversion.c - initialized - +Python/getversion.c - version - + +## public C-API - set during first init +Python/bootstrap_hash.c - _Py_HashSecret_Initialized - +Python/pyhash.c - _Py_HashSecret - + +## internal state - set lazily (*after* first init) +# XXX Move to _PyRuntimeState (i.e. tie to init/fini cycle)? Parser/action_helpers.c _PyPegen_dummy_name cache - ################################## -# state tied to C main() (only in main thread) +## state tied to Py_Main() +# (only in main thread) -#----------------------- -# handling C argv +##----------------------- +## handling C argv Python/getopt.c - _PyOS_optarg - Python/getopt.c - _PyOS_opterr - @@ -58,8 +67,8 @@ Python/getopt.c - _PyOS_optind - Python/getopt.c - opt_ptr - Python/pathconfig.c - _Py_path_config - -#----------------------- -# REPL +##----------------------- +## REPL Parser/myreadline.c - _PyOS_ReadlineLock - Parser/myreadline.c - _PyOS_ReadlineTState - @@ -68,28 +77,29 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - ################################## -# state tied to each runtime init/fini cycle +## runtime-global values - set once with each init -Python/pylifecycle.c - _PyRuntime - -Python/pylifecycle.c - runtime_initialized - +# These are effectively const. -# All uses of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py. +##----------------------- +## set by embedders before init +# (whether directly or through a call) -#----------------------- -# effectively const once init finishes - -# set by embedders before init (whether directly or through a call) Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - +# XXX This only gets cleared by Py_Main(). Python/initconfig.c - orig_argv - -# deprecated +##----------------------- +## public C-API + +## deprecated Python/preconfig.c - Py_FileSystemDefaultEncoding - Python/preconfig.c - Py_HasFileSystemDefaultEncoding - Python/preconfig.c - Py_FileSystemDefaultEncodeErrors - Python/preconfig.c - _Py_HasFileSystemDefaultEncodeErrors - -# legacy config flags +## legacy config flags Python/initconfig.c - Py_UTF8Mode - Python/initconfig.c - Py_DebugFlag - Python/initconfig.c - Py_VerboseFlag - @@ -109,373 +119,70 @@ Python/initconfig.c - Py_IsolatedFlag - Python/initconfig.c - Py_LegacyWindowsFSEncodingFlag - Python/initconfig.c - Py_LegacyWindowsStdioFlag - -# initialized statically, customized by embedders +##----------------------- +## initialized statically, may be customized by embedders + Python/frozen.c - PyImport_FrozenModules - Python/import.c - inittab_copy - Python/import.c - PyImport_Inittab - -# used temporarily during init -Python/sysmodule.c - _preinit_warnoptions - -Python/sysmodule.c - _preinit_xoptions - - ################################## -# special-use diagnistic state +## runtime-global state -Parser/pegen.c - memo_statistics - +##----------------------- +## tied to each init/fini cycle +## the consolidated runtime state +Python/pylifecycle.c - _PyRuntime - +Python/pylifecycle.c - runtime_initialized - -################################## -# one-off temporary state +# All cases of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py. -# This is safe enough. -Python/pylifecycle.c _Py_FatalErrorFormat reentrant - -Python/pylifecycle.c fatal_error reentrant - +## main interp state in stdlib modules +Modules/syslogmodule.c - S_ident_o - +Modules/syslogmodule.c - S_log_open - +##----------------------- +## *not* tied to init/fini cycle -################################## -# not used (kept for compatibility) +# These do not ge reset with each init/fini cycle. +# XXX These should probably be tied to init/fini. Move to _PyRuntimeState? -Python/pyfpe.c - PyFPE_counter - +# special-use diagnistic state +Parser/pegen.c - memo_statistics - + +##----------------------- +## one-off temporary state + +# used during runtime init +Python/sysmodule.c - _preinit_warnoptions - +Python/sysmodule.c - _preinit_xoptions - + +# thread-safety +# XXX need race protection? +Python/pylifecycle.c _Py_FatalErrorFormat reentrant - +Python/pylifecycle.c fatal_error reentrant - ################################## -# The analyzer should have ignored these. -# XXX Fix the analyzer. +## not significant -Modules/_io/_iomodule.c - _PyIO_Module - -Modules/_sqlite/module.c - _sqlite3module - +##----------------------- +## not used (kept for compatibility) -# forward/extern references +Python/pyfpe.c - PyFPE_counter - -Include/py_curses.h - PyCurses_API - -Include/pydecimal.h - _decimal_api - -Modules/_blake2/blake2module.c - blake2b_type_spec - -Modules/_blake2/blake2module.c - blake2s_type_spec - -Modules/_io/fileio.c - _Py_open_cloexec_works - -Modules/_io/_iomodule.h - PyIOBase_Type - -Modules/_io/_iomodule.h - PyRawIOBase_Type - -Modules/_io/_iomodule.h - PyBufferedIOBase_Type - -Modules/_io/_iomodule.h - PyTextIOBase_Type - -Modules/_io/_iomodule.h - PyFileIO_Type - -Modules/_io/_iomodule.h - PyBytesIO_Type - -Modules/_io/_iomodule.h - PyStringIO_Type - -Modules/_io/_iomodule.h - PyBufferedReader_Type - -Modules/_io/_iomodule.h - PyBufferedWriter_Type - -Modules/_io/_iomodule.h - PyBufferedRWPair_Type - -Modules/_io/_iomodule.h - PyBufferedRandom_Type - -Modules/_io/_iomodule.h - PyTextIOWrapper_Type - -Modules/_io/_iomodule.h - PyIncrementalNewlineDecoder_Type - -Modules/_io/_iomodule.h - _PyBytesIOBuffer_Type - -Modules/_io/_iomodule.h - _PyIO_Module - -Modules/_io/_iomodule.h - _PyIO_str_close - -Modules/_io/_iomodule.h - _PyIO_str_closed - -Modules/_io/_iomodule.h - _PyIO_str_decode - -Modules/_io/_iomodule.h - _PyIO_str_encode - -Modules/_io/_iomodule.h - _PyIO_str_fileno - -Modules/_io/_iomodule.h - _PyIO_str_flush - -Modules/_io/_iomodule.h - _PyIO_str_getstate - -Modules/_io/_iomodule.h - _PyIO_str_isatty - -Modules/_io/_iomodule.h - _PyIO_str_newlines - -Modules/_io/_iomodule.h - _PyIO_str_nl - -Modules/_io/_iomodule.h - _PyIO_str_peek - -Modules/_io/_iomodule.h - _PyIO_str_read - -Modules/_io/_iomodule.h - _PyIO_str_read1 - -Modules/_io/_iomodule.h - _PyIO_str_readable - -Modules/_io/_iomodule.h - _PyIO_str_readall - -Modules/_io/_iomodule.h - _PyIO_str_readinto - -Modules/_io/_iomodule.h - _PyIO_str_readline - -Modules/_io/_iomodule.h - _PyIO_str_reset - -Modules/_io/_iomodule.h - _PyIO_str_seek - -Modules/_io/_iomodule.h - _PyIO_str_seekable - -Modules/_io/_iomodule.h - _PyIO_str_setstate - -Modules/_io/_iomodule.h - _PyIO_str_tell - -Modules/_io/_iomodule.h - _PyIO_str_truncate - -Modules/_io/_iomodule.h - _PyIO_str_writable - -Modules/_io/_iomodule.h - _PyIO_str_write - -Modules/_io/_iomodule.h - _PyIO_empty_str - -Modules/_io/_iomodule.h - _PyIO_empty_bytes - -Modules/_multiprocessing/multiprocessing.h - _PyMp_SemLockType - -Modules/_sqlite/module.c - _pysqlite_converters - -Modules/_sqlite/module.c - _pysqlite_enable_callback_tracebacks - -Modules/_sqlite/module.c - pysqlite_BaseTypeAdapted - -Modules/_sqlite/module.h - pysqlite_global_state - -Modules/_testcapimodule.c - _PyBytesIOBuffer_Type - -Modules/posixmodule.c - _Py_open_cloexec_works - -Objects/object.c - _Py_GenericAliasIterType - -Objects/object.c - _PyMemoryIter_Type - -Objects/object.c - _PyLineIterator - -Objects/object.c - _PyPositionsIterator - -Python/perf_trampoline.c - _Py_trampoline_func_start - -Python/perf_trampoline.c - _Py_trampoline_func_end - -Python/importdl.h - _PyImport_DynLoadFiletab - +##----------------------- +## should be const +# XXX Make them const. -Modules/expat/xmlrole.c - prolog0 - -Modules/expat/xmlrole.c - prolog1 - -Modules/expat/xmlrole.c - prolog2 - -Modules/expat/xmlrole.c - doctype0 - -Modules/expat/xmlrole.c - doctype1 - -Modules/expat/xmlrole.c - doctype2 - -Modules/expat/xmlrole.c - doctype3 - -Modules/expat/xmlrole.c - doctype4 - -Modules/expat/xmlrole.c - doctype5 - -Modules/expat/xmlrole.c - internalSubset - -Modules/expat/xmlrole.c - entity0 - -Modules/expat/xmlrole.c - entity1 - -Modules/expat/xmlrole.c - entity2 - -Modules/expat/xmlrole.c - entity3 - -Modules/expat/xmlrole.c - entity4 - -Modules/expat/xmlrole.c - entity5 - -Modules/expat/xmlrole.c - entity6 - -Modules/expat/xmlrole.c - entity7 - -Modules/expat/xmlrole.c - entity8 - -Modules/expat/xmlrole.c - entity9 - -Modules/expat/xmlrole.c - entity10 - -Modules/expat/xmlrole.c - notation0 - -Modules/expat/xmlrole.c - notation1 - -Modules/expat/xmlrole.c - notation2 - -Modules/expat/xmlrole.c - notation3 - -Modules/expat/xmlrole.c - notation4 - -Modules/expat/xmlrole.c - attlist0 - -Modules/expat/xmlrole.c - attlist1 - -Modules/expat/xmlrole.c - attlist2 - -Modules/expat/xmlrole.c - attlist3 - -Modules/expat/xmlrole.c - attlist4 - -Modules/expat/xmlrole.c - attlist5 - -Modules/expat/xmlrole.c - attlist6 - -Modules/expat/xmlrole.c - attlist7 - -Modules/expat/xmlrole.c - attlist8 - -Modules/expat/xmlrole.c - attlist9 - -Modules/expat/xmlrole.c - element0 - -Modules/expat/xmlrole.c - element1 - -Modules/expat/xmlrole.c - element2 - -Modules/expat/xmlrole.c - element3 - -Modules/expat/xmlrole.c - element4 - -Modules/expat/xmlrole.c - element5 - -Modules/expat/xmlrole.c - element6 - -Modules/expat/xmlrole.c - element7 - -Modules/expat/xmlrole.c - externalSubset0 - -Modules/expat/xmlrole.c - externalSubset1 - -Modules/expat/xmlrole.c - condSect0 - -Modules/expat/xmlrole.c - condSect1 - -Modules/expat/xmlrole.c - condSect2 - -Modules/expat/xmlrole.c - declClose - -Modules/expat/xmlrole.c - error - +# These are all variables that we will be leaving global. +# All module defs, type defs, etc. are handled in c-analyzr/cpython/_analyzer.py. +# All kwlist arrays are handled in c-analyzr/cpython/_analyzer.py. -################################## -# test code - -Modules/_ctypes/_ctypes_test.c - _ctypes_test_slots - -Modules/_ctypes/_ctypes_test.c - _ctypes_testmodule - -Modules/_ctypes/_ctypes_test.c - _xxx_lib - -Modules/_ctypes/_ctypes_test.c - an_integer - -Modules/_ctypes/_ctypes_test.c - bottom - -Modules/_ctypes/_ctypes_test.c - last_tf_arg_s - -Modules/_ctypes/_ctypes_test.c - last_tf_arg_u - -Modules/_ctypes/_ctypes_test.c - last_tfrsuv_arg - -Modules/_ctypes/_ctypes_test.c - left - -Modules/_ctypes/_ctypes_test.c - module_methods - -Modules/_ctypes/_ctypes_test.c - my_eggs - -Modules/_ctypes/_ctypes_test.c - my_spams - -Modules/_ctypes/_ctypes_test.c - right - -Modules/_ctypes/_ctypes_test.c - top - -Modules/_testbuffer.c - NDArray_Type - -Modules/_testbuffer.c - StaticArray_Type - -Modules/_testbuffer.c - Struct - -Modules/_testbuffer.c - _testbuffer_functions - -Modules/_testbuffer.c - _testbuffermodule - -Modules/_testbuffer.c - calcsize - -Modules/_testbuffer.c - infobuf - -Modules/_testbuffer.c - ndarray_as_buffer - -Modules/_testbuffer.c - ndarray_as_mapping - -Modules/_testbuffer.c - ndarray_as_sequence - -Modules/_testbuffer.c - ndarray_getset - -Modules/_testbuffer.c - ndarray_methods - -Modules/_testbuffer.c - simple_fmt - -Modules/_testbuffer.c - simple_format - -Modules/_testbuffer.c - static_buffer - -Modules/_testbuffer.c - static_mem - -Modules/_testbuffer.c - static_shape - -Modules/_testbuffer.c - static_strides - -Modules/_testbuffer.c - staticarray_as_buffer - -Modules/_testbuffer.c - structmodule - -Modules/_testbuffer.c ndarray_init kwlist - -Modules/_testbuffer.c ndarray_memoryview_from_buffer format - -Modules/_testbuffer.c ndarray_memoryview_from_buffer info - -Modules/_testbuffer.c ndarray_memoryview_from_buffer shape - -Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - -Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - -Modules/_testbuffer.c ndarray_push kwlist - -Modules/_testbuffer.c staticarray_init kwlist - -Modules/_testcapi/heaptype.c - _testcapimodule - -Modules/_testcapi/unicode.c - _testcapimodule - -Modules/_testcapimodule.c - ContainerNoGC_members - -Modules/_testcapimodule.c - ContainerNoGC_type - -Modules/_testcapimodule.c - FmData - -Modules/_testcapimodule.c - FmHook - -Modules/_testcapimodule.c - GenericAlias_Type - -Modules/_testcapimodule.c - Generic_Type - -Modules/_testcapimodule.c - HeapCTypeSetattr_slots - -Modules/_testcapimodule.c - HeapCTypeSetattr_spec - -Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_slots - -Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_spec - -Modules/_testcapimodule.c - HeapCTypeSubclass_slots - -Modules/_testcapimodule.c - HeapCTypeSubclass_spec - -Modules/_testcapimodule.c - HeapCTypeWithBuffer_slots - -Modules/_testcapimodule.c - HeapCTypeWithBuffer_spec - -Modules/_testcapimodule.c - HeapCTypeWithDict_slots - -Modules/_testcapimodule.c - HeapCTypeWithDict_spec - -Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_slots - -Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_spec - -Modules/_testcapimodule.c - HeapCTypeWithWeakref_slots - -Modules/_testcapimodule.c - HeapCTypeWithWeakref_spec - -Modules/_testcapimodule.c - HeapCType_slots - -Modules/_testcapimodule.c - HeapCType_spec - -Modules/_testcapimodule.c - HeapDocCType_slots - -Modules/_testcapimodule.c - HeapDocCType_spec - -Modules/_testcapimodule.c - HeapGcCType_slots - -Modules/_testcapimodule.c - HeapGcCType_spec - -Modules/_testcapimodule.c - MethClass_Type - -Modules/_testcapimodule.c - MethInstance_Type - -Modules/_testcapimodule.c - MethStatic_Type - -Modules/_testcapimodule.c - MethodDescriptor2_Type - -Modules/_testcapimodule.c - MethodDescriptorBase_Type - -Modules/_testcapimodule.c - MethodDescriptorDerived_Type - -Modules/_testcapimodule.c - MethodDescriptorNopGet_Type - -Modules/_testcapimodule.c - MyList_Type - -Modules/_testcapimodule.c - PyRecursingInfinitelyError_Type - -Modules/_testcapimodule.c - TestError - -Modules/_testcapimodule.c - TestMethods - -Modules/_testcapimodule.c - _HashInheritanceTester_Type - -Modules/_testcapimodule.c - _testcapimodule - -Modules/_testcapimodule.c - awaitType - -Modules/_testcapimodule.c - awaitType_as_async - -Modules/_testcapimodule.c - capsule_context - -Modules/_testcapimodule.c - capsule_destructor_call_count - -Modules/_testcapimodule.c - capsule_error - -Modules/_testcapimodule.c - capsule_name - -Modules/_testcapimodule.c - capsule_pointer - -Modules/_testcapimodule.c - decimal_initialized - -Modules/_testcapimodule.c - generic_alias_methods - -Modules/_testcapimodule.c - generic_methods - -Modules/_testcapimodule.c - heapctype_members - -Modules/_testcapimodule.c - heapctypesetattr_members - -Modules/_testcapimodule.c - heapctypesubclass_members - -Modules/_testcapimodule.c - heapctypewithdict_getsetlist - -Modules/_testcapimodule.c - heapctypewithdict_members - -Modules/_testcapimodule.c - heapctypewithnegativedict_members - -Modules/_testcapimodule.c - heapctypewithweakref_members - -Modules/_testcapimodule.c - ipowType - -Modules/_testcapimodule.c - ipowType_as_number - -Modules/_testcapimodule.c - matmulType - -Modules/_testcapimodule.c - matmulType_as_number - -Modules/_testcapimodule.c - meth_class_methods - -Modules/_testcapimodule.c - meth_instance_methods - -Modules/_testcapimodule.c - meth_static_methods - -Modules/_testcapimodule.c - ml - -Modules/_testcapimodule.c - str1 - -Modules/_testcapimodule.c - str2 - -Modules/_testcapimodule.c - test_members - -Modules/_testcapimodule.c - test_run_counter - -Modules/_testcapimodule.c - test_structmembersType - -Modules/_testcapimodule.c - thread_done - -Modules/_testcapimodule.c - x - -Modules/_testcapimodule.c getargs_keyword_only keywords - -Modules/_testcapimodule.c getargs_keywords keywords - -Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - -Modules/_testcapimodule.c getargs_s_hash_int2 keywords static char*[] -Modules/_testcapimodule.c make_exception_with_doc kwlist - -Modules/_testcapimodule.c raise_SIGINT_then_send_None PyId_send - -Modules/_testcapimodule.c slot_tp_del PyId___tp_del__ - -Modules/_testcapimodule.c test_capsule buffer - -Modules/_testcapimodule.c test_empty_argparse kwlist - -Modules/_testcapimodule.c test_structmembers_new keywords - -Modules/_testcapimodule.c getargs_s_hash_int keywords - -Modules/_testcapimodule.c - g_dict_watch_events - -Modules/_testcapimodule.c - g_dict_watchers_installed - -Modules/_testcapimodule.c - g_type_modified_events - -Modules/_testcapimodule.c - g_type_watchers_installed - -Modules/_testimportmultiple.c - _barmodule - -Modules/_testimportmultiple.c - _foomodule - -Modules/_testimportmultiple.c - _testimportmultiple - -Modules/_testinternalcapi.c - TestMethods - -Modules/_testinternalcapi.c - _testcapimodule - -Modules/_testmultiphase.c - Example_Type_slots - -Modules/_testmultiphase.c - Example_Type_spec - -Modules/_testmultiphase.c - Example_methods - -Modules/_testmultiphase.c - StateAccessType_Type_slots - -Modules/_testmultiphase.c - StateAccessType_methods - -Modules/_testmultiphase.c - StateAccessType_spec - -Modules/_testmultiphase.c - Str_Type_slots - -Modules/_testmultiphase.c - Str_Type_spec - -Modules/_testmultiphase.c - def_bad_large - -Modules/_testmultiphase.c - def_bad_negative - -Modules/_testmultiphase.c - def_create_int_with_state - -Modules/_testmultiphase.c - def_create_null - -Modules/_testmultiphase.c - def_create_raise - -Modules/_testmultiphase.c - def_create_unreported_exception - -Modules/_testmultiphase.c - def_exec_err - -Modules/_testmultiphase.c - def_exec_raise - -Modules/_testmultiphase.c - def_exec_unreported_exception - -Modules/_testmultiphase.c - def_meth_state_access - -Modules/_testmultiphase.c - def_negative_size - -Modules/_testmultiphase.c - def_nonascii_kana - -Modules/_testmultiphase.c - def_nonascii_latin - -Modules/_testmultiphase.c - def_nonmodule - -Modules/_testmultiphase.c - def_nonmodule_with_exec_slots - -Modules/_testmultiphase.c - def_nonmodule_with_methods - -Modules/_testmultiphase.c - imp_dummy_def - -Modules/_testmultiphase.c - main_def - -Modules/_testmultiphase.c - main_slots - -Modules/_testmultiphase.c - meth_state_access_slots - -Modules/_testmultiphase.c - nonmodule_methods - -Modules/_testmultiphase.c - null_slots_def - -Modules/_testmultiphase.c - slots_bad_large - -Modules/_testmultiphase.c - slots_bad_negative - -Modules/_testmultiphase.c - slots_create_nonmodule - -Modules/_testmultiphase.c - slots_create_nonmodule - -Modules/_testmultiphase.c - slots_create_null - -Modules/_testmultiphase.c - slots_create_raise - -Modules/_testmultiphase.c - slots_create_unreported_exception - -Modules/_testmultiphase.c - slots_exec_err - -Modules/_testmultiphase.c - slots_exec_raise - -Modules/_testmultiphase.c - slots_exec_unreported_exception - -Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - -Modules/_testmultiphase.c - testexport_methods - -Modules/_testmultiphase.c - uninitialized_def - -Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - -Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - -Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - -Modules/_xxtestfuzz/fuzzer.c - ast_literal_eval_method - -Modules/_xxtestfuzz/fuzzer.c - compiled_patterns - -Modules/_xxtestfuzz/fuzzer.c - csv_error - -Modules/_xxtestfuzz/fuzzer.c - csv_module - -Modules/_xxtestfuzz/fuzzer.c - json_loads_method - -Modules/_xxtestfuzz/fuzzer.c - regex_patterns - -Modules/_xxtestfuzz/fuzzer.c - sre_compile_method - -Modules/_xxtestfuzz/fuzzer.c - sre_error_exception - -Modules/_xxtestfuzz/fuzzer.c - struct_error - -Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_MATCH_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput STRUCT_UNPACK_INITIALIZED - -Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED - - - -################################## -# should be const -# XXX Make them const. - -# These are all variables that we will be leaving global. - -# All module defs, type defs, etc. are handled in c-analyzr/cpython/_analyzer.py. -# All kwlist arrays are handled in c-analyzr/cpython/_analyzer.py. - -#----------------------- -# other vars that are actually constant +# other vars that are actually constant Include/internal/pycore_blocks_output_buffer.h - BUFFER_BLOCK_SIZE - Modules/_csv.c - quote_styles - @@ -646,3 +353,327 @@ Python/stdlib_module_names.h - _Py_stdlib_module_names - Python/sysmodule.c - _PySys_ImplCacheTag - Python/sysmodule.c - _PySys_ImplName - Python/sysmodule.c - whatstrings - + +##----------------------- +## test code + +Modules/_ctypes/_ctypes_test.c - _ctypes_test_slots - +Modules/_ctypes/_ctypes_test.c - _ctypes_testmodule - +Modules/_ctypes/_ctypes_test.c - _xxx_lib - +Modules/_ctypes/_ctypes_test.c - an_integer - +Modules/_ctypes/_ctypes_test.c - bottom - +Modules/_ctypes/_ctypes_test.c - last_tf_arg_s - +Modules/_ctypes/_ctypes_test.c - last_tf_arg_u - +Modules/_ctypes/_ctypes_test.c - last_tfrsuv_arg - +Modules/_ctypes/_ctypes_test.c - left - +Modules/_ctypes/_ctypes_test.c - module_methods - +Modules/_ctypes/_ctypes_test.c - my_eggs - +Modules/_ctypes/_ctypes_test.c - my_spams - +Modules/_ctypes/_ctypes_test.c - right - +Modules/_ctypes/_ctypes_test.c - top - +Modules/_testbuffer.c - NDArray_Type - +Modules/_testbuffer.c - StaticArray_Type - +Modules/_testbuffer.c - Struct - +Modules/_testbuffer.c - _testbuffer_functions - +Modules/_testbuffer.c - _testbuffermodule - +Modules/_testbuffer.c - calcsize - +Modules/_testbuffer.c - infobuf - +Modules/_testbuffer.c - ndarray_as_buffer - +Modules/_testbuffer.c - ndarray_as_mapping - +Modules/_testbuffer.c - ndarray_as_sequence - +Modules/_testbuffer.c - ndarray_getset - +Modules/_testbuffer.c - ndarray_methods - +Modules/_testbuffer.c - simple_fmt - +Modules/_testbuffer.c - simple_format - +Modules/_testbuffer.c - static_buffer - +Modules/_testbuffer.c - static_mem - +Modules/_testbuffer.c - static_shape - +Modules/_testbuffer.c - static_strides - +Modules/_testbuffer.c - staticarray_as_buffer - +Modules/_testbuffer.c - structmodule - +Modules/_testbuffer.c ndarray_init kwlist - +Modules/_testbuffer.c ndarray_memoryview_from_buffer format - +Modules/_testbuffer.c ndarray_memoryview_from_buffer info - +Modules/_testbuffer.c ndarray_memoryview_from_buffer shape - +Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - +Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - +Modules/_testbuffer.c ndarray_push kwlist - +Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/heaptype.c - _testcapimodule - +Modules/_testcapi/unicode.c - _testcapimodule - +Modules/_testcapimodule.c - ContainerNoGC_members - +Modules/_testcapimodule.c - ContainerNoGC_type - +Modules/_testcapimodule.c - FmData - +Modules/_testcapimodule.c - FmHook - +Modules/_testcapimodule.c - GenericAlias_Type - +Modules/_testcapimodule.c - Generic_Type - +Modules/_testcapimodule.c - HeapCTypeSetattr_slots - +Modules/_testcapimodule.c - HeapCTypeSetattr_spec - +Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_slots - +Modules/_testcapimodule.c - HeapCTypeSubclassWithFinalizer_spec - +Modules/_testcapimodule.c - HeapCTypeSubclass_slots - +Modules/_testcapimodule.c - HeapCTypeSubclass_spec - +Modules/_testcapimodule.c - HeapCTypeWithBuffer_slots - +Modules/_testcapimodule.c - HeapCTypeWithBuffer_spec - +Modules/_testcapimodule.c - HeapCTypeWithDict_slots - +Modules/_testcapimodule.c - HeapCTypeWithDict_spec - +Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_slots - +Modules/_testcapimodule.c - HeapCTypeWithNegativeDict_spec - +Modules/_testcapimodule.c - HeapCTypeWithWeakref_slots - +Modules/_testcapimodule.c - HeapCTypeWithWeakref_spec - +Modules/_testcapimodule.c - HeapCType_slots - +Modules/_testcapimodule.c - HeapCType_spec - +Modules/_testcapimodule.c - HeapDocCType_slots - +Modules/_testcapimodule.c - HeapDocCType_spec - +Modules/_testcapimodule.c - HeapGcCType_slots - +Modules/_testcapimodule.c - HeapGcCType_spec - +Modules/_testcapimodule.c - MethClass_Type - +Modules/_testcapimodule.c - MethInstance_Type - +Modules/_testcapimodule.c - MethStatic_Type - +Modules/_testcapimodule.c - MethodDescriptor2_Type - +Modules/_testcapimodule.c - MethodDescriptorBase_Type - +Modules/_testcapimodule.c - MethodDescriptorDerived_Type - +Modules/_testcapimodule.c - MethodDescriptorNopGet_Type - +Modules/_testcapimodule.c - MyList_Type - +Modules/_testcapimodule.c - PyRecursingInfinitelyError_Type - +Modules/_testcapimodule.c - TestError - +Modules/_testcapimodule.c - TestMethods - +Modules/_testcapimodule.c - _HashInheritanceTester_Type - +Modules/_testcapimodule.c - _testcapimodule - +Modules/_testcapimodule.c - awaitType - +Modules/_testcapimodule.c - awaitType_as_async - +Modules/_testcapimodule.c - capsule_context - +Modules/_testcapimodule.c - capsule_destructor_call_count - +Modules/_testcapimodule.c - capsule_error - +Modules/_testcapimodule.c - capsule_name - +Modules/_testcapimodule.c - capsule_pointer - +Modules/_testcapimodule.c - decimal_initialized - +Modules/_testcapimodule.c - generic_alias_methods - +Modules/_testcapimodule.c - generic_methods - +Modules/_testcapimodule.c - heapctype_members - +Modules/_testcapimodule.c - heapctypesetattr_members - +Modules/_testcapimodule.c - heapctypesubclass_members - +Modules/_testcapimodule.c - heapctypewithdict_getsetlist - +Modules/_testcapimodule.c - heapctypewithdict_members - +Modules/_testcapimodule.c - heapctypewithnegativedict_members - +Modules/_testcapimodule.c - heapctypewithweakref_members - +Modules/_testcapimodule.c - ipowType - +Modules/_testcapimodule.c - ipowType_as_number - +Modules/_testcapimodule.c - matmulType - +Modules/_testcapimodule.c - matmulType_as_number - +Modules/_testcapimodule.c - meth_class_methods - +Modules/_testcapimodule.c - meth_instance_methods - +Modules/_testcapimodule.c - meth_static_methods - +Modules/_testcapimodule.c - ml - +Modules/_testcapimodule.c - str1 - +Modules/_testcapimodule.c - str2 - +Modules/_testcapimodule.c - test_members - +Modules/_testcapimodule.c - test_run_counter - +Modules/_testcapimodule.c - test_structmembersType - +Modules/_testcapimodule.c - thread_done - +Modules/_testcapimodule.c - x - +Modules/_testcapimodule.c getargs_keyword_only keywords - +Modules/_testcapimodule.c getargs_keywords keywords - +Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - +Modules/_testcapimodule.c getargs_s_hash_int2 keywords static char*[] +Modules/_testcapimodule.c make_exception_with_doc kwlist - +Modules/_testcapimodule.c raise_SIGINT_then_send_None PyId_send - +Modules/_testcapimodule.c slot_tp_del PyId___tp_del__ - +Modules/_testcapimodule.c test_capsule buffer - +Modules/_testcapimodule.c test_empty_argparse kwlist - +Modules/_testcapimodule.c test_structmembers_new keywords - +Modules/_testcapimodule.c getargs_s_hash_int keywords - +Modules/_testcapimodule.c - g_dict_watch_events - +Modules/_testcapimodule.c - g_dict_watchers_installed - +Modules/_testcapimodule.c - g_type_modified_events - +Modules/_testcapimodule.c - g_type_watchers_installed - +Modules/_testimportmultiple.c - _barmodule - +Modules/_testimportmultiple.c - _foomodule - +Modules/_testimportmultiple.c - _testimportmultiple - +Modules/_testinternalcapi.c - TestMethods - +Modules/_testinternalcapi.c - _testcapimodule - +Modules/_testmultiphase.c - Example_Type_slots - +Modules/_testmultiphase.c - Example_Type_spec - +Modules/_testmultiphase.c - Example_methods - +Modules/_testmultiphase.c - StateAccessType_Type_slots - +Modules/_testmultiphase.c - StateAccessType_methods - +Modules/_testmultiphase.c - StateAccessType_spec - +Modules/_testmultiphase.c - Str_Type_slots - +Modules/_testmultiphase.c - Str_Type_spec - +Modules/_testmultiphase.c - def_bad_large - +Modules/_testmultiphase.c - def_bad_negative - +Modules/_testmultiphase.c - def_create_int_with_state - +Modules/_testmultiphase.c - def_create_null - +Modules/_testmultiphase.c - def_create_raise - +Modules/_testmultiphase.c - def_create_unreported_exception - +Modules/_testmultiphase.c - def_exec_err - +Modules/_testmultiphase.c - def_exec_raise - +Modules/_testmultiphase.c - def_exec_unreported_exception - +Modules/_testmultiphase.c - def_meth_state_access - +Modules/_testmultiphase.c - def_negative_size - +Modules/_testmultiphase.c - def_nonascii_kana - +Modules/_testmultiphase.c - def_nonascii_latin - +Modules/_testmultiphase.c - def_nonmodule - +Modules/_testmultiphase.c - def_nonmodule_with_exec_slots - +Modules/_testmultiphase.c - def_nonmodule_with_methods - +Modules/_testmultiphase.c - imp_dummy_def - +Modules/_testmultiphase.c - main_def - +Modules/_testmultiphase.c - main_slots - +Modules/_testmultiphase.c - meth_state_access_slots - +Modules/_testmultiphase.c - nonmodule_methods - +Modules/_testmultiphase.c - null_slots_def - +Modules/_testmultiphase.c - slots_bad_large - +Modules/_testmultiphase.c - slots_bad_negative - +Modules/_testmultiphase.c - slots_create_nonmodule - +Modules/_testmultiphase.c - slots_create_nonmodule - +Modules/_testmultiphase.c - slots_create_null - +Modules/_testmultiphase.c - slots_create_raise - +Modules/_testmultiphase.c - slots_create_unreported_exception - +Modules/_testmultiphase.c - slots_exec_err - +Modules/_testmultiphase.c - slots_exec_raise - +Modules/_testmultiphase.c - slots_exec_unreported_exception - +Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - +Modules/_testmultiphase.c - testexport_methods - +Modules/_testmultiphase.c - uninitialized_def - +Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - +Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - +Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - +Modules/_xxtestfuzz/fuzzer.c - ast_literal_eval_method - +Modules/_xxtestfuzz/fuzzer.c - compiled_patterns - +Modules/_xxtestfuzz/fuzzer.c - csv_error - +Modules/_xxtestfuzz/fuzzer.c - csv_module - +Modules/_xxtestfuzz/fuzzer.c - json_loads_method - +Modules/_xxtestfuzz/fuzzer.c - regex_patterns - +Modules/_xxtestfuzz/fuzzer.c - sre_compile_method - +Modules/_xxtestfuzz/fuzzer.c - sre_error_exception - +Modules/_xxtestfuzz/fuzzer.c - struct_error - +Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_MATCH_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput STRUCT_UNPACK_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED - + +##----------------------- +## the analyzer should have ignored these +# XXX Fix the analyzer. + +## forward/extern references +Include/py_curses.h - PyCurses_API - +Include/pydecimal.h - _decimal_api - +Modules/_blake2/blake2module.c - blake2b_type_spec - +Modules/_blake2/blake2module.c - blake2s_type_spec - +Modules/_io/fileio.c - _Py_open_cloexec_works - +Modules/_io/_iomodule.h - PyIOBase_Type - +Modules/_io/_iomodule.h - PyRawIOBase_Type - +Modules/_io/_iomodule.h - PyBufferedIOBase_Type - +Modules/_io/_iomodule.h - PyTextIOBase_Type - +Modules/_io/_iomodule.h - PyFileIO_Type - +Modules/_io/_iomodule.h - PyBytesIO_Type - +Modules/_io/_iomodule.h - PyStringIO_Type - +Modules/_io/_iomodule.h - PyBufferedReader_Type - +Modules/_io/_iomodule.h - PyBufferedWriter_Type - +Modules/_io/_iomodule.h - PyBufferedRWPair_Type - +Modules/_io/_iomodule.h - PyBufferedRandom_Type - +Modules/_io/_iomodule.h - PyTextIOWrapper_Type - +Modules/_io/_iomodule.h - PyIncrementalNewlineDecoder_Type - +Modules/_io/_iomodule.h - _PyBytesIOBuffer_Type - +Modules/_io/_iomodule.h - _PyIO_Module - +Modules/_io/_iomodule.h - _PyIO_str_close - +Modules/_io/_iomodule.h - _PyIO_str_closed - +Modules/_io/_iomodule.h - _PyIO_str_decode - +Modules/_io/_iomodule.h - _PyIO_str_encode - +Modules/_io/_iomodule.h - _PyIO_str_fileno - +Modules/_io/_iomodule.h - _PyIO_str_flush - +Modules/_io/_iomodule.h - _PyIO_str_getstate - +Modules/_io/_iomodule.h - _PyIO_str_isatty - +Modules/_io/_iomodule.h - _PyIO_str_newlines - +Modules/_io/_iomodule.h - _PyIO_str_nl - +Modules/_io/_iomodule.h - _PyIO_str_peek - +Modules/_io/_iomodule.h - _PyIO_str_read - +Modules/_io/_iomodule.h - _PyIO_str_read1 - +Modules/_io/_iomodule.h - _PyIO_str_readable - +Modules/_io/_iomodule.h - _PyIO_str_readall - +Modules/_io/_iomodule.h - _PyIO_str_readinto - +Modules/_io/_iomodule.h - _PyIO_str_readline - +Modules/_io/_iomodule.h - _PyIO_str_reset - +Modules/_io/_iomodule.h - _PyIO_str_seek - +Modules/_io/_iomodule.h - _PyIO_str_seekable - +Modules/_io/_iomodule.h - _PyIO_str_setstate - +Modules/_io/_iomodule.h - _PyIO_str_tell - +Modules/_io/_iomodule.h - _PyIO_str_truncate - +Modules/_io/_iomodule.h - _PyIO_str_writable - +Modules/_io/_iomodule.h - _PyIO_str_write - +Modules/_io/_iomodule.h - _PyIO_empty_str - +Modules/_io/_iomodule.h - _PyIO_empty_bytes - +Modules/_multiprocessing/multiprocessing.h - _PyMp_SemLockType - +Modules/_sqlite/module.c - _pysqlite_converters - +Modules/_sqlite/module.c - _pysqlite_enable_callback_tracebacks - +Modules/_sqlite/module.c - pysqlite_BaseTypeAdapted - +Modules/_sqlite/module.h - pysqlite_global_state - +Modules/_testcapimodule.c - _PyBytesIOBuffer_Type - +Modules/posixmodule.c - _Py_open_cloexec_works - +Modules/posixmodule.c - environ - +Objects/object.c - _Py_GenericAliasIterType - +Objects/object.c - _PyMemoryIter_Type - +Objects/object.c - _PyLineIterator - +Objects/object.c - _PyPositionsIterator - +Python/perf_trampoline.c - _Py_trampoline_func_start - +Python/perf_trampoline.c - _Py_trampoline_func_end - +Python/importdl.h - _PyImport_DynLoadFiletab - +Modules/expat/xmlrole.c - prolog0 - +Modules/expat/xmlrole.c - prolog1 - +Modules/expat/xmlrole.c - prolog2 - +Modules/expat/xmlrole.c - doctype0 - +Modules/expat/xmlrole.c - doctype1 - +Modules/expat/xmlrole.c - doctype2 - +Modules/expat/xmlrole.c - doctype3 - +Modules/expat/xmlrole.c - doctype4 - +Modules/expat/xmlrole.c - doctype5 - +Modules/expat/xmlrole.c - internalSubset - +Modules/expat/xmlrole.c - entity0 - +Modules/expat/xmlrole.c - entity1 - +Modules/expat/xmlrole.c - entity2 - +Modules/expat/xmlrole.c - entity3 - +Modules/expat/xmlrole.c - entity4 - +Modules/expat/xmlrole.c - entity5 - +Modules/expat/xmlrole.c - entity6 - +Modules/expat/xmlrole.c - entity7 - +Modules/expat/xmlrole.c - entity8 - +Modules/expat/xmlrole.c - entity9 - +Modules/expat/xmlrole.c - entity10 - +Modules/expat/xmlrole.c - notation0 - +Modules/expat/xmlrole.c - notation1 - +Modules/expat/xmlrole.c - notation2 - +Modules/expat/xmlrole.c - notation3 - +Modules/expat/xmlrole.c - notation4 - +Modules/expat/xmlrole.c - attlist0 - +Modules/expat/xmlrole.c - attlist1 - +Modules/expat/xmlrole.c - attlist2 - +Modules/expat/xmlrole.c - attlist3 - +Modules/expat/xmlrole.c - attlist4 - +Modules/expat/xmlrole.c - attlist5 - +Modules/expat/xmlrole.c - attlist6 - +Modules/expat/xmlrole.c - attlist7 - +Modules/expat/xmlrole.c - attlist8 - +Modules/expat/xmlrole.c - attlist9 - +Modules/expat/xmlrole.c - element0 - +Modules/expat/xmlrole.c - element1 - +Modules/expat/xmlrole.c - element2 - +Modules/expat/xmlrole.c - element3 - +Modules/expat/xmlrole.c - element4 - +Modules/expat/xmlrole.c - element5 - +Modules/expat/xmlrole.c - element6 - +Modules/expat/xmlrole.c - element7 - +Modules/expat/xmlrole.c - externalSubset0 - +Modules/expat/xmlrole.c - externalSubset1 - +Modules/expat/xmlrole.c - condSect0 - +Modules/expat/xmlrole.c - condSect1 - +Modules/expat/xmlrole.c - condSect2 - +Modules/expat/xmlrole.c - declClose - +Modules/expat/xmlrole.c - error - + +## other +Modules/_io/_iomodule.c - _PyIO_Module - +Modules/_sqlite/module.c - _sqlite3module - diff --git a/Tools/c-analyzer/table-file.py b/Tools/c-analyzer/table-file.py index 3cc05cc9de77..d36f814415c8 100644 --- a/Tools/c-analyzer/table-file.py +++ b/Tools/c-analyzer/table-file.py @@ -1,43 +1,59 @@ +KINDS = [ + 'section-major', + 'section-minor', + 'section-group', + 'row', +] + + def iter_clean_lines(lines): lines = iter(lines) - for line in lines: - line = line.strip() - if line.startswith('# XXX'): + for rawline in lines: + line = rawline.strip() + if line.startswith('#') and not rawline.startswith('##'): continue - yield line + yield line, rawline def parse_table_lines(lines): lines = iter_clean_lines(lines) - for line in lines: - if line.startswith(('####', '#----')): - kind = 0 if line[1] == '#' else 1 - try: - line = next(lines).strip() - except StopIteration: - line = '' - if not line.startswith('# '): - raise NotImplementedError(line) - yield kind, line[2:].lstrip() - continue - - maybe = None - while line.startswith('#'): - if line != '#' and line[1] == ' ': - maybe = line[2:].lstrip() - try: - line = next(lines).strip() - except StopIteration: - return - if not line: - break - else: - if line: - if maybe: - yield 2, maybe - yield 'row', line + group = None + prev = '' + for line, rawline in lines: + if line.startswith('## '): + assert not rawline.startswith(' '), (line, rawline) + if group: + assert prev, (line, rawline) + kind, after, _ = group + assert kind and kind != 'section-group', (group, line, rawline) + assert after is not None, (group, line, rawline) + else: + assert not prev, (prev, line, rawline) + kind, after = group = ('section-group', None) + title = line[3:].lstrip() + assert title, (line, rawline) + if after is not None: + try: + line, rawline = next(lines) + except StopIteration: + line = None + if line != after: + raise NotImplementedError((group, line, rawline)) + yield kind, title + group = None + elif group: + raise NotImplementedError((group, line, rawline)) + elif line.startswith('##---'): + assert line.rstrip('-') == '##', (line, rawline) + group = ('section-minor', '', line) + elif line.startswith('#####'): + assert not line.strip('#'), (line, rawline) + group = ('section-major', '', line) + elif line: + yield 'row', line + prev = line def iter_sections(lines): @@ -49,12 +65,13 @@ def iter_sections(lines): if header is None: header = value continue - raise NotImplementedError(value) + raise NotImplementedError(repr(value)) yield tuple(section), value else: if header is None: header = False - section[kind:] = [value] + start = KINDS.index(kind) + section[start:] = [value] def collect_sections(lines): From webhook-mailer at python.org Wed Dec 7 17:57:03 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Wed, 07 Dec 2022 22:57:03 -0000 Subject: [Python-checkins] gh-81057: Move More Globals to _PyRuntimeState (gh-100092) Message-ID: <mailman.2848.1670453824.3313.python-checkins@python.org> https://github.com/python/cpython/commit/91a8e002c21a5388c5152c5a4871b9a2d85f0fc1 commit: 91a8e002c21a5388c5152c5a4871b9a2d85f0fc1 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-07T15:56:31-07:00 summary: gh-81057: Move More Globals to _PyRuntimeState (gh-100092) https://github.com/python/cpython/issues/81057 files: M Include/internal/pycore_dtoa.h M Include/internal/pycore_obmalloc.h M Include/internal/pycore_obmalloc_init.h M Include/internal/pycore_parser.h M Include/internal/pycore_runtime.h M Include/internal/pycore_unicodeobject.h M Modules/_io/bufferedio.c M Objects/obmalloc.c M Objects/unicodeobject.c M Parser/pegen.c M Python/dtoa.c M Python/initconfig.c M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index fdc6e74ecd25..67189cf0ade6 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -25,7 +25,7 @@ Bigint { #ifdef Py_USING_MEMORY_DEBUGGER struct _dtoa_runtime_state { - int _not_used; + int _not_used; }; #define _dtoa_runtime_state_INIT {0} @@ -41,9 +41,12 @@ struct _dtoa_runtime_state { ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) struct _dtoa_runtime_state { - struct Bigint *freelist[Bigint_Kmax+1]; - double preallocated[Bigint_PREALLOC_SIZE]; - double *preallocated_next; + /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + // XXX This should be freed during runtime fini. + struct Bigint *p5s; + struct Bigint *freelist[Bigint_Kmax+1]; + double preallocated[Bigint_PREALLOC_SIZE]; + double *preallocated_next; }; #define _dtoa_runtime_state_INIT(runtime) \ { \ diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 93349d89c6ab..a5c7f4528f91 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -658,6 +658,7 @@ struct _obmalloc_usage { struct _obmalloc_state { + int dump_debug_stats; struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; struct _obmalloc_usage usage; diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index c0fb057d0665..c9f197e72de9 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -56,6 +56,7 @@ extern "C" { #define _obmalloc_state_INIT(obmalloc) \ { \ + .dump_debug_stats = -1, \ .pools = { \ .used = _obmalloc_pools_INIT(obmalloc.pools), \ }, \ diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index e2de24e2ca97..2d2b56bd824c 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -8,12 +8,31 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif + +#include "pycore_pyarena.h" // PyArena + + +#ifdef Py_DEBUG +#define _PYPEGEN_NSTATISTICS 2000 +#endif + +struct _parser_runtime_state { +#ifdef Py_DEBUG + long memo_statistics[_PYPEGEN_NSTATISTICS]; +#else + int _not_used; +#endif +}; + + + extern struct _mod* _PyParser_ASTFromString( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena); + extern struct _mod* _PyParser_ASTFromFile( FILE *fp, PyObject *filename_ob, @@ -25,6 +44,7 @@ extern struct _mod* _PyParser_ASTFromFile( int *errcode, PyArena *arena); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c1829cb1bdad..0720e2ed4422 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -17,6 +17,7 @@ extern "C" { #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState +#include "pycore_parser.h" // struct _parser_runtime_state #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state @@ -129,6 +130,10 @@ typedef struct pyruntimestate { unsigned long main_thread; + PyWideStringList orig_argv; + + struct _parser_runtime_state parser; + #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index b315ca1ae5b6..19faceebf1d8 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_fileutils.h" // _Py_error_handler +#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI void _PyUnicode_ExactDealloc(PyObject *op); @@ -52,6 +53,8 @@ struct _Py_unicode_ids { struct _Py_unicode_state { struct _Py_unicode_fs_codec fs_codec; + _PyUnicode_Name_CAPI *ucnhash_capi; + // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId() struct _Py_unicode_ids ids; }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 6df55b5b8303..ba8969f0bcd1 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -746,26 +746,26 @@ _buffered_init(buffered *self) int _PyIO_trap_eintr(void) { - static PyObject *eintr_int = NULL; PyObject *typ, *val, *tb; PyOSErrorObject *env_err; - - if (eintr_int == NULL) { - eintr_int = PyLong_FromLong(EINTR); - assert(eintr_int != NULL); - } if (!PyErr_ExceptionMatches(PyExc_OSError)) return 0; PyErr_Fetch(&typ, &val, &tb); PyErr_NormalizeException(&typ, &val, &tb); env_err = (PyOSErrorObject *) val; assert(env_err != NULL); - if (env_err->myerrno != NULL && - PyObject_RichCompareBool(env_err->myerrno, eintr_int, Py_EQ) > 0) { - Py_DECREF(typ); - Py_DECREF(val); - Py_XDECREF(tb); - return 1; + if (env_err->myerrno != NULL) { + assert(EINTR > 0 && EINTR < INT_MAX); + assert(PyLong_CheckExact(env_err->myerrno)); + int overflow; + int myerrno = PyLong_AsLongAndOverflow(env_err->myerrno, &overflow); + PyErr_Clear(); + if (myerrno == EINTR) { + Py_DECREF(typ); + Py_DECREF(val); + Py_XDECREF(tb); + return 1; + } } /* This silences any error set by PyObject_RichCompareBool() */ PyErr_Restore(typ, val, tb); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 4c08bc214cd2..276c5a276c06 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -908,11 +908,12 @@ new_arena(void) struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ void *address; - static int debug_stats = -1; + int debug_stats = _PyRuntime.obmalloc.dump_debug_stats; if (debug_stats == -1) { const char *opt = Py_GETENV("PYTHONMALLOCSTATS"); debug_stats = (opt != NULL && *opt != '\0'); + _PyRuntime.obmalloc.dump_debug_stats = debug_stats; } if (debug_stats) { _PyObject_DebugMallocStats(stderr); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index deeca35714b7..b721ccd805ed 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5697,8 +5697,6 @@ PyUnicode_AsUTF16String(PyObject *unicode) /* --- Unicode Escape Codec ----------------------------------------------- */ -static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; - PyObject * _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, Py_ssize_t size, @@ -5711,6 +5709,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, const char *end; PyObject *errorHandler = NULL; PyObject *exc = NULL; + _PyUnicode_Name_CAPI *ucnhash_capi; + PyInterpreterState *interp = _PyInterpreterState_Get(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; @@ -5858,6 +5858,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, /* \N{name} */ case 'N': + ucnhash_capi = interp->unicode.ucnhash_capi; if (ucnhash_capi == NULL) { /* load the unicode data module */ ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( @@ -5869,6 +5870,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, ); goto onError; } + interp->unicode.ucnhash_capi = ucnhash_capi; } message = "malformed \\N character escape"; @@ -15128,10 +15130,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) assert(get_interned_dict() == NULL); // bpo-47182: force a unicodedata CAPI capsule re-import on // subsequent initialization of main interpreter. - ucnhash_capi = NULL; } _PyUnicode_FiniEncodings(&state->fs_codec); + interp->unicode.ucnhash_capi = NULL; unicode_clear_identifiers(state); } diff --git a/Parser/pegen.c b/Parser/pegen.c index d34a86e9c883..d84e06861ede 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -246,8 +246,8 @@ _PyPegen_fill_token(Parser *p) // The array counts the number of tokens skipped by memoization, // indexed by type. -#define NSTATISTICS 2000 -static long memo_statistics[NSTATISTICS]; +#define NSTATISTICS _PYPEGEN_NSTATISTICS +#define memo_statistics _PyRuntime.parser.memo_statistics void _PyPegen_clear_memo_statistics() diff --git a/Python/dtoa.c b/Python/dtoa.c index 1b47d83bf77a..cff5f1b0658e 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -673,10 +673,6 @@ mult(Bigint *a, Bigint *b) #ifndef Py_USING_MEMORY_DEBUGGER -/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ - -static Bigint *p5s; - /* multiply the Bigint b by 5**k. Returns a pointer to the result, or NULL on failure; if the returned pointer is distinct from b then the original Bigint b will have been Bfree'd. Ignores the sign of b. */ @@ -696,7 +692,7 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; - p5 = p5s; + p5 = _PyRuntime.dtoa.p5s; if (!p5) { /* first time */ p5 = i2b(625); @@ -704,7 +700,7 @@ pow5mult(Bigint *b, int k) Bfree(b); return NULL; } - p5s = p5; + _PyRuntime.dtoa.p5s = p5; p5->next = 0; } for(;;) { diff --git a/Python/initconfig.c b/Python/initconfig.c index 67f6777d3b1d..64ae987b3f34 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -595,17 +595,13 @@ _Py_ClearStandardStreamEncoding(void) /* --- Py_GetArgcArgv() ------------------------------------------- */ -/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */ -static PyWideStringList orig_argv = {.length = 0, .items = NULL}; - - void _Py_ClearArgcArgv(void) { PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyWideStringList_Clear(&orig_argv); + _PyWideStringList_Clear(&_PyRuntime.orig_argv); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -620,7 +616,9 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - res = _PyWideStringList_Copy(&orig_argv, &argv_list); + // XXX _PyRuntime.orig_argv only gets cleared by Py_Main(), + // so it it currently leaks for embedders. + res = _PyWideStringList_Copy(&_PyRuntime.orig_argv, &argv_list); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return res; @@ -631,8 +629,8 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) void Py_GetArgcArgv(int *argc, wchar_t ***argv) { - *argc = (int)orig_argv.length; - *argv = orig_argv.items; + *argc = (int)_PyRuntime.orig_argv.length; + *argv = _PyRuntime.orig_argv.items; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index af5b7eb8eaf0..5c8164517f13 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -305,20 +305,12 @@ Objects/sliceobject.c - _Py_EllipsisObject - ##----------------------- ## effectively-const but initialized lazily -## idempotent -Python/dtoa.c - p5s - -Objects/obmalloc.c new_arena debug_stats - - ## others Python/perf_trampoline.c - perf_map_file - -Objects/unicodeobject.c - ucnhash_capi - ##----------------------- ## state -## local buffer -Python/suggestions.c levenshtein_distance buffer - - ## other Objects/object.c - _Py_RefTotal - Python/perf_trampoline.c - perf_status - @@ -398,7 +390,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/_io/bufferedio.c _PyIO_trap_eintr eintr_int - Modules/posixmodule.c os_dup2_impl dup3_works - Modules/posixmodule.c - structseq_new - Modules/posixmodule.c - ticks_per_second - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 128336a997eb..257823574fa8 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -87,8 +87,6 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - -# XXX This only gets cleared by Py_Main(). -Python/initconfig.c - orig_argv - ##----------------------- ## public C-API @@ -143,15 +141,6 @@ Python/pylifecycle.c - runtime_initialized - Modules/syslogmodule.c - S_ident_o - Modules/syslogmodule.c - S_log_open - -##----------------------- -## *not* tied to init/fini cycle - -# These do not ge reset with each init/fini cycle. -# XXX These should probably be tied to init/fini. Move to _PyRuntimeState? - -# special-use diagnistic state -Parser/pegen.c - memo_statistics - - ##----------------------- ## one-off temporary state From webhook-mailer at python.org Wed Dec 7 19:38:57 2022 From: webhook-mailer at python.org (vstinner) Date: Thu, 08 Dec 2022 00:38:57 -0000 Subject: [Python-checkins] gh-100086: Add build info to test.libregrtest (#100093) Message-ID: <mailman.2849.1670459938.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3c892022472eb975360fb3f0caa6f6fcc6fbf220 commit: 3c892022472eb975360fb3f0caa6f6fcc6fbf220 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-08T01:38:47+01:00 summary: gh-100086: Add build info to test.libregrtest (#100093) The Python test runner (libregrtest) now logs Python build information like "debug" vs "release" build, or LTO and PGO optimizations. files: A Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst M Lib/test/libregrtest/main.py M Lib/test/libregrtest/utils.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3eeef029b22d..19ccf2db5e7f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -17,7 +17,8 @@ ChildError, DidNotRun) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests -from test.libregrtest.utils import removepy, count, format_duration, printlist +from test.libregrtest.utils import (removepy, count, format_duration, + printlist, get_build_info) from test import support from test.support import os_helper from test.support import threading_helper @@ -491,6 +492,7 @@ def display_header(self): print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), "%s-endian" % sys.byteorder) + print("== Python build:", ' '.join(get_build_info())) print("== cwd:", os.getcwd()) cpu_count = os.cpu_count() if cpu_count: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index e6909170334e..fb13fa0e243b 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,6 +1,7 @@ import math import os.path import sys +import sysconfig import textwrap from test import support @@ -208,3 +209,87 @@ def clear_caches(): pass else: fractions._hash_algorithm.cache_clear() + + +def get_build_info(): + # Get most important configure and build options as a list of strings. + # Example: ['debug', 'ASAN+MSAN'] or ['release', 'LTO+PGO']. + + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + cflags = sysconfig.get_config_var('PY_CFLAGS') or '' + cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' + + build = [] + if hasattr(sys, 'gettotalrefcount'): + # --with-pydebug + build.append('debug') + + if '-DNDEBUG' in (cflags + cflags_nodist): + build.append('without_assert') + else: + build.append('release') + + if '--with-assertions' in config_args: + build.append('with_assert') + elif '-DNDEBUG' not in (cflags + cflags_nodist): + build.append('with_assert') + + # --enable-framework=name + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + if framework: + build.append(f'framework={framework}') + + # --enable-shared + shared = int(sysconfig.get_config_var('PY_ENABLE_SHARED') or '0') + if shared: + build.append('shared') + + # --with-lto + optimizations = [] + if '-flto=thin' in ldflags_nodist: + optimizations.append('ThinLTO') + elif '-flto' in ldflags_nodist: + optimizations.append('LTO') + + # --enable-optimizations + pgo_options = ( + # GCC + '-fprofile-use', + # clang: -fprofile-instr-use=code.profclangd + '-fprofile-instr-use', + # ICC + "-prof-use", + ) + if any(option in cflags_nodist for option in pgo_options): + optimizations.append('PGO') + if optimizations: + build.append('+'.join(optimizations)) + + # --with-address-sanitizer + sanitizers = [] + if support.check_sanitizer(address=True): + sanitizers.append("ASAN") + # --with-memory-sanitizer + if support.check_sanitizer(memory=True): + sanitizers.append("MSAN") + # --with-undefined-behavior-sanitizer + if support.check_sanitizer(ub=True): + sanitizers.append("UBSAN") + if sanitizers: + build.append('+'.join(sanitizers)) + + # --with-trace-refs + if hasattr(sys, 'getobjects'): + build.append("TraceRefs") + # --enable-pystats + if hasattr(sys, '_stats_on'): + build.append("pystats") + # --with-valgrind + if sysconfig.get_config_var('WITH_VALGRIND'): + build.append("valgrind") + # --with-dtrace + if sysconfig.get_config_var('WITH_DTRACE'): + build.append("dtrace") + + return build diff --git a/Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst b/Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst new file mode 100644 index 000000000000..a5f1bb9f5a5e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-08-00-03-37.gh-issue-100086.1zYpto.rst @@ -0,0 +1,3 @@ +The Python test runner (libregrtest) now logs Python build information like +"debug" vs "release" build, or LTO and PGO optimizations. Patch by Victor +Stinner. From webhook-mailer at python.org Wed Dec 7 21:21:03 2022 From: webhook-mailer at python.org (corona10) Date: Thu, 08 Dec 2022 02:21:03 -0000 Subject: [Python-checkins] gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) Message-ID: <mailman.2850.1670466064.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dc8a86893df37e137cfe992e95e7d66cd68e9eaf commit: dc8a86893df37e137cfe992e95e7d66cd68e9eaf branch: main author: Dong-hee Na <donghee.na at python.org> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-08T11:20:34+09:00 summary: gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) files: A Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst M Lib/test/test_urllib2.py M Lib/urllib/error.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 28f88412fdca..498c0382d213 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1824,6 +1824,10 @@ def test_HTTPError_interface(self): expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + def test_parse_proxy(self): parse_proxy_test_cases = [ ('proxy.example.com', diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8e..feec0e7f848e 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.StringIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst new file mode 100644 index 000000000000..b1c170dff3ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst @@ -0,0 +1,2 @@ +Update :exc:`~urllib.error.HTTPError` to be initialized properly, even if +the ``fp`` is ``None``. Patch by Dong-hee Na. From webhook-mailer at python.org Wed Dec 7 21:24:58 2022 From: webhook-mailer at python.org (ethanfurman) Date: Thu, 08 Dec 2022 02:24:58 -0000 Subject: [Python-checkins] gh-92120: The docstring of enum.Enum is invalid in reST (GH-92122) Message-ID: <mailman.2851.1670466298.3313.python-checkins@python.org> https://github.com/python/cpython/commit/90d5c9b195a8133688c2d7b6ad9e0ca8b76df1df commit: 90d5c9b195a8133688c2d7b6ad9e0ca8b76df1df branch: main author: Takeshi KOMIYA <i.tkomiya at gmail.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-07T18:24:52-08:00 summary: gh-92120: The docstring of enum.Enum is invalid in reST (GH-92122) Closes #92120 files: A Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst M Lib/enum.py diff --git a/Lib/enum.py b/Lib/enum.py index f07b821f1a60..b2f7b9f8d4ad 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1054,20 +1054,20 @@ class Enum(metaclass=EnumType): Access them by: - - attribute access:: + - attribute access: - >>> Color.RED - <Color.RED: 1> + >>> Color.RED + <Color.RED: 1> - value lookup: - >>> Color(1) - <Color.RED: 1> + >>> Color(1) + <Color.RED: 1> - name lookup: - >>> Color['RED'] - <Color.RED: 1> + >>> Color['RED'] + <Color.RED: 1> Enumerations can be iterated over, and know how many members they have: diff --git a/Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst b/Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst new file mode 100644 index 000000000000..d585535ee38d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-06-01-53-34.gh-issue-92122.96Lf2p.rst @@ -0,0 +1 @@ +Fix reStructuredText syntax errors in docstrings in the :mod:`enum` module. From webhook-mailer at python.org Wed Dec 7 23:18:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 04:18:02 -0000 Subject: [Python-checkins] gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) Message-ID: <mailman.2852.1670473083.3313.python-checkins@python.org> https://github.com/python/cpython/commit/846898e5ab0fb1bf740263573f01669be186645f commit: 846898e5ab0fb1bf740263573f01669be186645f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T20:17:23-08:00 summary: gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) (cherry picked from commit dc8a86893df37e137cfe992e95e7d66cd68e9eaf) Co-authored-by: Dong-hee Na <donghee.na at python.org> files: A Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst M Lib/test/test_urllib2.py M Lib/urllib/error.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 46a0ab1d83e6..db92ba56c34f 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1825,6 +1825,10 @@ def test_HTTPError_interface(self): expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + def test_parse_proxy(self): parse_proxy_test_cases = [ ('proxy.example.com', diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8e..feec0e7f848e 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.StringIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst new file mode 100644 index 000000000000..b1c170dff3ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst @@ -0,0 +1,2 @@ +Update :exc:`~urllib.error.HTTPError` to be initialized properly, even if +the ``fp`` is ``None``. Patch by Dong-hee Na. From webhook-mailer at python.org Wed Dec 7 23:18:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 04:18:02 -0000 Subject: [Python-checkins] gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) Message-ID: <mailman.2853.1670473084.3313.python-checkins@python.org> https://github.com/python/cpython/commit/85be25827bb94112037c391eca66a0d796468825 commit: 85be25827bb94112037c391eca66a0d796468825 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T20:17:35-08:00 summary: gh-98778: Update HTTPError to initialize properly even if fp is None (gh-99966) (cherry picked from commit dc8a86893df37e137cfe992e95e7d66cd68e9eaf) Co-authored-by: Dong-hee Na <donghee.na at python.org> files: A Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst M Lib/test/test_urllib2.py M Lib/urllib/error.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 399c94213a63..4ef391c4540c 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1823,6 +1823,10 @@ def test_HTTPError_interface(self): expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + def test_parse_proxy(self): parse_proxy_test_cases = [ ('proxy.example.com', diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8e..feec0e7f848e 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.StringIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst new file mode 100644 index 000000000000..b1c170dff3ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-03-20-06-16.gh-issue-98778.t5U9uc.rst @@ -0,0 +1,2 @@ +Update :exc:`~urllib.error.HTTPError` to be initialized properly, even if +the ``fp`` is ``None``. Patch by Dong-hee Na. From webhook-mailer at python.org Wed Dec 7 23:59:43 2022 From: webhook-mailer at python.org (gpshead) Date: Thu, 08 Dec 2022 04:59:43 -0000 Subject: [Python-checkins] gh-98030: socket: add missing TCP socket options (#98031) Message-ID: <mailman.2854.1670475583.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cce836296016463032495c6ca739ab469ed13d3c commit: cce836296016463032495c6ca739ab469ed13d3c branch: main author: Matthieu Baerts <matttbe at gmail.com> committer: gpshead <greg at krypto.org> date: 2022-12-07T20:59:37-08:00 summary: gh-98030: socket: add missing TCP socket options (#98031) A few TCP socket options have been added to the Linux kernel these last few years. This commit adds all the ones available in Linux 6.0: https://elixir.bootlin.com/linux/v6.0/source/include/uapi/linux/tcp.h#L91 While at it, the TCP_FASTOPEN option has been moved lower in the list just to keep the same order as in tcp.h to ease future synchronisations. Signed-off-by: Matthieu Baerts <matthieu.baerts at tessares.net> files: A Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst M Doc/library/socket.rst M Modules/socketmodule.c diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c946407ea1d8..de2e1aa3868b 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -428,7 +428,14 @@ Constants .. versionchanged:: 3.12 Added ``SO_RTABLE`` and ``SO_USER_COOKIE``. On OpenBSD and FreeBSD respectively those constants can be used in the same way that - ``SO_MARK`` is used on Linux. + ``SO_MARK`` is used on Linux. Also added missing TCP socket options from + Linux: ``TCP_MD5SIG``, ``TCP_THIN_LINEAR_TIMEOUTS``, ``TCP_THIN_DUPACK``, + ``TCP_REPAIR``, ``TCP_REPAIR_QUEUE``, ``TCP_QUEUE_SEQ``, + ``TCP_REPAIR_OPTIONS``, ``TCP_TIMESTAMP``, ``TCP_CC_INFO``, + ``TCP_SAVE_SYN``, ``TCP_SAVED_SYN``, ``TCP_REPAIR_WINDOW``, + ``TCP_FASTOPEN_CONNECT``, ``TCP_ULP``, ``TCP_MD5SIG_EXT``, + ``TCP_FASTOPEN_KEY``, ``TCP_FASTOPEN_NO_COOKIE``, + ``TCP_ZEROCOPY_RECEIVE``, ``TCP_INQ``, ``TCP_TX_DELAY``. .. data:: AF_CAN PF_CAN diff --git a/Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst b/Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst new file mode 100644 index 000000000000..7768ed0817e8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-07-18-16-00.gh-issue-98030.2oQCZy.rst @@ -0,0 +1,7 @@ +Add missing TCP socket options from Linux: ``TCP_MD5SIG``, +``TCP_THIN_LINEAR_TIMEOUTS``, ``TCP_THIN_DUPACK``, ``TCP_REPAIR``, +``TCP_REPAIR_QUEUE``, ``TCP_QUEUE_SEQ``, ``TCP_REPAIR_OPTIONS``, +``TCP_TIMESTAMP``, ``TCP_CC_INFO``, ``TCP_SAVE_SYN``, ``TCP_SAVED_SYN``, +``TCP_REPAIR_WINDOW``, ``TCP_FASTOPEN_CONNECT``, ``TCP_ULP``, +``TCP_MD5SIG_EXT``, ``TCP_FASTOPEN_KEY``, ``TCP_FASTOPEN_NO_COOKIE``, +``TCP_ZEROCOPY_RECEIVE``, ``TCP_INQ``, ``TCP_TX_DELAY``. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7d438448ef36..2c59c2f2c89b 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8484,18 +8484,78 @@ PyInit__socket(void) #ifdef TCP_QUICKACK PyModule_AddIntMacro(m, TCP_QUICKACK); #endif -#ifdef TCP_FASTOPEN - PyModule_AddIntMacro(m, TCP_FASTOPEN); -#endif #ifdef TCP_CONGESTION PyModule_AddIntMacro(m, TCP_CONGESTION); #endif +#ifdef TCP_MD5SIG + PyModule_AddIntMacro(m, TCP_MD5SIG); +#endif +#ifdef TCP_THIN_LINEAR_TIMEOUTS + PyModule_AddIntMacro(m, TCP_THIN_LINEAR_TIMEOUTS); +#endif +#ifdef TCP_THIN_DUPACK + PyModule_AddIntMacro(m, TCP_THIN_DUPACK); +#endif #ifdef TCP_USER_TIMEOUT PyModule_AddIntMacro(m, TCP_USER_TIMEOUT); #endif +#ifdef TCP_REPAIR + PyModule_AddIntMacro(m, TCP_REPAIR); +#endif +#ifdef TCP_REPAIR_QUEUE + PyModule_AddIntMacro(m, TCP_REPAIR_QUEUE); +#endif +#ifdef TCP_QUEUE_SEQ + PyModule_AddIntMacro(m, TCP_QUEUE_SEQ); +#endif +#ifdef TCP_REPAIR_OPTIONS + PyModule_AddIntMacro(m, TCP_REPAIR_OPTIONS); +#endif +#ifdef TCP_FASTOPEN + PyModule_AddIntMacro(m, TCP_FASTOPEN); +#endif +#ifdef TCP_TIMESTAMP + PyModule_AddIntMacro(m, TCP_TIMESTAMP); +#endif #ifdef TCP_NOTSENT_LOWAT PyModule_AddIntMacro(m, TCP_NOTSENT_LOWAT); #endif +#ifdef TCP_CC_INFO + PyModule_AddIntMacro(m, TCP_CC_INFO); +#endif +#ifdef TCP_SAVE_SYN + PyModule_AddIntMacro(m, TCP_SAVE_SYN); +#endif +#ifdef TCP_SAVED_SYN + PyModule_AddIntMacro(m, TCP_SAVED_SYN); +#endif +#ifdef TCP_REPAIR_WINDOW + PyModule_AddIntMacro(m, TCP_REPAIR_WINDOW); +#endif +#ifdef TCP_FASTOPEN_CONNECT + PyModule_AddIntMacro(m, TCP_FASTOPEN_CONNECT); +#endif +#ifdef TCP_ULP + PyModule_AddIntMacro(m, TCP_ULP); +#endif +#ifdef TCP_MD5SIG_EXT + PyModule_AddIntMacro(m, TCP_MD5SIG_EXT); +#endif +#ifdef TCP_FASTOPEN_KEY + PyModule_AddIntMacro(m, TCP_FASTOPEN_KEY); +#endif +#ifdef TCP_FASTOPEN_NO_COOKIE + PyModule_AddIntMacro(m, TCP_FASTOPEN_NO_COOKIE); +#endif +#ifdef TCP_ZEROCOPY_RECEIVE + PyModule_AddIntMacro(m, TCP_ZEROCOPY_RECEIVE); +#endif +#ifdef TCP_INQ + PyModule_AddIntMacro(m, TCP_INQ); +#endif +#ifdef TCP_TX_DELAY + PyModule_AddIntMacro(m, TCP_TX_DELAY); +#endif /* IPX options */ #ifdef IPX_TYPE From webhook-mailer at python.org Thu Dec 8 01:58:15 2022 From: webhook-mailer at python.org (ethanfurman) Date: Thu, 08 Dec 2022 06:58:15 -0000 Subject: [Python-checkins] gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) Message-ID: <mailman.2855.1670482696.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ded02ca54d7bfa32c8eab0871d56e4547cd356eb commit: ded02ca54d7bfa32c8eab0871d56e4547cd356eb branch: main author: Ethan Furman <ethan at stoneleaf.us> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-07T22:58:08-08:00 summary: gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) When checking for auto() instances, only top-level usage is supported, which means either alone or as part of a regular tuple. Other containers, such as lists, dicts, or namedtuples, will not have auto() transformed into a value. files: A Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst M Lib/enum.py M Lib/test/test_enum.py diff --git a/Lib/enum.py b/Lib/enum.py index b2f7b9f8d4ad..a0cad066dc23 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -436,7 +436,9 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if isinstance(value, tuple): + if type(value) is tuple and any(isinstance(v, auto) for v in value): + # insist on an actual tuple, no subclasses, in keeping with only supporting + # top-level auto() usage (not contained in any other data structure) auto_valued = [] for v in value: if isinstance(v, auto): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 376347e4c111..d876c8e5fb77 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2841,6 +2841,19 @@ class MyIntFlag(IntFlag): self.assertEqual(deep, flags) self.assertEqual(copied.value, 1 | 2 | 8) + def test_namedtuple_as_value(self): + from collections import namedtuple + TTuple = namedtuple('TTuple', 'id a blist') + class NTEnum(Enum): + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTEnum.NONE), "<NTEnum.NONE: TTuple(id=0, a=0, blist=[])>") + self.assertEqual(NTEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual( + [x.value for x in NTEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" diff --git a/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst new file mode 100644 index 000000000000..256f2bcd39f8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst @@ -0,0 +1 @@ +Fix ``tuple`` subclasses being cast to ``tuple`` when used as enum values. From webhook-mailer at python.org Thu Dec 8 02:21:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 07:21:55 -0000 Subject: [Python-checkins] gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) Message-ID: <mailman.2856.1670484116.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d4426c829565e3ef922c091ee9bd48bc556f2550 commit: d4426c829565e3ef922c091ee9bd48bc556f2550 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-07T23:21:29-08:00 summary: gh-100098: [Enum] insist on actual tuples, no subclasses, for auto (GH-100099) When checking for auto() instances, only top-level usage is supported, which means either alone or as part of a regular tuple. Other containers, such as lists, dicts, or namedtuples, will not have auto() transformed into a value. (cherry picked from commit ded02ca54d7bfa32c8eab0871d56e4547cd356eb) Co-authored-by: Ethan Furman <ethan at stoneleaf.us> files: A Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst M Lib/enum.py M Lib/test/test_enum.py diff --git a/Lib/enum.py b/Lib/enum.py index 1efddfa1a0ac..7ad599bb1e62 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -430,7 +430,9 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if isinstance(value, tuple): + if type(value) is tuple and any(isinstance(v, auto) for v in value): + # insist on an actual tuple, no subclasses, in keeping with only supporting + # top-level auto() usage (not contained in any other data structure) auto_valued = [] for v in value: if isinstance(v, auto): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index a9b80ad0d3db..0e2da1d64c19 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2727,6 +2727,19 @@ class MyIntFlag(IntFlag): self.assertEqual(deep, flags) self.assertEqual(copied.value, 1 | 2 | 8) + def test_namedtuple_as_value(self): + from collections import namedtuple + TTuple = namedtuple('TTuple', 'id a blist') + class NTEnum(Enum): + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTEnum.NONE), "<NTEnum.NONE: TTuple(id=0, a=0, blist=[])>") + self.assertEqual(NTEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual( + [x.value for x in NTEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" diff --git a/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst new file mode 100644 index 000000000000..256f2bcd39f8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-08-06-18-06.gh-issue-100098.uBvPlp.rst @@ -0,0 +1 @@ +Fix ``tuple`` subclasses being cast to ``tuple`` when used as enum values. From webhook-mailer at python.org Thu Dec 8 03:21:20 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 08 Dec 2022 08:21:20 -0000 Subject: [Python-checkins] Fix `test_run_until_complete_baseexception` test to check for `KeyboardInterrupt` in asyncio (#24477) Message-ID: <mailman.2857.1670487681.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e8fff515f056737d6d055ea5438b2135863548b1 commit: e8fff515f056737d6d055ea5438b2135863548b1 branch: main author: Fantix King <fantix.king at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-08T13:51:04+05:30 summary: Fix `test_run_until_complete_baseexception` test to check for `KeyboardInterrupt` in asyncio (#24477) files: M Lib/test/test_asyncio/test_base_events.py diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 65dd4d42708b..532a7e5385ae 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -861,20 +861,15 @@ async def raise_keyboard_interrupt(): self.loop._process_events = mock.Mock() - try: + with self.assertRaises(KeyboardInterrupt): self.loop.run_until_complete(raise_keyboard_interrupt()) - except KeyboardInterrupt: - pass def func(): self.loop.stop() func.called = True func.called = False - try: - self.loop.call_soon(func) - self.loop.run_forever() - except KeyboardInterrupt: - pass + self.loop.call_later(0.01, func) + self.loop.run_forever() self.assertTrue(func.called) def test_single_selecter_event_callback_after_stopping(self): From webhook-mailer at python.org Thu Dec 8 03:26:44 2022 From: webhook-mailer at python.org (vstinner) Date: Thu, 08 Dec 2022 08:26:44 -0000 Subject: [Python-checkins] test_ast uses infinite_recursion() to prevent crash (#100104) Message-ID: <mailman.2858.1670488005.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cd67c1bb30eccd0c6fd1386405df225aed4c91a9 commit: cd67c1bb30eccd0c6fd1386405df225aed4c91a9 branch: main author: Victor Stinner <vstinner at python.org> committer: vstinner <vstinner at python.org> date: 2022-12-08T09:26:38+01:00 summary: test_ast uses infinite_recursion() to prevent crash (#100104) test.test_ast_recursion_limit() now uses infinite_recursion() of test.support to prevent crashes on debug builds. Before this change, the test crashed on ARM64 Windows 3.x buildbot worker which builds Python in debug mode. files: M Lib/test/test_ast.py diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 773fba87632b..ab6a63faa590 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -837,7 +837,8 @@ def check_limit(prefix, repeated): details = "Compiling ({!r} + {!r} * {})".format( prefix, repeated, depth) with self.assertRaises(RecursionError, msg=details): - ast.parse(broken) + with support.infinite_recursion(): + ast.parse(broken) check_limit("a", "()") check_limit("a", ".b") From webhook-mailer at python.org Thu Dec 8 03:52:08 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 08:52:08 -0000 Subject: [Python-checkins] test_ast uses infinite_recursion() to prevent crash (GH-100104) Message-ID: <mailman.2859.1670489529.3313.python-checkins@python.org> https://github.com/python/cpython/commit/42fde2d164eb6f9c31a01b290114026d857676ad commit: 42fde2d164eb6f9c31a01b290114026d857676ad branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T00:52:02-08:00 summary: test_ast uses infinite_recursion() to prevent crash (GH-100104) test.test_ast_recursion_limit() now uses infinite_recursion() of test.support to prevent crashes on debug builds. Before this change, the test crashed on ARM64 Windows 3.x buildbot worker which builds Python in debug mode. (cherry picked from commit cd67c1bb30eccd0c6fd1386405df225aed4c91a9) Co-authored-by: Victor Stinner <vstinner at python.org> files: M Lib/test/test_ast.py diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 7581adc8fc27..c33c0ae22379 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -831,7 +831,8 @@ def check_limit(prefix, repeated): details = "Compiling ({!r} + {!r} * {})".format( prefix, repeated, depth) with self.assertRaises(RecursionError, msg=details): - ast.parse(broken) + with support.infinite_recursion(): + ast.parse(broken) check_limit("a", "()") check_limit("a", ".b") From webhook-mailer at python.org Thu Dec 8 04:02:12 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 09:02:12 -0000 Subject: [Python-checkins] [3.11] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100022) Message-ID: <mailman.2860.1670490134.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f43b3d51a9488a7f2334fffde1c0209ebf054a58 commit: f43b3d51a9488a7f2334fffde1c0209ebf054a58 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T01:02:06-08:00 summary: [3.11] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100022) (cherry picked from commit 124ecd657646f808d1d3282c37ee19aae6bcb47f) Co-authored-by: Michael F?rderer <michael.foerderer at gmx.de> files: A Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst M Lib/ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 73b1bd12ddca..10c6799300f6 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -639,12 +639,15 @@ def _getfinalpathname_nonstrict(path): # 21: ERROR_NOT_READY (implies drive with no media) # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file) # 50: ERROR_NOT_SUPPORTED + # 53: ERROR_BAD_NETPATH + # 65: ERROR_NETWORK_ACCESS_DENIED # 67: ERROR_BAD_NET_NAME (implies remote server unavailable) # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME + # 161: ERROR_BAD_PATHNAME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst new file mode 100644 index 000000000000..79f8c506b54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst @@ -0,0 +1,2 @@ +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) +and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). From webhook-mailer at python.org Thu Dec 8 04:02:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 09:02:34 -0000 Subject: [Python-checkins] [3.10] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100023) Message-ID: <mailman.2861.1670490154.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0e2c7839bd297ad284fd07bf3736f722b87175df commit: 0e2c7839bd297ad284fd07bf3736f722b87175df branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T01:02:28-08:00 summary: [3.10] bpo-44817: Ignore additional errors in ntpath.realpath (GH-27574) (GH-100023) (cherry picked from commit 124ecd657646f808d1d3282c37ee19aae6bcb47f) Co-authored-by: Michael F?rderer <michael.foerderer at gmx.de> files: A Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst M Lib/ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 97edfa52aaaf..c14e5c7ceca5 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -625,12 +625,15 @@ def _getfinalpathname_nonstrict(path): # 21: ERROR_NOT_READY (implies drive with no media) # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file) # 50: ERROR_NOT_SUPPORTED + # 53: ERROR_BAD_NETPATH + # 65: ERROR_NETWORK_ACCESS_DENIED # 67: ERROR_BAD_NET_NAME (implies remote server unavailable) # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME + # 161: ERROR_BAD_PATHNAME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst new file mode 100644 index 000000000000..79f8c506b54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst @@ -0,0 +1,2 @@ +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) +and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). From webhook-mailer at python.org Thu Dec 8 12:35:05 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 17:35:05 -0000 Subject: [Python-checkins] Use sphinxext-opengraph to generate OpenGraph metadata (GH-99931) Message-ID: <mailman.2862.1670520907.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0274a3bc3b1a479c766915a562db6fb19ef97daa commit: 0274a3bc3b1a479c766915a562db6fb19ef97daa branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T09:34:52-08:00 summary: Use sphinxext-opengraph to generate OpenGraph metadata (GH-99931) (cherry picked from commit f49c735e525cf031ddbfc19161aafac4fb18837b) Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: A Doc/_static/og-image.png A Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst M Doc/conf.py M Doc/requirements.txt diff --git a/Doc/_static/og-image.png b/Doc/_static/og-image.png new file mode 100644 index 000000000000..0e80751e7403 Binary files /dev/null and b/Doc/_static/og-image.png differ diff --git a/Doc/conf.py b/Doc/conf.py index fd4ee2d5eee8..98b076c0ee57 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -13,9 +13,25 @@ # General configuration # --------------------- -extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', - 'pyspecific', 'c_annotations', 'escape4chm', - 'asdl_highlight', 'peg_highlight', 'glossary_search'] +extensions = [ + 'asdl_highlight', + 'c_annotations', + 'escape4chm', + 'glossary_search', + 'peg_highlight', + 'pyspecific', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', +] + +# Skip if downstream redistributors haven't installed it +try: + import sphinxext.opengraph +except ImportError: + pass +else: + extensions.append('sphinxext.opengraph') + doctest_global_setup = ''' try: @@ -109,7 +125,7 @@ html_use_opensearch = 'https://docs.python.org/' + version # Additional static files. -html_static_path = ['tools/static'] +html_static_path = ['_static', 'tools/static'] # Output file base name for HTML help builder. htmlhelp_basename = 'python' + release.replace('.', '') @@ -234,3 +250,13 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' + +# sphinxext-opengraph config +ogp_site_url = 'https://docs.python.org/3/' +ogp_site_name = 'Python documentation' +ogp_image = '_static/og-image.png' +ogp_custom_meta_tags = [ + '<meta property="og:image:width" content="200">', + '<meta property="og:image:height" content="200">', + '<meta name="theme-color" content="#3776ab">', +] diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 958665db69e2..134f39d6d7b3 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -8,6 +8,7 @@ sphinx==4.5.0 blurb sphinx-lint==0.6.7 +sphinxext-opengraph>=0.7.1 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst new file mode 100644 index 000000000000..0c01a2cb2cfa --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-02-17-08-08.gh-issue-99931.wC46hE.rst @@ -0,0 +1,2 @@ +Use `sphinxext-opengraph <https://sphinxext-opengraph.readthedocs.io/>`__ +to generate `OpenGraph metadata <https://ogp.me/>`__. From webhook-mailer at python.org Thu Dec 8 15:37:27 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 08 Dec 2022 20:37:27 -0000 Subject: [Python-checkins] gh-96250: Improve sqlite3 injection attack example (#99270) Message-ID: <mailman.2863.1670531848.3313.python-checkins@python.org> https://github.com/python/cpython/commit/41d4ac9da348ca33056e271d71588b2dc3a6d48d commit: 41d4ac9da348ca33056e271d71588b2dc3a6d48d branch: main author: Jia Junjie <62194633+jiajunjie at users.noreply.github.com> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-08T21:37:08+01:00 summary: gh-96250: Improve sqlite3 injection attack example (#99270) Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 960f2966afe1..2b6387cbfb52 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1929,12 +1929,16 @@ How to use placeholders to bind values in SQL queries SQL operations usually need to use values from Python variables. However, beware of using Python's string operations to assemble queries, as they -are vulnerable to `SQL injection attacks`_ (see the `xkcd webcomic -<https://xkcd.com/327/>`_ for a humorous example of what can go wrong):: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) +are vulnerable to `SQL injection attacks`_. For example, an attacker can simply +close the single quote and inject ``OR TRUE`` to select all rows:: + + >>> # Never do this -- insecure! + >>> symbol = input() + ' OR TRUE; -- + >>> sql = "SELECT * FROM stocks WHERE symbol = '%s'" % symbol + >>> print(sql) + SELECT * FROM stocks WHERE symbol = '' OR TRUE; --' + >>> cur.execute(sql) Instead, use the DB-API's parameter substitution. To insert a variable into a query string, use a placeholder in the string, and substitute the actual values From webhook-mailer at python.org Thu Dec 8 15:45:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 20:45:59 -0000 Subject: [Python-checkins] gh-96250: Improve sqlite3 injection attack example (GH-99270) Message-ID: <mailman.2864.1670532361.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8ef604518745989a3991a37f58368d2ce3c27d84 commit: 8ef604518745989a3991a37f58368d2ce3c27d84 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T12:45:32-08:00 summary: gh-96250: Improve sqlite3 injection attack example (GH-99270) (cherry picked from commit 41d4ac9da348ca33056e271d71588b2dc3a6d48d) Co-authored-by: Jia Junjie <62194633+jiajunjie at users.noreply.github.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9775f80624a0..65fa1b613153 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1427,12 +1427,16 @@ How to use placeholders to bind values in SQL queries SQL operations usually need to use values from Python variables. However, beware of using Python's string operations to assemble queries, as they -are vulnerable to `SQL injection attacks`_ (see the `xkcd webcomic -<https://xkcd.com/327/>`_ for a humorous example of what can go wrong):: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) +are vulnerable to `SQL injection attacks`_. For example, an attacker can simply +close the single quote and inject ``OR TRUE`` to select all rows:: + + >>> # Never do this -- insecure! + >>> symbol = input() + ' OR TRUE; -- + >>> sql = "SELECT * FROM stocks WHERE symbol = '%s'" % symbol + >>> print(sql) + SELECT * FROM stocks WHERE symbol = '' OR TRUE; --' + >>> cur.execute(sql) Instead, use the DB-API's parameter substitution. To insert a variable into a query string, use a placeholder in the string, and substitute the actual values From webhook-mailer at python.org Thu Dec 8 15:45:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 08 Dec 2022 20:45:59 -0000 Subject: [Python-checkins] gh-96250: Improve sqlite3 injection attack example (GH-99270) Message-ID: <mailman.2865.1670532361.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ecb16d5d63b2299c37c321ea4d8cf6b51c626ac1 commit: ecb16d5d63b2299c37c321ea4d8cf6b51c626ac1 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-08T12:45:40-08:00 summary: gh-96250: Improve sqlite3 injection attack example (GH-99270) (cherry picked from commit 41d4ac9da348ca33056e271d71588b2dc3a6d48d) Co-authored-by: Jia Junjie <62194633+jiajunjie at users.noreply.github.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index b720ac995c44..e8db6c55c938 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1813,12 +1813,16 @@ How to use placeholders to bind values in SQL queries SQL operations usually need to use values from Python variables. However, beware of using Python's string operations to assemble queries, as they -are vulnerable to `SQL injection attacks`_ (see the `xkcd webcomic -<https://xkcd.com/327/>`_ for a humorous example of what can go wrong):: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) +are vulnerable to `SQL injection attacks`_. For example, an attacker can simply +close the single quote and inject ``OR TRUE`` to select all rows:: + + >>> # Never do this -- insecure! + >>> symbol = input() + ' OR TRUE; -- + >>> sql = "SELECT * FROM stocks WHERE symbol = '%s'" % symbol + >>> print(sql) + SELECT * FROM stocks WHERE symbol = '' OR TRUE; --' + >>> cur.execute(sql) Instead, use the DB-API's parameter substitution. To insert a variable into a query string, use a placeholder in the string, and substitute the actual values From webhook-mailer at python.org Thu Dec 8 16:08:25 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 08 Dec 2022 21:08:25 -0000 Subject: [Python-checkins] GH-98363: Have batched() return tuples (GH-100118) Message-ID: <mailman.2866.1670533705.3313.python-checkins@python.org> https://github.com/python/cpython/commit/35cc0ea736a323119157117d93e5d68d8247e89f commit: 35cc0ea736a323119157117d93e5d68d8247e89f branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-08T15:08:16-06:00 summary: GH-98363: Have batched() return tuples (GH-100118) files: M Doc/library/itertools.rst M Lib/test/test_itertools.py M Modules/clinic/itertoolsmodule.c.h M Modules/itertoolsmodule.c diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0b5978505a96..624d2430ac20 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -52,7 +52,7 @@ Iterator Arguments Results Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`batched` p, n [p0, p1, ..., p_n-1], ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` @@ -166,11 +166,11 @@ loops that truncate the stream. .. function:: batched(iterable, n) - Batch data from the *iterable* into lists of length *n*. The last + Batch data from the *iterable* into tuples of length *n*. The last batch may be shorter than *n*. - Loops over the input iterable and accumulates data into lists up to - size *n*. The input is consumed lazily, just enough to fill a list. + Loops over the input iterable and accumulates data into tuples up to + size *n*. The input is consumed lazily, just enough to fill a batch. The result is yielded as soon as the batch is full or when the input iterable is exhausted: @@ -179,14 +179,14 @@ loops that truncate the stream. >>> flattened_data = ['roses', 'red', 'violets', 'blue', 'sugar', 'sweet'] >>> unflattened = list(batched(flattened_data, 2)) >>> unflattened - [['roses', 'red'], ['violets', 'blue'], ['sugar', 'sweet']] + [('roses', 'red'), ('violets', 'blue'), ('sugar', 'sweet')] >>> for batch in batched('ABCDEFG', 3): ... print(batch) ... - ['A', 'B', 'C'] - ['D', 'E', 'F'] - ['G'] + ('A', 'B', 'C') + ('D', 'E', 'F') + ('G',) Roughly equivalent to:: @@ -195,7 +195,7 @@ loops that truncate the stream. if n < 1: raise ValueError('n must be at least one') it = iter(iterable) - while (batch := list(islice(it, n))): + while (batch := tuple(islice(it, n))): yield batch .. versionadded:: 3.12 diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 5f5bcbc7cfb8..b447b6cbab9c 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -161,11 +161,11 @@ def test_accumulate(self): def test_batched(self): self.assertEqual(list(batched('ABCDEFG', 3)), - [['A', 'B', 'C'], ['D', 'E', 'F'], ['G']]) + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 2)), - [['A', 'B'], ['C', 'D'], ['E', 'F'], ['G']]) + [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 1)), - [['A'], ['B'], ['C'], ['D'], ['E'], ['F'], ['G']]) + [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)]) with self.assertRaises(TypeError): # Too few arguments list(batched('ABCDEFG')) @@ -188,8 +188,8 @@ def test_batched(self): with self.subTest(s=s, n=n, batches=batches): # Order is preserved and no data is lost self.assertEqual(''.join(chain(*batches)), s) - # Each batch is an exact list - self.assertTrue(all(type(batch) is list for batch in batches)) + # Each batch is an exact tuple + self.assertTrue(all(type(batch) is tuple for batch in batches)) # All but the last batch is of size n if batches: last_batch = batches.pop() @@ -1809,12 +1809,12 @@ class TestPurePythonRoughEquivalents(unittest.TestCase): def test_batched_recipe(self): def batched_recipe(iterable, n): - "Batch data into lists of length n. The last batch may be shorter." + "Batch data into tuples of length n. The last batch may be shorter." # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: raise ValueError('n must be at least one') it = iter(iterable) - while (batch := list(islice(it, n))): + while (batch := tuple(islice(it, n))): yield batch for iterable, n in product( @@ -2087,7 +2087,7 @@ def test_accumulate(self): def test_batched(self): s = 'abcde' - r = [['a', 'b'], ['c', 'd'], ['e']] + r = [('a', 'b'), ('c', 'd'), ('e',)] n = 2 for g in (G, I, Ig, L, R): with self.subTest(g=g): diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 17f9ebb24939..287de524e913 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -12,19 +12,19 @@ PyDoc_STRVAR(batched_new__doc__, "batched(iterable, n)\n" "--\n" "\n" -"Batch data into lists of length n. The last batch may be shorter than n.\n" +"Batch data into tuples of length n. The last batch may be shorter than n.\n" "\n" -"Loops over the input iterable and accumulates data into lists\n" +"Loops over the input iterable and accumulates data into tuples\n" "up to size n. The input is consumed lazily, just enough to\n" -"fill a list. The result is yielded as soon as a batch is full\n" +"fill a batch. The result is yielded as soon as a batch is full\n" "or when the input iterable is exhausted.\n" "\n" " >>> for batch in batched(\'ABCDEFG\', 3):\n" " ... print(batch)\n" " ...\n" -" [\'A\', \'B\', \'C\']\n" -" [\'D\', \'E\', \'F\']\n" -" [\'G\']"); +" (\'A\', \'B\', \'C\')\n" +" (\'D\', \'E\', \'F\')\n" +" (\'G\',)"); static PyObject * batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n); @@ -913,4 +913,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=efea8cd1e647bd17 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0229ebd72962f130 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d45000788c53..60ec11c32d01 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -56,11 +56,13 @@ static PyTypeObject pairwise_type; /* batched object ************************************************************/ /* Note: The built-in zip() function includes a "strict" argument - that is needed because that function can silently truncate data - and there is no easy way for a user to detect that condition. - The same reasoning does not apply to batched() which never drops - data. Instead, it produces a shorter list which can be handled - as the user sees fit. + that was needed because that function would silently truncate data, + and there was no easy way for a user to detect the data loss. + The same reasoning does not apply to batched() which never drops data. + Instead, batched() produces a shorter tuple which can be handled + as the user sees fit. If requested, it would be reasonable to add + "fillvalue" support which had demonstrated value in zip_longest(). + For now, the API is kept simple and clean. */ typedef struct { @@ -74,25 +76,25 @@ typedef struct { itertools.batched.__new__ as batched_new iterable: object n: Py_ssize_t -Batch data into lists of length n. The last batch may be shorter than n. +Batch data into tuples of length n. The last batch may be shorter than n. -Loops over the input iterable and accumulates data into lists +Loops over the input iterable and accumulates data into tuples up to size n. The input is consumed lazily, just enough to -fill a list. The result is yielded as soon as a batch is full +fill a batch. The result is yielded as soon as a batch is full or when the input iterable is exhausted. >>> for batch in batched('ABCDEFG', 3): ... print(batch) ... - ['A', 'B', 'C'] - ['D', 'E', 'F'] - ['G'] + ('A', 'B', 'C') + ('D', 'E', 'F') + ('G',) [clinic start generated code]*/ static PyObject * batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) -/*[clinic end generated code: output=7ebc954d655371b6 input=f28fd12cb52365f0]*/ +/*[clinic end generated code: output=7ebc954d655371b6 input=ffd70726927c5129]*/ { PyObject *it; batchedobject *bo; @@ -150,12 +152,12 @@ batched_next(batchedobject *bo) if (it == NULL) { return NULL; } - result = PyList_New(n); + result = PyTuple_New(n); if (result == NULL) { return NULL; } iternextfunc iternext = *Py_TYPE(it)->tp_iternext; - PyObject **items = _PyList_ITEMS(result); + PyObject **items = _PyTuple_ITEMS(result); for (i=0 ; i < n ; i++) { item = iternext(it); if (item == NULL) { From webhook-mailer at python.org Thu Dec 8 16:31:33 2022 From: webhook-mailer at python.org (gvanrossum) Date: Thu, 08 Dec 2022 21:31:33 -0000 Subject: [Python-checkins] GH-98831: Typed stack effects, and more instructions converted (#99764) Message-ID: <mailman.2867.1670535095.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c85be734d1823f02a3600d7cd9195cecbf51afe8 commit: c85be734d1823f02a3600d7cd9195cecbf51afe8 branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-08T13:31:27-08:00 summary: GH-98831: Typed stack effects, and more instructions converted (#99764) Stack effects can now have a type, e.g. `inst(X, (left, right -- jump/uint64_t)) { ... }`. Instructions converted to the non-legacy format: * COMPARE_OP * COMPARE_OP_FLOAT_JUMP * COMPARE_OP_INT_JUMP * COMPARE_OP_STR_JUMP * STORE_ATTR * DELETE_ATTR * STORE_GLOBAL * STORE_ATTR_INSTANCE_VALUE * STORE_ATTR_WITH_HINT * STORE_ATTR_SLOT, and complete the store_attr family * Complete the store_subscr family: STORE_SUBSCR{,DICT,LIST_INT} (STORE_SUBSCR was alread half converted, but wasn't using cache effects yet.) * DELETE_SUBSCR * PRINT_EXPR * INTERPRETER_EXIT (a bit weird, ends in return) * RETURN_VALUE * GET_AITER (had to restructure it some) The original had mysterious `SET_TOP(NULL)` before `goto error`. I assume those just account for `obj` having been decref'ed, so I got rid of them in favor of the cleanup implied by `ERROR_IF()`. * LIST_APPEND (a bit unhappy with it) * SET_ADD (also a bit unhappy with it) Various other improvements/refactorings as well. files: M Python/bytecodes.c M Python/generated_cases.c.h M Tools/cases_generator/generate_cases.py M Tools/cases_generator/parser.py diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d0480ac01eb6..7a1bfdcc9e26 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -81,8 +81,17 @@ do { \ // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; static PyObject *container, *start, *stop, *v, *lhs, *rhs; -static PyObject *list, *tuple, *dict; -static PyObject *exit_func, *lasti, *val; +static PyObject *list, *tuple, *dict, *owner; +static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter; +static size_t jump; +// Dummy variables for cache effects +static _Py_CODEUNIT when_to_jump_mask, invert, counter, index, hint; +static uint32_t type_version; +// Dummy opcode names for 'op' opcodes +#define _COMPARE_OP_FLOAT 1003 +#define _COMPARE_OP_INT 1004 +#define _COMPARE_OP_STR 1005 +#define _JUMP_IF 1006 static PyObject * dummy_func( @@ -205,7 +214,7 @@ dummy_func( }; - inst(BINARY_OP_MULTIPLY_INT, (left, right, unused/1 -- prod)) { + inst(BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- prod)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); @@ -216,7 +225,7 @@ dummy_func( ERROR_IF(prod == NULL, error); } - inst(BINARY_OP_MULTIPLY_FLOAT, (left, right, unused/1 -- prod)) { + inst(BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- prod)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); @@ -229,7 +238,7 @@ dummy_func( ERROR_IF(prod == NULL, error); } - inst(BINARY_OP_SUBTRACT_INT, (left, right, unused/1 -- sub)) { + inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); @@ -240,7 +249,7 @@ dummy_func( ERROR_IF(sub == NULL, error); } - inst(BINARY_OP_SUBTRACT_FLOAT, (left, right, unused/1 -- sub)) { + inst(BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- sub)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); @@ -252,7 +261,7 @@ dummy_func( ERROR_IF(sub == NULL, error); } - inst(BINARY_OP_ADD_UNICODE, (left, right, unused/1 -- res)) { + inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -299,7 +308,7 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); } - inst(BINARY_OP_ADD_FLOAT, (left, right, unused/1 -- sum)) { + inst(BINARY_OP_ADD_FLOAT, (unused/1, left, right -- sum)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -312,7 +321,7 @@ dummy_func( ERROR_IF(sum == NULL, error); } - inst(BINARY_OP_ADD_INT, (left, right, unused/1 -- sum)) { + inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -331,7 +340,7 @@ dummy_func( BINARY_SUBSCR_TUPLE_INT, }; - inst(BINARY_SUBSCR, (container, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR, (unused/4, container, sub -- res)) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -377,7 +386,7 @@ dummy_func( ERROR_IF(err, error); } - inst(BINARY_SUBSCR_LIST_INT, (list, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR_LIST_INT, (unused/4, list, sub -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -396,7 +405,7 @@ dummy_func( Py_DECREF(list); } - inst(BINARY_SUBSCR_TUPLE_INT, (tuple, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR_TUPLE_INT, (unused/4, tuple, sub -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -415,7 +424,7 @@ dummy_func( Py_DECREF(tuple); } - inst(BINARY_SUBSCR_DICT, (dict, sub, unused/4 -- res)) { + inst(BINARY_SUBSCR_DICT, (unused/4, dict, sub -- res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -426,14 +435,14 @@ dummy_func( } Py_DECREF(dict); Py_DECREF(sub); - ERROR_IF(1, error); + ERROR_IF(true, error); } Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); Py_DECREF(sub); } - inst(BINARY_SUBSCR_GETITEM, (container, sub, unused/1, type_version/2, func_version/1 -- unused)) { + inst(BINARY_SUBSCR_GETITEM, (unused/1, type_version/2, func_version/1, container, sub -- unused)) { PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -457,52 +466,48 @@ dummy_func( DISPATCH_INLINED(new_frame); } - // stack effect: (__0 -- ) - inst(LIST_APPEND) { - PyObject *v = POP(); - PyObject *list = PEEK(oparg); - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) - goto error; + // Alternative: (list, unused[oparg], v -- list, unused[oparg]) + inst(LIST_APPEND, (v --)) { + PyObject *list = PEEK(oparg + 1); // +1 to account for v staying on stack + ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error); PREDICT(JUMP_BACKWARD); } - // stack effect: (__0 -- ) - inst(SET_ADD) { - PyObject *v = POP(); - PyObject *set = PEEK(oparg); - int err; - err = PySet_Add(set, v); + // Alternative: (set, unused[oparg], v -- set, unused[oparg]) + inst(SET_ADD, (v --)) { + PyObject *set = PEEK(oparg + 1); // +1 to account for v staying on stack + int err = PySet_Add(set, v); Py_DECREF(v); - if (err != 0) - goto error; + ERROR_IF(err, error); PREDICT(JUMP_BACKWARD); } - inst(STORE_SUBSCR, (v, container, sub -- )) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + family(store_subscr) = { + STORE_SUBSCR, + STORE_SUBSCR_DICT, + STORE_SUBSCR_LIST_INT, + }; + + inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - ERROR_IF(err != 0, error); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + ERROR_IF(err, error); } - // stack effect: (__0, __1, __2 -- ) - inst(STORE_SUBSCR_LIST_INT) { + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *list = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -515,60 +520,42 @@ dummy_func( PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); - STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); } - // stack effect: (__0, __1, __2 -- ) - inst(STORE_SUBSCR_DICT) { + inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *dict = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STACK_SHRINK(3); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + ERROR_IF(err, error); } - // stack effect: (__0, __1 -- ) - inst(DELETE_SUBSCR) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - int err; - STACK_SHRINK(2); + inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ - err = PyObject_DelItem(container, sub); + int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) - goto error; + ERROR_IF(err, error); } - // stack effect: (__0 -- ) - inst(PRINT_EXPR) { - PyObject *value = POP(); + inst(PRINT_EXPR, (value --)) { PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook)); PyObject *res; + // Can't use ERROR_IF here. if (hook == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); Py_DECREF(value); - goto error; + ERROR_IF(true, error); } res = PyObject_CallOneArg(hook, value); Py_DECREF(value); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); Py_DECREF(res); } @@ -595,11 +582,10 @@ dummy_func( goto error; } - // stack effect: (__0 -- ) - inst(INTERPRETER_EXIT) { + inst(INTERPRETER_EXIT, (retval --)) { assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - PyObject *retval = POP(); + STACK_SHRINK(1); // Since we're not going to DISPATCH() assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; @@ -610,9 +596,8 @@ dummy_func( return retval; } - // stack effect: (__0 -- ) - inst(RETURN_VALUE) { - PyObject *retval = POP(); + inst(RETURN_VALUE, (retval --)) { + STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); @@ -627,48 +612,37 @@ dummy_func( goto resume_frame; } - // stack effect: ( -- ) - inst(GET_AITER) { + inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; - PyObject *iter = NULL; - PyObject *obj = TOP(); PyTypeObject *type = Py_TYPE(obj); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter != NULL) { - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) { - SET_TOP(NULL); - goto error; - } - } - else { - SET_TOP(NULL); + if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - goto error; + ERROR_IF(true, error); } + iter = (*getter)(obj); + Py_DECREF(obj); + ERROR_IF(iter == NULL, error); + if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - SET_TOP(NULL); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - goto error; + ERROR_IF(true, error); } - - SET_TOP(iter); } // stack effect: ( -- __0) @@ -1119,53 +1093,43 @@ dummy_func( Py_DECREF(seq); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + family(store_attr) = { + STORE_ATTR, + STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR_SLOT, + STORE_ATTR_WITH_HINT, + }; + + inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); + int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + ERROR_IF(err, error); } - // stack effect: (__0 -- ) - inst(DELETE_ATTR) { + inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(names, oparg); - PyObject *owner = POP(); - int err; - err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); - if (err != 0) - goto error; + ERROR_IF(err, error); } - // stack effect: (__0 -- ) - inst(STORE_GLOBAL) { + inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); - int err; - err = PyDict_SetItem(GLOBALS(), name, v); + int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err != 0) - goto error; + ERROR_IF(err, error); } inst(DELETE_GLOBAL, (--)) { @@ -1954,22 +1918,15 @@ dummy_func( DISPATCH_INLINED(new_frame); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR_INSTANCE_VALUE) { + inst(STORE_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, value, owner --)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); STAT_INC(STORE_ATTR, hit); - Py_ssize_t index = cache->index; - STACK_SHRINK(1); - PyObject *value = POP(); PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; values->values[index] = value; @@ -1980,16 +1937,11 @@ dummy_func( Py_DECREF(old_value); } Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR_WITH_HINT) { + inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -1999,17 +1951,14 @@ dummy_func( DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg); - uint16_t hint = cache->index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *value, *old_value; + PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2018,8 +1967,6 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2032,36 +1979,32 @@ dummy_func( /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR_SLOT) { + inst(STORE_ATTR_SLOT, (unused/1, type_version/2, index/1, value, owner --)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - char *addr = (char *)owner + cache->index; + char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); - STACK_SHRINK(1); - PyObject *value = POP(); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } - // stack effect: (__0 -- ) - inst(COMPARE_OP) { + family(compare_op) = { + COMPARE_OP, + _COMPARE_OP_FLOAT, + _COMPARE_OP_INT, + _COMPARE_OP_STR, + }; + + inst(COMPARE_OP, (unused/2, left, right -- res)) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *right = TOP(); - PyObject *left = SECOND(); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2069,57 +2012,43 @@ dummy_func( STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); + res = PyObject_RichCompare(left, right, oparg); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + ERROR_IF(res == NULL, error); } - // stack effect: (__0 -- ) - inst(COMPARE_OP_FLOAT_JUMP) { + // The result is an int disguised as an object pointer. + op(_COMPARE_OP_FLOAT, (unused/1, when_to_jump_mask/1, left, right -- jump: size_t)) { assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - int sign = (dleft > dright) - (dleft < dright); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); DEOPT_IF(isnan(dleft), COMPARE_OP); DEOPT_IF(isnan(dright), COMPARE_OP); STAT_INC(COMPARE_OP, hit); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + jump = sign_ish & when_to_jump_mask; + } + // The input is an int disguised as an object pointer! + op(_JUMP_IF, (jump: size_t --)) { assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); + if (jump) { + JUMPBY(oparg); } } + // We're praying that the compiler optimizes the flags manipuations. + super(COMPARE_OP_FLOAT_JUMP) = _COMPARE_OP_FLOAT + _JUMP_IF; - // stack effect: (__0 -- ) - inst(COMPARE_OP_INT_JUMP) { + // Similar to COMPARE_OP_FLOAT + op(_COMPARE_OP_INT, (unused/1, when_to_jump_mask/1, left, right -- jump: size_t)) { assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); @@ -2128,51 +2057,30 @@ dummy_func( assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - int sign = (ileft > iright) - (ileft < iright); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } + jump = sign_ish & when_to_jump_mask; } + super(COMPARE_OP_INT_JUMP) = _COMPARE_OP_INT + _JUMP_IF; - // stack effect: (__0 -- ) - inst(COMPARE_OP_STR_JUMP) { + // Similar to COMPARE_OP_FLOAT, but for ==, != only + op(_COMPARE_OP_STR, (unused/1, invert/1, left, right -- jump: size_t)) { assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int invert = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); int res = _PyUnicode_Equal(left, right); assert(oparg == Py_EQ || oparg == Py_NE); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - STACK_SHRINK(2); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(res == 0 || res == 1); assert(invert == 0 || invert == 1); - int jump = res ^ invert; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } + jump = res ^ invert; } + super(COMPARE_OP_STR_JUMP) = _COMPARE_OP_STR + _JUMP_IF; // stack effect: (__0 -- ) inst(IS_OP) { @@ -3633,7 +3541,7 @@ dummy_func( PUSH(Py_NewRef(peek)); } - inst(BINARY_OP, (lhs, rhs, unused/1 -- res)) { + inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -3691,9 +3599,6 @@ dummy_func( // Future families go below this point // -family(binary_subscr) = { - BINARY_SUBSCR, BINARY_SUBSCR_DICT, - BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, BINARY_SUBSCR_TUPLE_INT }; family(call) = { CALL, CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS, @@ -3702,9 +3607,6 @@ family(call) = { CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, CALL_NO_KW_TYPE_1 }; -family(compare_op) = { - COMPARE_OP, COMPARE_OP_FLOAT_JUMP, - COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP }; family(for_iter) = { FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; @@ -3719,13 +3621,7 @@ family(load_fast) = { LOAD_FAST, LOAD_FAST__LOAD_CONST, LOAD_FAST__LOAD_FAST }; family(load_global) = { LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN, LOAD_GLOBAL_MODULE }; -family(store_attr) = { - STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, - STORE_ATTR_SLOT, STORE_ATTR_WITH_HINT }; family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST }; -family(store_subscr) = { - STORE_SUBSCR, STORE_SUBSCR_DICT, - STORE_SUBSCR_LIST_INT }; family(unpack_sequence) = { UNPACK_SEQUENCE, UNPACK_SEQUENCE_LIST, UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_TWO_TUPLE }; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0805386866b3..510b6c4a75b8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -139,7 +139,7 @@ if (prod == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, prod); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -159,7 +159,7 @@ if (prod == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, prod); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -177,7 +177,7 @@ if (sub == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sub); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -196,7 +196,7 @@ if (sub == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sub); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -214,7 +214,7 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, res); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -268,7 +268,7 @@ if (sum == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sum); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -286,7 +286,7 @@ if (sum == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, sum); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -311,7 +311,7 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -380,7 +380,7 @@ Py_DECREF(list); STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -406,7 +406,7 @@ Py_DECREF(tuple); STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -424,14 +424,14 @@ } Py_DECREF(dict); Py_DECREF(sub); - if (1) goto pop_2_error; + if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); POKE(1, res); - next_instr += 4; + JUMPBY(4); DISPATCH(); } @@ -464,22 +464,21 @@ } TARGET(LIST_APPEND) { - PyObject *v = POP(); - PyObject *list = PEEK(oparg); - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) - goto error; + PyObject *v = PEEK(1); + PyObject *list = PEEK(oparg + 1); // +1 to account for v staying on stack + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); } TARGET(SET_ADD) { - PyObject *v = POP(); - PyObject *set = PEEK(oparg); - int err; - err = PySet_Add(set, v); + PyObject *v = PEEK(1); + PyObject *set = PEEK(oparg + 1); // +1 to account for v staying on stack + int err = PySet_Add(set, v); Py_DECREF(v); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); } @@ -489,31 +488,32 @@ PyObject *sub = PEEK(1); PyObject *container = PEEK(2); PyObject *v = PEEK(3); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + uint16_t counter = read_u16(next_instr + 0); + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) goto pop_3_error; - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (err) goto pop_3_error; STACK_SHRINK(3); + JUMPBY(1); DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { + PyObject *sub = PEEK(1); + PyObject *list = PEEK(2); + PyObject *value = PEEK(3); assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *list = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -526,61 +526,58 @@ PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); - STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + STACK_SHRINK(3); + JUMPBY(1); DISPATCH(); } TARGET(STORE_SUBSCR_DICT) { + PyObject *sub = PEEK(1); + PyObject *dict = PEEK(2); + PyObject *value = PEEK(3); assert(cframe.use_tracing == 0); - PyObject *sub = TOP(); - PyObject *dict = SECOND(); - PyObject *value = THIRD(); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STACK_SHRINK(3); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (err) goto pop_3_error; + STACK_SHRINK(3); + JUMPBY(1); DISPATCH(); } TARGET(DELETE_SUBSCR) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - int err; - STACK_SHRINK(2); + PyObject *sub = PEEK(1); + PyObject *container = PEEK(2); /* del container[sub] */ - err = PyObject_DelItem(container, sub); + int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) - goto error; + if (err) goto pop_2_error; + STACK_SHRINK(2); DISPATCH(); } TARGET(PRINT_EXPR) { - PyObject *value = POP(); + PyObject *value = PEEK(1); PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook)); PyObject *res; + // Can't use ERROR_IF here. if (hook == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); Py_DECREF(value); - goto error; + if (true) goto pop_1_error; } res = PyObject_CallOneArg(hook, value); Py_DECREF(value); - if (res == NULL) - goto error; + if (res == NULL) goto pop_1_error; Py_DECREF(res); + STACK_SHRINK(1); DISPATCH(); } @@ -607,9 +604,10 @@ } TARGET(INTERPRETER_EXIT) { + PyObject *retval = PEEK(1); assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - PyObject *retval = POP(); + STACK_SHRINK(1); // Since we're not going to DISPATCH() assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; @@ -621,7 +619,8 @@ } TARGET(RETURN_VALUE) { - PyObject *retval = POP(); + PyObject *retval = PEEK(1); + STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); @@ -637,46 +636,39 @@ } TARGET(GET_AITER) { + PyObject *obj = PEEK(1); + PyObject *iter; unaryfunc getter = NULL; - PyObject *iter = NULL; - PyObject *obj = TOP(); PyTypeObject *type = Py_TYPE(obj); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter != NULL) { - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) { - SET_TOP(NULL); - goto error; - } - } - else { - SET_TOP(NULL); + if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - goto error; + if (true) goto pop_1_error; } + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; + if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - SET_TOP(NULL); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - goto error; + if (true) goto pop_1_error; } - - SET_TOP(iter); + POKE(1, iter); DISPATCH(); } @@ -1131,51 +1123,46 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + PyObject *owner = PEEK(1); + PyObject *v = PEEK(2); + uint16_t counter = read_u16(next_instr + 0); + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); + int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + if (err) goto pop_2_error; + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(DELETE_ATTR) { + PyObject *owner = PEEK(1); PyObject *name = GETITEM(names, oparg); - PyObject *owner = POP(); - int err; - err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { + PyObject *v = PEEK(1); PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); - int err; - err = PyDict_SetItem(GLOBALS(), name, v); + int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } @@ -1970,20 +1957,18 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { + PyObject *owner = PEEK(1); + PyObject *value = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t index = read_u16(next_instr + 3); assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); STAT_INC(STORE_ATTR, hit); - Py_ssize_t index = cache->index; - STACK_SHRINK(1); - PyObject *value = POP(); PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; values->values[index] = value; @@ -1994,16 +1979,18 @@ Py_DECREF(old_value); } Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(STORE_ATTR_WITH_HINT) { + PyObject *owner = PEEK(1); + PyObject *value = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t hint = read_u16(next_instr + 3); assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -2013,17 +2000,14 @@ DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg); - uint16_t hint = cache->index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *value, *old_value; + PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2032,8 +2016,6 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - STACK_SHRINK(1); - value = POP(); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2046,37 +2028,39 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(STORE_ATTR_SLOT) { + PyObject *owner = PEEK(1); + PyObject *value = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t index = read_u16(next_instr + 3); assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - char *addr = (char *)owner + cache->index; + char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); - STACK_SHRINK(1); - PyObject *value = POP(); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + STACK_SHRINK(2); + JUMPBY(4); DISPATCH(); } TARGET(COMPARE_OP) { PREDICTED(COMPARE_OP); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *res; _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *right = TOP(); - PyObject *left = SECOND(); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2084,109 +2068,13 @@ STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); + res = PyObject_RichCompare(left, right, oparg); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - DISPATCH(); - } - - TARGET(COMPARE_OP_FLOAT_JUMP) { - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - int sign = (dleft > dright) - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } - DISPATCH(); - } - - TARGET(COMPARE_OP_INT_JUMP) { - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); - Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; - Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - int sign = (ileft > iright) - (ileft < iright); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - int jump = (1 << (sign + 1)) & when_to_jump_mask; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } - DISPATCH(); - } - - TARGET(COMPARE_OP_STR_JUMP) { - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int invert = cache->mask; - PyObject *right = TOP(); - PyObject *left = SECOND(); - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int res = _PyUnicode_Equal(left, right); - assert(oparg == Py_EQ || oparg == Py_NE); - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - NEXTOPARG(); - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(res == 0 || res == 1); - assert(invert == 0 || invert == 1); - int jump = res ^ invert; - if (!jump) { - next_instr++; - } - else { - JUMPBY(1 + oparg); - } + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(2); DISPATCH(); } @@ -3681,7 +3569,7 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); POKE(1, res); - next_instr += 1; + JUMPBY(1); DISPATCH(); } @@ -3714,20 +3602,20 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_1 = value; + _tmp_2 = value; } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_2 = value; + _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_2); - POKE(2, _tmp_1); + POKE(1, _tmp_1); + POKE(2, _tmp_2); DISPATCH(); } @@ -3739,19 +3627,19 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_1 = value; + _tmp_2 = value; } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETITEM(consts, oparg); Py_INCREF(value); - _tmp_2 = value; + _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_2); - POKE(2, _tmp_1); + POKE(1, _tmp_1); + POKE(2, _tmp_2); DISPATCH(); } @@ -3762,7 +3650,7 @@ SETLOCAL(oparg, value); } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETLOCAL(oparg); @@ -3775,16 +3663,16 @@ } TARGET(STORE_FAST__STORE_FAST) { - PyObject *_tmp_1 = PEEK(2); - PyObject *_tmp_2 = PEEK(1); + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); { - PyObject *value = _tmp_2; + PyObject *value = _tmp_1; SETLOCAL(oparg, value); } NEXTOPARG(); - next_instr++; + JUMPBY(1); { - PyObject *value = _tmp_1; + PyObject *value = _tmp_2; SETLOCAL(oparg, value); } STACK_SHRINK(2); @@ -3798,32 +3686,145 @@ PyObject *value; value = GETITEM(consts, oparg); Py_INCREF(value); - _tmp_1 = value; + _tmp_2 = value; } NEXTOPARG(); - next_instr++; + JUMPBY(1); { PyObject *value; value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - _tmp_2 = value; + _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_2); - POKE(2, _tmp_1); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + + TARGET(COMPARE_OP_FLOAT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); + DEOPT_IF(isnan(dleft), COMPARE_OP); + DEOPT_IF(isnan(dright), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_INT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); + Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; + Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_STR_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t invert = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int res = _PyUnicode_Equal(left, right); + assert(oparg == Py_EQ || oparg == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(res == 0 || res == 1); + assert(invert == 0 || invert == 1); + jump = res ^ invert; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); DISPATCH(); } TARGET(END_FOR) { - PyObject *_tmp_1 = PEEK(2); - PyObject *_tmp_2 = PEEK(1); + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); { - PyObject *value = _tmp_2; + PyObject *value = _tmp_1; Py_DECREF(value); } { - PyObject *value = _tmp_1; + PyObject *value = _tmp_2; Py_DECREF(value); } STACK_SHRINK(2); diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 2952634a3cda..844272291245 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -13,6 +13,7 @@ import typing import parser +from parser import StackEffect DEFAULT_INPUT = os.path.relpath( os.path.join(os.path.dirname(__file__), "../../Python/bytecodes.c") @@ -22,7 +23,7 @@ ) BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" -RE_PREDICTED = r"(?s)(?:PREDICT$|GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)$;" +RE_PREDICTED = r"^\s*(?:PREDICT$|GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)$;\s*$" UNUSED = "unused" BITS_PER_CODE_UNIT = 16 @@ -73,6 +74,34 @@ def block(self, head: str): yield self.emit("}") + def stack_adjust(self, diff: int): + if diff > 0: + self.emit(f"STACK_GROW({diff});") + elif diff < 0: + self.emit(f"STACK_SHRINK({-diff});") + + def declare(self, dst: StackEffect, src: StackEffect | None): + if dst.name == UNUSED: + return + typ = f"{dst.type} " if dst.type else "PyObject *" + init = "" + if src: + cast = self.cast(dst, src) + init = f" = {cast}{src.name}" + self.emit(f"{typ}{dst.name}{init};") + + def assign(self, dst: StackEffect, src: StackEffect): + if src.name == UNUSED: + return + cast = self.cast(dst, src) + if m := re.match(r"^PEEK$(\d+)$$", dst.name): + self.emit(f"POKE({m.group(1)}, {cast}{src.name});") + else: + self.emit(f"{dst.name} = {cast}{src.name};") + + def cast(self, dst: StackEffect, src: StackEffect) -> str: + return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" + @dataclasses.dataclass class Instruction: @@ -83,13 +112,15 @@ class Instruction: kind: typing.Literal["inst", "op"] name: str block: parser.Block + block_text: list[str] # Block.text, less curlies, less PREDICT() calls + predictions: list[str] # Prediction targets (instruction names) # Computed by constructor always_exits: bool cache_offset: int cache_effects: list[parser.CacheEffect] - input_effects: list[parser.StackEffect] - output_effects: list[parser.StackEffect] + input_effects: list[StackEffect] + output_effects: list[StackEffect] # Set later family: parser.Family | None = None @@ -100,13 +131,14 @@ def __init__(self, inst: parser.InstDef): self.kind = inst.kind self.name = inst.name self.block = inst.block - self.always_exits = always_exits(self.block) + self.block_text, self.predictions = extract_block_text(self.block) + self.always_exits = always_exits(self.block_text) self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parser.CacheEffect) ] self.cache_offset = sum(c.size for c in self.cache_effects) self.input_effects = [ - effect for effect in inst.inputs if isinstance(effect, parser.StackEffect) + effect for effect in inst.inputs if isinstance(effect, StackEffect) ] self.output_effects = inst.outputs # For consistency/completeness @@ -122,42 +154,39 @@ def write(self, out: Formatter) -> None: ) # Write input stack effect variable declarations and initializations - for i, seffect in enumerate(reversed(self.input_effects), 1): - if seffect.name != UNUSED: - out.emit(f"PyObject *{seffect.name} = PEEK({i});") + for i, ieffect in enumerate(reversed(self.input_effects), 1): + src = StackEffect(f"PEEK({i})", "") + out.declare(ieffect, src) # Write output stack effect variable declarations - input_names = {seffect.name for seffect in self.input_effects} - input_names.add(UNUSED) - for seffect in self.output_effects: - if seffect.name not in input_names: - out.emit(f"PyObject *{seffect.name};") + input_names = {ieffect.name for ieffect in self.input_effects} + for oeffect in self.output_effects: + if oeffect.name not in input_names: + out.declare(oeffect, None) self.write_body(out, 0) # Skip the rest if the block always exits - if always_exits(self.block): + if self.always_exits: return # Write net stack growth/shrinkage diff = len(self.output_effects) - len(self.input_effects) - if diff > 0: - out.emit(f"STACK_GROW({diff});") - elif diff < 0: - out.emit(f"STACK_SHRINK({-diff});") + out.stack_adjust(diff) # Write output stack effect assignments - unmoved_names = {UNUSED} + unmoved_names: set[str] = set() for ieffect, oeffect in zip(self.input_effects, self.output_effects): if ieffect.name == oeffect.name: unmoved_names.add(ieffect.name) - for i, seffect in enumerate(reversed(self.output_effects)): - if seffect.name not in unmoved_names: - out.emit(f"POKE({i+1}, {seffect.name});") + for i, oeffect in enumerate(reversed(self.output_effects), 1): + if oeffect.name not in unmoved_names: + dst = StackEffect(f"PEEK({i})", "") + out.assign(dst, oeffect) # Write cache effect if self.cache_offset: - out.emit(f"next_instr += {self.cache_offset};") + out.emit(f"JUMPBY({self.cache_offset});") def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None: """Write the instruction body.""" @@ -171,36 +200,19 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None # is always an object pointer. # If this becomes false, we need a way to specify # syntactically what type the cache data is. - type = "PyObject *" + typ = "PyObject *" func = "read_obj" else: - type = f"uint{bits}_t " + typ = f"uint{bits}_t " func = f"read_u{bits}" - out.emit(f"{type}{ceffect.name} = {func}(next_instr + {cache_offset});") + out.emit(f"{typ}{ceffect.name} = {func}(next_instr + {cache_offset});") cache_offset += ceffect.size assert cache_offset == self.cache_offset + cache_adjust - # Get lines of text with proper dedent - blocklines = self.block.to_text(dedent=dedent).splitlines(True) - - # Remove blank lines from both ends - while blocklines and not blocklines[0].strip(): - blocklines.pop(0) - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Remove leading and trailing braces - assert blocklines and blocklines[0].strip() == "{" - assert blocklines and blocklines[-1].strip() == "}" - blocklines.pop() - blocklines.pop(0) - - # Remove trailing blank lines - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - # Write the body, substituting a goto for ERROR_IF() - for line in blocklines: + assert dedent <= 0 + extra = " " * -dedent + for line in self.block_text: if m := re.match(r"(\s*)ERROR_IF$(.+), (\w+)$;\s*$", line): space, cond, label = m.groups() # ERROR_IF() must pop the inputs from the stack. @@ -215,34 +227,36 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None else: break if ninputs: - out.write_raw(f"{space}if ({cond}) goto pop_{ninputs}_{label};\n") + out.write_raw( + f"{extra}{space}if ({cond}) goto pop_{ninputs}_{label};\n" + ) else: - out.write_raw(f"{space}if ({cond}) goto {label};\n") + out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") else: - out.write_raw(line) + out.write_raw(extra + line) InstructionOrCacheEffect = Instruction | parser.CacheEffect +StackEffectMapping = list[tuple[StackEffect, StackEffect]] @dataclasses.dataclass class Component: instr: Instruction - input_mapping: dict[str, parser.StackEffect] - output_mapping: dict[str, parser.StackEffect] + input_mapping: StackEffectMapping + output_mapping: StackEffectMapping def write_body(self, out: Formatter, cache_adjust: int) -> None: with out.block(""): - for var, ieffect in self.input_mapping.items(): - out.emit(f"PyObject *{ieffect.name} = {var};") - for oeffect in self.output_mapping.values(): - out.emit(f"PyObject *{oeffect.name};") - self.instr.write_body(out, dedent=-4, cache_adjust=cache_adjust) - for var, oeffect in self.output_mapping.items(): - out.emit(f"{var} = {oeffect.name};") + for var, ieffect in self.input_mapping: + out.declare(ieffect, var) + for _, oeffect in self.output_mapping: + out.declare(oeffect, None) + self.instr.write_body(out, dedent=-4, cache_adjust=cache_adjust) -# TODO: Use a common base class for {Super,Macro}Instruction + for var, oeffect in self.output_mapping: + out.assign(var, oeffect) @dataclasses.dataclass @@ -250,7 +264,7 @@ class SuperOrMacroInstruction: """Common fields for super- and macro instructions.""" name: str - stack: list[str] + stack: list[StackEffect] initial_sp: int final_sp: int @@ -369,7 +383,11 @@ def analyze(self) -> None: def find_predictions(self) -> None: """Find the instructions that need PREDICTED() labels.""" for instr in self.instrs.values(): - for target in re.findall(RE_PREDICTED, instr.block.text): + targets = set(instr.predictions) + for line in instr.block_text: + if m := re.match(RE_PREDICTED, line): + targets.add(m.group(1)) + for target in targets: if target_instr := self.instrs.get(target): target_instr.predicted = True else: @@ -440,24 +458,9 @@ def analyze_super(self, super: parser.Super) -> SuperInstruction: stack, initial_sp = self.stack_analysis(components) sp = initial_sp parts: list[Component] = [] - for component in components: - match component: - case parser.CacheEffect() as ceffect: - parts.append(ceffect) - case Instruction() as instr: - input_mapping = {} - for ieffect in reversed(instr.input_effects): - sp -= 1 - if ieffect.name != UNUSED: - input_mapping[stack[sp]] = ieffect - output_mapping = {} - for oeffect in instr.output_effects: - if oeffect.name != UNUSED: - output_mapping[stack[sp]] = oeffect - sp += 1 - parts.append(Component(instr, input_mapping, output_mapping)) - case _: - typing.assert_never(component) + for instr in components: + part, sp = self.analyze_instruction(instr, stack, sp) + parts.append(part) final_sp = sp return SuperInstruction(super.name, stack, initial_sp, final_sp, super, parts) @@ -471,22 +474,26 @@ def analyze_macro(self, macro: parser.Macro) -> MacroInstruction: case parser.CacheEffect() as ceffect: parts.append(ceffect) case Instruction() as instr: - input_mapping = {} - for ieffect in reversed(instr.input_effects): - sp -= 1 - if ieffect.name != UNUSED: - input_mapping[stack[sp]] = ieffect - output_mapping = {} - for oeffect in instr.output_effects: - if oeffect.name != UNUSED: - output_mapping[stack[sp]] = oeffect - sp += 1 - parts.append(Component(instr, input_mapping, output_mapping)) + part, sp = self.analyze_instruction(instr, stack, sp) + parts.append(part) case _: typing.assert_never(component) final_sp = sp return MacroInstruction(macro.name, stack, initial_sp, final_sp, macro, parts) + def analyze_instruction( + self, instr: Instruction, stack: list[StackEffect], sp: int + ) -> tuple[Component, int]: + input_mapping: StackEffectMapping = [] + for ieffect in reversed(instr.input_effects): + sp -= 1 + input_mapping.append((stack[sp], ieffect)) + output_mapping: StackEffectMapping = [] + for oeffect in instr.output_effects: + output_mapping.append((stack[sp], oeffect)) + sp += 1 + return Component(instr, input_mapping, output_mapping), sp + def check_super_components(self, super: parser.Super) -> list[Instruction]: components: list[Instruction] = [] for op in super.ops: @@ -514,7 +521,7 @@ def check_macro_components( def stack_analysis( self, components: typing.Iterable[InstructionOrCacheEffect] - ) -> tuple[list[str], int]: + ) -> tuple[list[StackEffect], int]: """Analyze a super-instruction or macro. Print an error if there's a cache effect (which we don't support yet). @@ -536,7 +543,10 @@ def stack_analysis( # At this point, 'current' is the net stack effect, # and 'lowest' and 'highest' are the extremes. # Note that 'lowest' may be negative. - stack = [f"_tmp_{i+1}" for i in range(highest - lowest)] + # TODO: Reverse the numbering. + stack = [ + StackEffect(f"_tmp_{i+1}", "") for i in reversed(range(highest - lowest)) + ] return stack, -lowest def write_instructions(self) -> None: @@ -561,7 +571,9 @@ def write_instructions(self) -> None: if instr.predicted: self.out.emit(f"PREDICTED({name});") instr.write(self.out) - if not always_exits(instr.block): + if not instr.always_exits: + for prediction in instr.predictions: + self.out.emit(f"PREDICT({prediction});") self.out.emit(f"DISPATCH();") # Write and count super-instructions @@ -589,11 +601,11 @@ def write_super(self, sup: SuperInstruction) -> None: for comp in sup.parts: if not first: self.out.emit("NEXTOPARG();") - self.out.emit("next_instr++;") + self.out.emit("JUMPBY(1);") first = False comp.write_body(self.out, 0) if comp.instr.cache_offset: - self.out.emit(f"next_instr += {comp.instr.cache_offset};") + self.out.emit(f"JUMPBY({comp.instr.cache_offset});") def write_macro(self, mac: MacroInstruction) -> None: """Write code for a macro instruction.""" @@ -608,43 +620,68 @@ def write_macro(self, mac: MacroInstruction) -> None: cache_adjust += comp.instr.cache_offset if cache_adjust: - self.out.emit(f"next_instr += {cache_adjust};") + self.out.emit(f"JUMPBY({cache_adjust});") @contextlib.contextmanager def wrap_super_or_macro(self, up: SuperOrMacroInstruction): """Shared boilerplate for super- and macro instructions.""" + # TODO: Somewhere (where?) make it so that if one instruction + # has an output that is input to another, and the variable names + # and types match and don't conflict with other instructions, + # that variable is declared with the right name and type in the + # outer block, rather than trusting the compiler to optimize it. self.out.emit("") with self.out.block(f"TARGET({up.name})"): - for i, var in enumerate(up.stack): + for i, var in reversed(list(enumerate(up.stack))): + src = None if i < up.initial_sp: - self.out.emit(f"PyObject *{var} = PEEK({up.initial_sp - i});") - else: - self.out.emit(f"PyObject *{var};") + src = StackEffect(f"PEEK({up.initial_sp - i})", "") + self.out.declare(var, src) yield - if up.final_sp > up.initial_sp: - self.out.emit(f"STACK_GROW({up.final_sp - up.initial_sp});") - elif up.final_sp < up.initial_sp: - self.out.emit(f"STACK_SHRINK({up.initial_sp - up.final_sp});") + self.out.stack_adjust(up.final_sp - up.initial_sp) for i, var in enumerate(reversed(up.stack[: up.final_sp]), 1): - self.out.emit(f"POKE({i}, {var});") + dst = StackEffect(f"PEEK({i})", "") + self.out.assign(dst, var) self.out.emit(f"DISPATCH();") -def always_exits(block: parser.Block) -> bool: +def extract_block_text(block: parser.Block) -> tuple[list[str], list[str]]: + # Get lines of text with proper dedent + blocklines = block.text.splitlines(True) + + # Remove blank lines from both ends + while blocklines and not blocklines[0].strip(): + blocklines.pop(0) + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + + # Remove leading and trailing braces + assert blocklines and blocklines[0].strip() == "{" + assert blocklines and blocklines[-1].strip() == "}" + blocklines.pop() + blocklines.pop(0) + + # Remove trailing blank lines + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + + # Separate PREDICT(...) macros from end + predictions: list[str] = [] + while blocklines and (m := re.match(r"^\s*PREDICT$(\w+)$;\s*$", blocklines[-1])): + predictions.insert(0, m.group(1)) + blocklines.pop() + + return blocklines, predictions + + +def always_exits(lines: list[str]) -> bool: """Determine whether a block always ends in a return/goto/etc.""" - text = block.text - lines = text.splitlines() - while lines and not lines[-1].strip(): - lines.pop() - if not lines or lines[-1].strip() != "}": - return False - lines.pop() if not lines: return False - line = lines.pop().rstrip() + line = lines[-1].rstrip() # Indent must match exactly (TODO: Do something better) if line[:12] != " " * 12: return False diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 02a7834d2215..d802c733dfd1 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -62,7 +62,8 @@ class Block(Node): @dataclass class StackEffect(Node): name: str - # TODO: type, condition + type: str = "" + # TODO: array, condition @dataclass @@ -147,7 +148,7 @@ def inst_header(self) -> InstHeader | None: if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)): name = tkn.text if self.expect(lx.COMMA): - inp, outp = self.stack_effect() + inp, outp = self.io_effect() if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: return InstHeader(kind, name, inp, outp) @@ -156,7 +157,7 @@ def inst_header(self) -> InstHeader | None: return InstHeader(kind, name, [], []) return None - def stack_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: + def io_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: # '(' [inputs] '--' [outputs] ')' if self.expect(lx.LPAREN): inputs = self.inputs() or [] @@ -181,23 +182,7 @@ def inputs(self) -> list[InputEffect] | None: @contextual def input(self) -> InputEffect | None: - # IDENTIFIER '/' INTEGER (CacheEffect) - # IDENTIFIER (StackEffect) - if tkn := self.expect(lx.IDENTIFIER): - if self.expect(lx.DIVIDE): - if num := self.expect(lx.NUMBER): - try: - size = int(num.text) - except ValueError: - raise self.make_syntax_error( - f"Expected integer, got {num.text!r}" - ) - else: - return CacheEffect(tkn.text, size) - raise self.make_syntax_error("Expected integer") - else: - # TODO: Arrays, conditions - return StackEffect(tkn.text) + return self.cache_effect() or self.stack_effect() def outputs(self) -> list[OutputEffect] | None: # output (, output)* @@ -214,8 +199,30 @@ def outputs(self) -> list[OutputEffect] | None: @contextual def output(self) -> OutputEffect | None: + return self.stack_effect() + + @contextual + def cache_effect(self) -> CacheEffect | None: + # IDENTIFIER '/' NUMBER + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.DIVIDE): + num = self.require(lx.NUMBER).text + try: + size = int(num) + except ValueError: + raise self.make_syntax_error(f"Expected integer, got {num!r}") + else: + return CacheEffect(tkn.text, size) + + @contextual + def stack_effect(self) -> StackEffect | None: + # IDENTIFIER [':' IDENTIFIER] + # TODO: Arrays, conditions if tkn := self.expect(lx.IDENTIFIER): - return StackEffect(tkn.text) + type = "" + if self.expect(lx.COLON): + type = self.require(lx.IDENTIFIER).text + return StackEffect(tkn.text, type) @contextual def super_def(self) -> Super | None: From webhook-mailer at python.org Thu Dec 8 17:38:12 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Thu, 08 Dec 2022 22:38:12 -0000 Subject: [Python-checkins] gh-81057: Move OS-Related Globals to _PyRuntimeState (gh-100082) Message-ID: <mailman.2868.1670539093.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cda9f0236fd7800c6db5eec207668c4bfec4a45d commit: cda9f0236fd7800c6db5eec207668c4bfec4a45d branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T15:38:06-07:00 summary: gh-81057: Move OS-Related Globals to _PyRuntimeState (gh-100082) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_os.h M Include/internal/pycore_global_objects_fini_generated.h M Include/internal/pycore_global_strings.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Include/internal/pycore_runtime_init_generated.h M Include/internal/pycore_unicodeobject_generated.h M Lib/test/test_os.py M Makefile.pre.in M Modules/posixmodule.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9951fa9951e6..6aba2f19ebde 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1051,6 +1051,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(node_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ns)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nstype)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(null)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(number)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(obj)); @@ -1089,6 +1090,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(posix)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 12144b02f455..acb9a4fbb92d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -537,6 +537,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(node_offset) STRUCT_FOR_ID(ns) STRUCT_FOR_ID(nstype) + STRUCT_FOR_ID(nt) STRUCT_FOR_ID(null) STRUCT_FOR_ID(number) STRUCT_FOR_ID(obj) @@ -575,6 +576,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(pos) STRUCT_FOR_ID(pos1) STRUCT_FOR_ID(pos2) + STRUCT_FOR_ID(posix) STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) diff --git a/Include/internal/pycore_os.h b/Include/internal/pycore_os.h new file mode 100644 index 000000000000..e4899bd5032d --- /dev/null +++ b/Include/internal/pycore_os.h @@ -0,0 +1,38 @@ +#ifndef Py_INTERNAL_OS_H +#define Py_INTERNAL_OS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#ifndef MS_WINDOWS +#define _OS_NEED_TICKS_PER_SECOND +# define need_ticks_per_second_STATE \ + long ticks_per_second; +# define need_ticks_per_second_INIT \ + .ticks_per_second = -1, +#else +# define need_ticks_per_second_STATE +# define need_ticks_per_second_INIT +#endif /* MS_WINDOWS */ + + +struct _os_runtime_state { + int _not_used; + need_ticks_per_second_STATE +}; +# define _OS_RUNTIME_INIT \ + { \ + ._not_used = 0, \ + need_ticks_per_second_INIT \ + } + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OS_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0720e2ed4422..c6b48cae7760 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_os.h" // struct _os_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -103,6 +104,7 @@ typedef struct pyruntimestate { * KeyboardInterrupt exception, suggesting the user pressed ^C. */ int unhandled_keyboard_interrupt; } signals; + struct _os_runtime_state os; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ab53876e355f..70263892e570 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -26,6 +26,7 @@ extern "C" { }, \ .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ .pyhash_state = pyhash_state_INIT, \ + .os = _OS_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 87b0f2ed8dfa..6d1b8702c776 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1043,6 +1043,7 @@ extern "C" { INIT_ID(node_offset), \ INIT_ID(ns), \ INIT_ID(nstype), \ + INIT_ID(nt), \ INIT_ID(null), \ INIT_ID(number), \ INIT_ID(obj), \ @@ -1081,6 +1082,7 @@ extern "C" { INIT_ID(pos), \ INIT_ID(pos1), \ INIT_ID(pos2), \ + INIT_ID(posix), \ INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 80be342b5b3b..7f407c0141b8 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -980,6 +980,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(nstype); PyUnicode_InternInPlace(&string); + string = &_Py_ID(nt); + PyUnicode_InternInPlace(&string); string = &_Py_ID(null); PyUnicode_InternInPlace(&string); string = &_Py_ID(number); @@ -1056,6 +1058,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(pos2); PyUnicode_InternInPlace(&string); + string = &_Py_ID(posix); + PyUnicode_InternInPlace(&string); string = &_Py_ID(print_file_and_line); PyUnicode_InternInPlace(&string); string = &_Py_ID(priority); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 94db8bb7737a..e0577916428a 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -606,12 +606,13 @@ def test_stat_attributes_bytes(self): def test_stat_result_pickle(self): result = os.stat(self.fname) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - p = pickle.dumps(result, proto) - self.assertIn(b'stat_result', p) - if proto < 4: - self.assertIn(b'cos\nstat_result\n', p) - unpickled = pickle.loads(p) - self.assertEqual(result, unpickled) + with self.subTest(f'protocol {proto}'): + p = pickle.dumps(result, proto) + self.assertIn(b'stat_result', p) + if proto < 4: + self.assertIn(b'cos\nstat_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index f6df7a620dea..80144e528875 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1654,6 +1654,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ + $(srcdir)/Include/internal/pycore_os.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7fc8aef9b303..cbf4d5b3fcec 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -495,9 +495,11 @@ extern char *ctermid_r(char *); #ifdef MS_WINDOWS # define INITFUNC PyInit_nt # define MODNAME "nt" +# define MODNAME_OBJ &_Py_ID(nt) #else # define INITFUNC PyInit_posix # define MODNAME "posix" +# define MODNAME_OBJ &_Py_ID(posix) #endif #if defined(__sun) @@ -974,6 +976,7 @@ typedef struct { #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) PyObject *SchedParamType; #endif + newfunc statresult_new_orig; PyObject *StatResultType; PyObject *StatVFSResultType; PyObject *TerminalSizeType; @@ -2225,7 +2228,6 @@ static PyStructSequence_Desc waitid_result_desc = { 5 }; #endif -static newfunc structseq_new; static PyObject * statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -2233,6 +2235,18 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyStructSequence *result; int i; + // ht_module doesn't get set in PyStructSequence_NewType(), + // so we can't use PyType_GetModule(). + PyObject *mod = PyImport_GetModule(MODNAME_OBJ); + if (mod == NULL) { + return NULL; + } + _posixstate *state = get_posix_state(mod); + if (state == NULL) { + return NULL; + } +#define structseq_new state->statresult_new_orig + result = (PyStructSequence*)structseq_new(type, args, kwds); if (!result) return NULL; @@ -9051,10 +9065,23 @@ build_times_result(PyObject *module, double user, double system, } -#ifndef MS_WINDOWS -#define NEED_TICKS_PER_SECOND -static long ticks_per_second = -1; -#endif /* MS_WINDOWS */ +#ifdef _OS_NEED_TICKS_PER_SECOND +#define ticks_per_second _PyRuntime.os.ticks_per_second +static void +ticks_per_second_init(void) +{ + if (ticks_per_second != -1) { + return; + } +# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) + ticks_per_second = sysconf(_SC_CLK_TCK); +# elif defined(HZ) + ticks_per_second = HZ; +# else + ticks_per_second = 60; /* magic fallback value; may be bogus */ +# endif +} +#endif /*[clinic input] os.times @@ -9089,10 +9116,10 @@ os_times_impl(PyObject *module) (double)0, (double)0); } +#elif !defined(_OS_NEED_TICKS_PER_SECOND) +# error "missing ticks_per_second" #else /* MS_WINDOWS */ { - - struct tms t; clock_t c; errno = 0; @@ -15912,7 +15939,7 @@ posixmodule_exec(PyObject *m) } PyModule_AddObject(m, "stat_result", Py_NewRef(StatResultType)); state->StatResultType = StatResultType; - structseq_new = ((PyTypeObject *)StatResultType)->tp_new; + state->statresult_new_orig = ((PyTypeObject *)StatResultType)->tp_new; ((PyTypeObject *)StatResultType)->tp_new = statresult_new; statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */ @@ -15922,14 +15949,9 @@ posixmodule_exec(PyObject *m) } PyModule_AddObject(m, "statvfs_result", Py_NewRef(StatVFSResultType)); state->StatVFSResultType = StatVFSResultType; -#ifdef NEED_TICKS_PER_SECOND -# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); -# elif defined(HZ) - ticks_per_second = HZ; -# else - ticks_per_second = 60; /* magic fallback value; may be bogus */ -# endif + +#ifdef _OS_NEED_TICKS_PER_SECOND + ticks_per_second_init(); #endif #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f62434370cfd..fa3924968a6b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -236,6 +236,7 @@ <ClInclude Include="..\Include\internal\pycore_object.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h" /> + <ClInclude Include="..\Include\internal\pycore_os.h" /> <ClInclude Include="..\Include\internal\pycore_pathconfig.h" /> <ClInclude Include="..\Include\internal\pycore_pyarena.h" /> <ClInclude Include="..\Include\internal\pycore_pyerrors.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f44a1ad8550a..e29c6b2330d1 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -612,6 +612,9 @@ <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_os.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_pathconfig.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 5c8164517f13..8e05bc3e4f23 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -390,9 +390,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/posixmodule.c os_dup2_impl dup3_works - -Modules/posixmodule.c - structseq_new - -Modules/posixmodule.c - ticks_per_second - Modules/timemodule.c _PyTime_GetClockWithInfo initialized - Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 257823574fa8..814c55b75c09 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -11,14 +11,18 @@ filename funcname name reason # These are effectively const. ##----------------------- -## process-global resources - set during first init +## process-global resources ## indicators for resource availability/capability +# (set during first init) Python/bootstrap_hash.c py_getrandom getrandom_works - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c set_inheritable ioctl_works - +# (set lazily, *after* first init) +# XXX Is this thread-safe? +Modules/posixmodule.c os_dup2_impl dup3_works - -## resource init +## resource init - set during first init Python/thread.c - initialized - Python/thread_pthread.h - condattr_monotonic - # safe static buffer used during one-time initialization From webhook-mailer at python.org Thu Dec 8 18:46:14 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Thu, 08 Dec 2022 23:46:14 -0000 Subject: [Python-checkins] gh-81057: Move time Globals to _PyRuntimeState (gh-100122) Message-ID: <mailman.2869.1670543176.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8a3f06c54b52e5e7490a131b261384baac9d6090 commit: 8a3f06c54b52e5e7490a131b261384baac9d6090 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T16:46:09-07:00 summary: gh-81057: Move time Globals to _PyRuntimeState (gh-100122) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_time.h D Include/internal/pycore_os.h M Include/internal/pycore_pylifecycle.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M Modules/posixmodule.c M Modules/timemodule.c M PCbuild/_freeze_module.vcxproj M PCbuild/_freeze_module.vcxproj.filters M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/pylifecycle.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_os.h b/Include/internal/pycore_os.h deleted file mode 100644 index e4899bd5032d..000000000000 --- a/Include/internal/pycore_os.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef Py_INTERNAL_OS_H -#define Py_INTERNAL_OS_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - - -#ifndef MS_WINDOWS -#define _OS_NEED_TICKS_PER_SECOND -# define need_ticks_per_second_STATE \ - long ticks_per_second; -# define need_ticks_per_second_INIT \ - .ticks_per_second = -1, -#else -# define need_ticks_per_second_STATE -# define need_ticks_per_second_INIT -#endif /* MS_WINDOWS */ - - -struct _os_runtime_state { - int _not_used; - need_ticks_per_second_STATE -}; -# define _OS_RUNTIME_INIT \ - { \ - ._not_used = 0, \ - need_ticks_per_second_INIT \ - } - - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_OS_H */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 4c0ffa7a9b1a..370e4cbd59f9 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -44,6 +44,7 @@ extern void _PySys_Fini(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); +extern PyStatus _PyTime_Init(void); extern PyStatus _PyImportZip_Init(PyThreadState *tstate); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c6b48cae7760..458493eac70f 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -21,7 +21,7 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state -#include "pycore_os.h" // struct _os_runtime_state +#include "pycore_time.h" // struct _time_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -104,7 +104,7 @@ typedef struct pyruntimestate { * KeyboardInterrupt exception, suggesting the user pressed ^C. */ int unhandled_keyboard_interrupt; } signals; - struct _os_runtime_state os; + struct _time_runtime_state time; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 70263892e570..ab53876e355f 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -26,7 +26,6 @@ extern "C" { }, \ .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ .pyhash_state = pyhash_state_INIT, \ - .os = _OS_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h new file mode 100644 index 000000000000..949170c44937 --- /dev/null +++ b/Include/internal/pycore_time.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_TIME_H +#define Py_INTERNAL_TIME_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +struct _time_runtime_state { +#ifdef HAVE_TIMES + int ticks_per_second_initialized; + long ticks_per_second; +#else + int _not_used; +#endif +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TIME_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 80144e528875..c21826fbb94f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1654,7 +1654,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_os.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ @@ -1673,6 +1672,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tuple.h \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cbf4d5b3fcec..f5175350e12a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9065,24 +9065,6 @@ build_times_result(PyObject *module, double user, double system, } -#ifdef _OS_NEED_TICKS_PER_SECOND -#define ticks_per_second _PyRuntime.os.ticks_per_second -static void -ticks_per_second_init(void) -{ - if (ticks_per_second != -1) { - return; - } -# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); -# elif defined(HZ) - ticks_per_second = HZ; -# else - ticks_per_second = 60; /* magic fallback value; may be bogus */ -# endif -} -#endif - /*[clinic input] os.times @@ -9116,22 +9098,24 @@ os_times_impl(PyObject *module) (double)0, (double)0); } -#elif !defined(_OS_NEED_TICKS_PER_SECOND) -# error "missing ticks_per_second" #else /* MS_WINDOWS */ { struct tms t; clock_t c; errno = 0; c = times(&t); - if (c == (clock_t) -1) + if (c == (clock_t) -1) { return posix_error(); + } + assert(_PyRuntime.time.ticks_per_second_initialized); +#define ticks_per_second _PyRuntime.time.ticks_per_second return build_times_result(module, (double)t.tms_utime / ticks_per_second, (double)t.tms_stime / ticks_per_second, (double)t.tms_cutime / ticks_per_second, (double)t.tms_cstime / ticks_per_second, (double)c / ticks_per_second); +#undef ticks_per_second } #endif /* MS_WINDOWS */ #endif /* HAVE_TIMES */ @@ -15950,10 +15934,6 @@ posixmodule_exec(PyObject *m) PyModule_AddObject(m, "statvfs_result", Py_NewRef(StatVFSResultType)); state->StatVFSResultType = StatVFSResultType; -#ifdef _OS_NEED_TICKS_PER_SECOND - ticks_per_second_init(); -#endif - #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) sched_param_desc.name = MODNAME ".sched_param"; PyObject *SchedParamType = (PyObject *)PyStructSequence_NewType(&sched_param_desc); diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 11c888af03e8..ba4128c0fdf5 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -62,6 +62,54 @@ #define SEC_TO_NS (1000 * 1000 * 1000) +#ifdef HAVE_TIMES + +static int +check_ticks_per_second(long tps, const char *context) +{ + /* Effectively, check that _PyTime_MulDiv(t, SEC_TO_NS, ticks_per_second) + cannot overflow. */ + if (tps >= 0 && (_PyTime_t)tps > _PyTime_MAX / SEC_TO_NS) { + PyErr_Format(PyExc_OverflowError, "%s is too large", context); + return -1; + } + return 0; +} + +# define ticks_per_second _PyRuntime.time.ticks_per_second + +static void +ensure_ticks_per_second(void) +{ + if (_PyRuntime.time.ticks_per_second_initialized) { + return; + } + _PyRuntime.time.ticks_per_second_initialized = 1; +# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) + ticks_per_second = sysconf(_SC_CLK_TCK); + if (ticks_per_second < 1) { + ticks_per_second = -1; + } +# elif defined(HZ) + ticks_per_second = HZ; +# else + ticks_per_second = 60; /* magic fallback value; may be bogus */ +# endif +} + +#endif /* HAVE_TIMES */ + + +PyStatus +_PyTime_Init(void) +{ +#ifdef HAVE_TIMES + ensure_ticks_per_second(); +#endif + return PyStatus_Ok(); +} + + /* Forward declarations */ static int pysleep(_PyTime_t timeout); @@ -140,18 +188,8 @@ Return the current time in nanoseconds since the Epoch."); static int _PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) { - static int initialized = 0; - - if (!initialized) { - initialized = 1; - - /* Make sure that _PyTime_MulDiv(ticks, SEC_TO_NS, CLOCKS_PER_SEC) - above cannot overflow */ - if ((_PyTime_t)CLOCKS_PER_SEC > _PyTime_MAX / SEC_TO_NS) { - PyErr_SetString(PyExc_OverflowError, - "CLOCKS_PER_SEC is too large"); - return -1; - } + if (check_ticks_per_second(CLOCKS_PER_SEC, "CLOCKS_PER_SEC") < 0) { + return -1; } if (info) { @@ -1308,36 +1346,10 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) struct tms t; if (times(&t) != (clock_t)-1) { - static long ticks_per_second = -1; - - if (ticks_per_second == -1) { - long freq; -#if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - freq = sysconf(_SC_CLK_TCK); - if (freq < 1) { - freq = -1; - } -#elif defined(HZ) - freq = HZ; -#else - freq = 60; /* magic fallback value; may be bogus */ -#endif - - if (freq != -1) { - /* check that _PyTime_MulDiv(t, SEC_TO_NS, ticks_per_second) - cannot overflow below */ -#if LONG_MAX > _PyTime_MAX / SEC_TO_NS - if ((_PyTime_t)freq > _PyTime_MAX / SEC_TO_NS) { - PyErr_SetString(PyExc_OverflowError, - "_SC_CLK_TCK is too large"); - return -1; - } -#endif - - ticks_per_second = freq; - } + assert(_PyRuntime.time.ticks_per_second_initialized); + if (check_ticks_per_second(ticks_per_second, "_SC_CLK_TCK") < 0) { + return -1; } - if (ticks_per_second != -1) { if (info) { info->implementation = "times()"; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 8454bd67b1db..fce1f6705100 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -110,6 +110,7 @@ <ClCompile Include="..\Modules\getpath_noop.c" /> <ClCompile Include="..\Modules\posixmodule.c" /> <ClCompile Include="..\Modules\signalmodule.c" /> + <ClCompile Include="..\Modules\timemodule.c" /> <ClCompile Include="..\Modules\_tracemalloc.c" /> <ClCompile Include="..\Modules\_io\_iomodule.c" /> <ClCompile Include="..\Modules\_io\bufferedio.c" /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 6e8498dceb1c..dce6278987c5 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -367,6 +367,9 @@ <ClCompile Include="..\Python\thread.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\timemodule.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Parser\token.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index fa3924968a6b..f9f0c191b5eb 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -236,7 +236,6 @@ <ClInclude Include="..\Include\internal\pycore_object.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc.h" /> <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h" /> - <ClInclude Include="..\Include\internal\pycore_os.h" /> <ClInclude Include="..\Include\internal\pycore_pathconfig.h" /> <ClInclude Include="..\Include\internal\pycore_pyarena.h" /> <ClInclude Include="..\Include\internal\pycore_pyerrors.h" /> @@ -255,6 +254,7 @@ <ClInclude Include="..\Include\internal\pycore_structseq.h" /> <ClInclude Include="..\Include\internal\pycore_sysmodule.h" /> <ClInclude Include="..\Include\internal\pycore_symtable.h" /> + <ClInclude Include="..\Include\internal\pycore_time.h" /> <ClInclude Include="..\Include\internal\pycore_token.h" /> <ClInclude Include="..\Include\internal\pycore_traceback.h" /> <ClInclude Include="..\Include\internal\pycore_tuple.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index e29c6b2330d1..e683650ad7da 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -612,9 +612,6 @@ <ClInclude Include="..\Include\internal\pycore_obmalloc_init.h"> <Filter>Include\internal</Filter> </ClInclude> - <ClInclude Include="..\Include\internal\pycore_os.h"> - <Filter>Include\internal</Filter> - </ClInclude> <ClInclude Include="..\Include\internal\pycore_pathconfig.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -666,6 +663,9 @@ <ClInclude Include="..\Include\internal\pycore_symtable.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_time.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_token.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8209132ebc6c..e13660f51138 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -606,6 +606,11 @@ pycore_init_runtime(_PyRuntimeState *runtime, return status; } + status = _PyTime_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _PyImport_Init(); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 8e05bc3e4f23..3f6e1c740535 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -387,12 +387,6 @@ Modules/faulthandler.c - old_stack - ################################## ## global non-objects to fix in builtin modules -##----------------------- -## initialized once - -Modules/timemodule.c _PyTime_GetClockWithInfo initialized - -Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - - ##----------------------- ## state From webhook-mailer at python.org Thu Dec 8 18:54:14 2022 From: webhook-mailer at python.org (gvanrossum) Date: Thu, 08 Dec 2022 23:54:14 -0000 Subject: [Python-checkins] GH-98831: Generate things in the input order (#100123) Message-ID: <mailman.2870.1670543655.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1cfa704f64193701e400a77d2287f3526ff026f8 commit: 1cfa704f64193701e400a77d2287f3526ff026f8 branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-08T15:54:07-08:00 summary: GH-98831: Generate things in the input order (#100123) This makes it easier to see what changed in the generated code when converting an instruction to super or macro. files: M Python/generated_cases.c.h M Tools/cases_generator/generate_cases.py diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 510b6c4a75b8..76eb6085ec57 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -63,6 +63,115 @@ DISPATCH(); } + TARGET(LOAD_FAST__LOAD_FAST) { + PyObject *_tmp_1; + PyObject *_tmp_2; + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_2 = value; + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_1 = value; + } + STACK_GROW(2); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + + TARGET(LOAD_FAST__LOAD_CONST) { + PyObject *_tmp_1; + PyObject *_tmp_2; + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_2 = value; + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETITEM(consts, oparg); + Py_INCREF(value); + _tmp_1 = value; + } + STACK_GROW(2); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + + TARGET(STORE_FAST__LOAD_FAST) { + PyObject *_tmp_1 = PEEK(1); + { + PyObject *value = _tmp_1; + SETLOCAL(oparg, value); + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_1 = value; + } + POKE(1, _tmp_1); + DISPATCH(); + } + + TARGET(STORE_FAST__STORE_FAST) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *value = _tmp_1; + SETLOCAL(oparg, value); + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value = _tmp_2; + SETLOCAL(oparg, value); + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(LOAD_CONST__LOAD_FAST) { + PyObject *_tmp_1; + PyObject *_tmp_2; + { + PyObject *value; + value = GETITEM(consts, oparg); + Py_INCREF(value); + _tmp_2 = value; + } + NEXTOPARG(); + JUMPBY(1); + { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + _tmp_1 = value; + } + STACK_GROW(2); + POKE(1, _tmp_1); + POKE(2, _tmp_2); + DISPATCH(); + } + TARGET(POP_TOP) { PyObject *value = PEEK(1); Py_DECREF(value); @@ -78,6 +187,21 @@ DISPATCH(); } + TARGET(END_FOR) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *value = _tmp_1; + Py_DECREF(value); + } + { + PyObject *value = _tmp_2; + Py_DECREF(value); + } + STACK_SHRINK(2); + DISPATCH(); + } + TARGET(UNARY_POSITIVE) { PyObject *value = PEEK(1); PyObject *res; @@ -2078,6 +2202,119 @@ DISPATCH(); } + TARGET(COMPARE_OP_FLOAT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); + DEOPT_IF(isnan(dleft), COMPARE_OP); + DEOPT_IF(isnan(dright), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_INT_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t when_to_jump_mask = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); + Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; + Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; + // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask + int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + jump = sign_ish & when_to_jump_mask; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(COMPARE_OP_STR_JUMP) { + PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_2 = PEEK(2); + { + PyObject *right = _tmp_1; + PyObject *left = _tmp_2; + size_t jump; + uint16_t invert = read_u16(next_instr + 1); + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int res = _PyUnicode_Equal(left, right); + assert(oparg == Py_EQ || oparg == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(res == 0 || res == 1); + assert(invert == 0 || invert == 1); + jump = res ^ invert; + _tmp_2 = (PyObject *)jump; + } + JUMPBY(2); + NEXTOPARG(); + JUMPBY(1); + { + size_t jump = (size_t)_tmp_2; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + if (jump) { + JUMPBY(oparg); + } + } + STACK_SHRINK(2); + DISPATCH(); + } + TARGET(IS_OP) { PyObject *right = POP(); PyObject *left = TOP(); @@ -3593,240 +3830,3 @@ TARGET(CACHE) { Py_UNREACHABLE(); } - - TARGET(LOAD_FAST__LOAD_FAST) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_2 = value; - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_1 = value; - } - STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); - DISPATCH(); - } - - TARGET(LOAD_FAST__LOAD_CONST) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_2 = value; - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETITEM(consts, oparg); - Py_INCREF(value); - _tmp_1 = value; - } - STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); - DISPATCH(); - } - - TARGET(STORE_FAST__LOAD_FAST) { - PyObject *_tmp_1 = PEEK(1); - { - PyObject *value = _tmp_1; - SETLOCAL(oparg, value); - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_1 = value; - } - POKE(1, _tmp_1); - DISPATCH(); - } - - TARGET(STORE_FAST__STORE_FAST) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *value = _tmp_1; - SETLOCAL(oparg, value); - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value = _tmp_2; - SETLOCAL(oparg, value); - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(LOAD_CONST__LOAD_FAST) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - PyObject *value; - value = GETITEM(consts, oparg); - Py_INCREF(value); - _tmp_2 = value; - } - NEXTOPARG(); - JUMPBY(1); - { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - _tmp_1 = value; - } - STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); - DISPATCH(); - } - - TARGET(COMPARE_OP_FLOAT_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - jump = sign_ish & when_to_jump_mask; - _tmp_2 = (PyObject *)jump; - } - JUMPBY(2); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(COMPARE_OP_INT_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); - Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; - Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - jump = sign_ish & when_to_jump_mask; - _tmp_2 = (PyObject *)jump; - } - JUMPBY(2); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(COMPARE_OP_STR_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - uint16_t invert = read_u16(next_instr + 1); - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int res = _PyUnicode_Equal(left, right); - assert(oparg == Py_EQ || oparg == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(res == 0 || res == 1); - assert(invert == 0 || invert == 1); - jump = res ^ invert; - _tmp_2 = (PyObject *)jump; - } - JUMPBY(2); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(END_FOR) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *value = _tmp_1; - Py_DECREF(value); - } - { - PyObject *value = _tmp_2; - Py_DECREF(value); - } - STACK_SHRINK(2); - DISPATCH(); - } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 844272291245..5930c797b8a4 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -311,6 +311,7 @@ def error(self, msg: str, node: parser.Node) -> None: print(f"{self.filename}:{lineno}: {msg}", file=sys.stderr) self.errors += 1 + everything: list[parser.InstDef | parser.Super | parser.Macro] instrs: dict[str, Instruction] # Includes ops supers: dict[str, parser.Super] super_instrs: dict[str, SuperInstruction] @@ -344,6 +345,7 @@ def parse(self) -> None: # Parse from start psr.setpos(start) + self.everything = [] self.instrs = {} self.supers = {} self.macros = {} @@ -352,10 +354,13 @@ def parse(self) -> None: match thing: case parser.InstDef(name=name): self.instrs[name] = Instruction(thing) + self.everything.append(thing) case parser.Super(name): self.supers[name] = thing + self.everything.append(thing) case parser.Macro(name): self.macros[name] = thing + self.everything.append(thing) case parser.Family(name): self.families[name] = thing case _: @@ -560,33 +565,24 @@ def write_instructions(self) -> None: # Create formatter; the rest of the code uses this. self.out = Formatter(f, 8) - # Write and count regular instructions + # Write and count instructions of all kinds n_instrs = 0 - for name, instr in self.instrs.items(): - if instr.kind != "inst": - continue # ops are not real instructions - n_instrs += 1 - self.out.emit("") - with self.out.block(f"TARGET({name})"): - if instr.predicted: - self.out.emit(f"PREDICTED({name});") - instr.write(self.out) - if not instr.always_exits: - for prediction in instr.predictions: - self.out.emit(f"PREDICT({prediction});") - self.out.emit(f"DISPATCH();") - - # Write and count super-instructions n_supers = 0 - for sup in self.super_instrs.values(): - n_supers += 1 - self.write_super(sup) - - # Write and count macro instructions n_macros = 0 - for macro in self.macro_instrs.values(): - n_macros += 1 - self.write_macro(macro) + for thing in self.everything: + match thing: + case parser.InstDef(): + if thing.kind == "inst": + n_instrs += 1 + self.write_instr(self.instrs[thing.name]) + case parser.Super(): + n_supers += 1 + self.write_super(self.super_instrs[thing.name]) + case parser.Macro(): + n_macros += 1 + self.write_macro(self.macro_instrs[thing.name]) + case _: + typing.assert_never(thing) print( f"Wrote {n_instrs} instructions, {n_supers} supers, " @@ -594,6 +590,18 @@ def write_instructions(self) -> None: file=sys.stderr, ) + def write_instr(self, instr: Instruction) -> None: + name = instr.name + self.out.emit("") + with self.out.block(f"TARGET({name})"): + if instr.predicted: + self.out.emit(f"PREDICTED({name});") + instr.write(self.out) + if not instr.always_exits: + for prediction in instr.predictions: + self.out.emit(f"PREDICT({prediction});") + self.out.emit(f"DISPATCH();") + def write_super(self, sup: SuperInstruction) -> None: """Write code for a super-instruction.""" with self.wrap_super_or_macro(sup): From webhook-mailer at python.org Thu Dec 8 19:17:26 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 00:17:26 -0000 Subject: [Python-checkins] gh-81057: Move Ceval Trampoline Globals to _PyRuntimeState (gh-100083) Message-ID: <mailman.2871.1670545046.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bc8cdf8c3d58f5d28c9e70c72eaae52c6d13f961 commit: bc8cdf8c3d58f5d28c9e70c72eaae52c6d13f961 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T17:17:20-07:00 summary: gh-81057: Move Ceval Trampoline Globals to _PyRuntimeState (gh-100083) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_ceval_state.h M Include/internal/pycore_interp.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/perf_trampoline.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h new file mode 100644 index 000000000000..9ba42eb03b26 --- /dev/null +++ b/Include/internal/pycore_ceval_state.h @@ -0,0 +1,100 @@ +#ifndef Py_INTERNAL_CEVAL_STATE_H +#define Py_INTERNAL_CEVAL_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_gil.h" // struct _gil_runtime_state + + +typedef enum { + PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state + PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized + PERF_STATUS_OK = 1, // Perf trampoline is ready to be executed +} perf_status_t; + + +#ifdef PY_HAVE_PERF_TRAMPOLINE +struct code_arena_st; + +struct trampoline_api_st { + void* (*init_state)(void); + void (*write_state)(void* state, const void *code_addr, + unsigned int code_size, PyCodeObject* code); + int (*free_state)(void* state); + void *state; +}; +#endif + +struct _ceval_runtime_state { + struct { +#ifdef PY_HAVE_PERF_TRAMPOLINE + perf_status_t status; + Py_ssize_t extra_code_index; + struct code_arena_st *code_arena; + struct trampoline_api_st trampoline_api; + FILE *map_file; +#else + int _not_used; +#endif + } perf; + /* Request for checking signals. It is shared by all interpreters (see + bpo-40513). Any thread of any interpreter can receive a signal, but only + the main thread of the main interpreter can handle signals: see + _Py_ThreadCanHandleSignals(). */ + _Py_atomic_int signals_pending; + struct _gil_runtime_state gil; +}; + +#ifdef PY_HAVE_PERF_TRAMPOLINE +# define _PyEval_RUNTIME_PERF_INIT \ + { \ + .status = PERF_STATUS_NO_INIT, \ + .extra_code_index = -1, \ + } +#else +# define _PyEval_RUNTIME_PERF_INIT {0} +#endif + + +struct _pending_calls { + int busy; + PyThread_type_lock lock; + /* Request for running pending calls. */ + _Py_atomic_int calls_to_do; + /* Request for looking at the `async_exc` field of the current + thread state. + Guarded by the GIL. */ + int async_exc; +#define NPENDINGCALLS 32 + struct { + int (*func)(void *); + void *arg; + } calls[NPENDINGCALLS]; + int first; + int last; +}; + +struct _ceval_state { + int recursion_limit; + /* This single variable consolidates all requests to break out of + the fast path in the eval loop. */ + _Py_atomic_int eval_breaker; + /* Request for dropping the GIL */ + _Py_atomic_int gil_drop_request; + /* The GC is ready to be executed */ + _Py_atomic_int gc_scheduled; + struct _pending_calls pending; +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CEVAL_STATE_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b5c46773c90f..ffda1351952d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -12,6 +12,7 @@ extern "C" { #include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ast_state.h" // struct ast_state +#include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache #include "pycore_context.h" // struct _Py_context_state #include "pycore_dict_state.h" // struct _Py_dict_state @@ -28,37 +29,6 @@ extern "C" { #include "pycore_warnings.h" // struct _warnings_runtime_state -struct _pending_calls { - int busy; - PyThread_type_lock lock; - /* Request for running pending calls. */ - _Py_atomic_int calls_to_do; - /* Request for looking at the `async_exc` field of the current - thread state. - Guarded by the GIL. */ - int async_exc; -#define NPENDINGCALLS 32 - struct { - int (*func)(void *); - void *arg; - } calls[NPENDINGCALLS]; - int first; - int last; -}; - -struct _ceval_state { - int recursion_limit; - /* This single variable consolidates all requests to break out of - the fast path in the eval loop. */ - _Py_atomic_int eval_breaker; - /* Request for dropping the GIL */ - _Py_atomic_int gil_drop_request; - /* The GC is ready to be executed */ - _Py_atomic_int gc_scheduled; - struct _pending_calls pending; -}; - - // atexit state typedef struct { PyObject *func; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 458493eac70f..ac89c7d378e2 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,11 +9,11 @@ extern "C" { #endif #include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_ceval_state.h" // struct _ceval_runtime_state #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_dtoa.h" // struct _dtoa_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_function.h" // struct _func_runtime_state -#include "pycore_gil.h" // struct _gil_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState @@ -31,15 +31,6 @@ struct _getargs_runtime_state { /* ceval state */ -struct _ceval_runtime_state { - /* Request for checking signals. It is shared by all interpreters (see - bpo-40513). Any thread of any interpreter can receive a signal, but only - the main thread of the main interpreter can handle signals: see - _Py_ThreadCanHandleSignals(). */ - _Py_atomic_int signals_pending; - struct _gil_runtime_state gil; -}; - /* GIL state */ struct _gilstate_runtime_state { diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ab53876e355f..b569e5833f1d 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -41,6 +41,9 @@ extern "C" { .header = 1, \ }, \ }, \ + .ceval = { \ + .perf = _PyEval_RUNTIME_PERF_INIT, \ + }, \ .gilstate = { \ .check_enabled = 1, \ /* A TSS key must be initialized with Py_tss_NEEDS_INIT \ diff --git a/Makefile.pre.in b/Makefile.pre.in index c21826fbb94f..312031999df0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1623,6 +1623,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_bytesobject.h \ $(srcdir)/Include/internal/pycore_call.h \ $(srcdir)/Include/internal/pycore_ceval.h \ + $(srcdir)/Include/internal/pycore_ceval_state.h \ $(srcdir)/Include/internal/pycore_code.h \ $(srcdir)/Include/internal/pycore_compile.h \ $(srcdir)/Include/internal/pycore_condvar.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f9f0c191b5eb..58ea6ae7b4aa 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -204,6 +204,7 @@ <ClInclude Include="..\Include\internal\pycore_bytesobject.h" /> <ClInclude Include="..\Include\internal\pycore_call.h" /> <ClInclude Include="..\Include\internal\pycore_ceval.h" /> + <ClInclude Include="..\Include\internal\pycore_ceval_state.h" /> <ClInclude Include="..\Include\internal\pycore_code.h" /> <ClInclude Include="..\Include\internal\pycore_compile.h" /> <ClInclude Include="..\Include\internal\pycore_condvar.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index e683650ad7da..96c3bd6c30e9 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -519,6 +519,9 @@ <ClInclude Include="..\Include\internal\pycore_ceval.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_ceval_state.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_code.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 161e0ef74cf1..1957ab82c339 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -134,11 +134,6 @@ any DWARF information available for them). #include "pycore_frame.h" #include "pycore_interp.h" -typedef enum { - PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state - PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized - PERF_STATUS_OK = 1, // Perf trampoline is ready to be executed -} perf_status_t; #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -190,24 +185,13 @@ struct code_arena_st { }; typedef struct code_arena_st code_arena_t; - -struct trampoline_api_st { - void* (*init_state)(void); - void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); - int (*free_state)(void* state); - void *state; -}; - typedef struct trampoline_api_st trampoline_api_t; - -static perf_status_t perf_status = PERF_STATUS_NO_INIT; -static Py_ssize_t extra_code_index = -1; -static code_arena_t *code_arena; -static trampoline_api_t trampoline_api; - -static FILE *perf_map_file; +#define perf_status _PyRuntime.ceval.perf.status +#define extra_code_index _PyRuntime.ceval.perf.extra_code_index +#define perf_code_arena _PyRuntime.ceval.perf.code_arena +#define trampoline_api _PyRuntime.ceval.perf.trampoline_api +#define perf_map_file _PyRuntime.ceval.perf.map_file static void * perf_map_get_file(void) @@ -344,17 +328,17 @@ new_code_arena(void) new_arena->size = mem_size; new_arena->size_left = mem_size; new_arena->code_size = code_size; - new_arena->prev = code_arena; - code_arena = new_arena; + new_arena->prev = perf_code_arena; + perf_code_arena = new_arena; return 0; } static void free_code_arenas(void) { - code_arena_t *cur = code_arena; + code_arena_t *cur = perf_code_arena; code_arena_t *prev; - code_arena = NULL; // invalid static pointer + perf_code_arena = NULL; // invalid static pointer while (cur) { munmap(cur->start_addr, cur->size); prev = cur->prev; @@ -375,14 +359,14 @@ code_arena_new_code(code_arena_t *code_arena) static inline py_trampoline compile_trampoline(void) { - if ((code_arena == NULL) || - (code_arena->size_left <= code_arena->code_size)) { + if ((perf_code_arena == NULL) || + (perf_code_arena->size_left <= perf_code_arena->code_size)) { if (new_code_arena() < 0) { return NULL; } } - assert(code_arena->size_left <= code_arena->size); - return code_arena_new_code(code_arena); + assert(perf_code_arena->size_left <= perf_code_arena->size); + return code_arena_new_code(perf_code_arena); } static PyObject * @@ -405,7 +389,7 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, goto default_eval; } trampoline_api.write_state(trampoline_api.state, new_trampoline, - code_arena->code_size, co); + perf_code_arena->code_size, co); _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); f = new_trampoline; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 3f6e1c740535..f774cdeab386 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -302,21 +302,10 @@ Objects/sliceobject.c - _Py_EllipsisObject - ################################## ## global non-objects to fix in core code -##----------------------- -## effectively-const but initialized lazily - -## others -Python/perf_trampoline.c - perf_map_file - - ##----------------------- ## state -## other Objects/object.c - _Py_RefTotal - -Python/perf_trampoline.c - perf_status - -Python/perf_trampoline.c - extra_code_index - -Python/perf_trampoline.c - code_arena - -Python/perf_trampoline.c - trampoline_api - Python/thread_pthread_stubs.h - py_tls_entries - @@ -374,9 +363,8 @@ Modules/itertoolsmodule.c - teedataobject_type - Modules/itertoolsmodule.c - ziplongest_type - ##----------------------- -## other - ## state + Modules/faulthandler.c - fatal_error - Modules/faulthandler.c - thread - Modules/faulthandler.c - user_signals - From webhook-mailer at python.org Thu Dec 8 19:51:03 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 00:51:03 -0000 Subject: [Python-checkins] gh-81057: Move Threading-Related Globals to _PyRuntimeState (#100084) Message-ID: <mailman.2872.1670547065.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1160001b34615066b1188d5fb457131b1ebb928d commit: 1160001b34615066b1188d5fb457131b1ebb928d branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T17:50:58-07:00 summary: gh-81057: Move Threading-Related Globals to _PyRuntimeState (#100084) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_pythread.h M Include/internal/pycore_runtime.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/thread.c M Python/thread_nt.h M Python/thread_pthread.h M Python/thread_pthread_stubs.h M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h new file mode 100644 index 000000000000..4aeb285b89a8 --- /dev/null +++ b/Include/internal/pycore_pythread.h @@ -0,0 +1,81 @@ +#ifndef Py_INTERNAL_PYTHREAD_H +#define Py_INTERNAL_PYTHREAD_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#ifndef _POSIX_THREADS +/* This means pthreads are not implemented in libc headers, hence the macro + not present in unistd.h. But they still can be implemented as an external + library (e.g. gnu pth in pthread emulation) */ +# ifdef HAVE_PTHREAD_H +# include <pthread.h> /* _POSIX_THREADS */ +# endif +# ifndef _POSIX_THREADS +/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then + enough of the Posix threads package is implemented to support python + threads. + + This is valid for HP-UX 11.23 running on an ia64 system. If needed, add + a check of __ia64 to verify that we're running on an ia64 system instead + of a pa-risc system. +*/ +# ifdef __hpux +# ifdef _SC_THREADS +# define _POSIX_THREADS +# endif +# endif +# endif /* _POSIX_THREADS */ +#endif /* _POSIX_THREADS */ + +#if defined(_POSIX_THREADS) && !defined(HAVE_PTHREAD_STUBS) +# define _USE_PTHREADS +#endif + +#if defined(_USE_PTHREADS) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) +// monotonic is supported statically. It doesn't mean it works on runtime. +# define CONDATTR_MONOTONIC +#endif + + +#if defined(HAVE_PTHREAD_STUBS) +// pthread_key +struct py_stub_tls_entry { + bool in_use; + void *value; +}; +#endif + +struct _pythread_runtime_state { + int initialized; + +#ifdef _USE_PTHREADS + // This matches when thread_pthread.h is used. + struct { + /* NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported. */ + pthread_condattr_t *ptr; +# ifdef CONDATTR_MONOTONIC + /* The value to which condattr_monotonic is set. */ + pthread_condattr_t val; +# endif + } _condattr_monotonic; + +#endif // USE_PTHREADS + +#if defined(HAVE_PTHREAD_STUBS) + struct { + struct py_stub_tls_entry tls_entries[PTHREAD_KEYS_MAX]; + } stubs; +#endif +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYTHREAD_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index ac89c7d378e2..fe2de5feb47a 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -20,6 +20,7 @@ extern "C" { #include "pycore_parser.h" // struct _parser_runtime_state #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state +#include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state #include "pycore_time.h" // struct _time_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids @@ -96,6 +97,7 @@ typedef struct pyruntimestate { int unhandled_keyboard_interrupt; } signals; struct _time_runtime_state time; + struct _pythread_runtime_state threads; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Makefile.pre.in b/Makefile.pre.in index 312031999df0..de42d684f166 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1663,6 +1663,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pymem.h \ $(srcdir)/Include/internal/pycore_pymem_init.h \ $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pythread.h \ $(srcdir)/Include/internal/pycore_range.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 58ea6ae7b4aa..35fbff320f46 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -245,6 +245,7 @@ <ClInclude Include="..\Include\internal\pycore_pymem.h" /> <ClInclude Include="..\Include\internal\pycore_pymem_init.h" /> <ClInclude Include="..\Include\internal\pycore_pystate.h" /> + <ClInclude Include="..\Include\internal\pycore_pythread.h" /> <ClInclude Include="..\Include\internal\pycore_range.h" /> <ClInclude Include="..\Include\internal\pycore_runtime.h" /> <ClInclude Include="..\Include\internal\pycore_runtime_init.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 96c3bd6c30e9..19cb5cf1c807 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -639,6 +639,9 @@ <ClInclude Include="..\Include\internal\pycore_pystate.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_pythread.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_range.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Python/thread.c b/Python/thread.c index 3c1e78ed1bca..4581f1af043a 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -8,15 +8,7 @@ #include "Python.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniType() - -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include <pthread.h> /* _POSIX_THREADS */ -# endif -#endif +#include "pycore_pythread.h" #ifndef DONT_HAVE_STDIO_H #include <stdio.h> @@ -24,33 +16,17 @@ #include <stdlib.h> -#ifndef _POSIX_THREADS - -/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then - enough of the Posix threads package is implemented to support python - threads. - - This is valid for HP-UX 11.23 running on an ia64 system. If needed, add - a check of __ia64 to verify that we're running on an ia64 system instead - of a pa-risc system. -*/ -#ifdef __hpux -#ifdef _SC_THREADS -#define _POSIX_THREADS -#endif -#endif - -#endif /* _POSIX_THREADS */ - -static int initialized; static void PyThread__init_thread(void); /* Forward */ +#define initialized _PyRuntime.threads.initialized + void PyThread_init_thread(void) { - if (initialized) + if (initialized) { return; + } initialized = 1; PyThread__init_thread(); } @@ -58,7 +34,7 @@ PyThread_init_thread(void) #if defined(HAVE_PTHREAD_STUBS) # define PYTHREAD_NAME "pthread-stubs" # include "thread_pthread_stubs.h" -#elif defined(_POSIX_THREADS) +#elif defined(_USE_PTHREADS) /* AKA _PTHREADS */ # if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__) # define PYTHREAD_NAME "pthread-stubs" # else diff --git a/Python/thread_nt.h b/Python/thread_nt.h index d1f1323948a6..26f441bd6d3c 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -152,11 +152,12 @@ unsigned long PyThread_get_thread_native_id(void); #endif /* - * Initialization of the C package, should not be needed. + * Initialization for the current runtime. */ static void PyThread__init_thread(void) { + // Initialization of the C package should not be needed. } /* diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 1c5b320813af..ae312e987bd6 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -119,20 +119,16 @@ * pthread_cond support */ -#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) -// monotonic is supported statically. It doesn't mean it works on runtime. -#define CONDATTR_MONOTONIC -#endif - -// NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported. -static pthread_condattr_t *condattr_monotonic = NULL; +#define condattr_monotonic _PyRuntime.threads._condattr_monotonic.ptr static void init_condattr(void) { #ifdef CONDATTR_MONOTONIC - static pthread_condattr_t ca; +# define ca _PyRuntime.threads._condattr_monotonic.val + // XXX We need to check the return code? pthread_condattr_init(&ca); + // XXX We need to run pthread_condattr_destroy() during runtime fini. if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) { condattr_monotonic = &ca; // Use monotonic clock } @@ -192,15 +188,21 @@ typedef struct { "%s: %s\n", name, strerror(status)); error = 1; } /* - * Initialization. + * Initialization for the current runtime. */ static void PyThread__init_thread(void) { + // The library is only initialized once in the process, + // regardless of how many times the Python runtime is initialized. + static int lib_initialized = 0; + if (!lib_initialized) { + lib_initialized = 1; #if defined(_AIX) && defined(__GNUC__) - extern void pthread_init(void); - pthread_init(); + extern void pthread_init(void); + pthread_init(); #endif + } init_condattr(); } diff --git a/Python/thread_pthread_stubs.h b/Python/thread_pthread_stubs.h index 8b80c0f87e25..56e5b6141924 100644 --- a/Python/thread_pthread_stubs.h +++ b/Python/thread_pthread_stubs.h @@ -124,13 +124,10 @@ pthread_attr_destroy(pthread_attr_t *attr) return 0; } -// pthread_key -typedef struct { - bool in_use; - void *value; -} py_tls_entry; -static py_tls_entry py_tls_entries[PTHREAD_KEYS_MAX] = {0}; +typedef struct py_stub_tls_entry py_tls_entry; + +#define py_tls_entries (_PyRuntime.threads.stubs.tls_entries) int pthread_key_create(pthread_key_t *key, void (*destr_function)(void *)) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index f774cdeab386..94e9831db1fd 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -306,7 +306,6 @@ Objects/sliceobject.c - _Py_EllipsisObject - ## state Objects/object.c - _Py_RefTotal - -Python/thread_pthread_stubs.h - py_tls_entries - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 814c55b75c09..7d6ff0ba6d64 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -22,11 +22,8 @@ Python/fileutils.c set_inheritable ioctl_works - # XXX Is this thread-safe? Modules/posixmodule.c os_dup2_impl dup3_works - -## resource init - set during first init -Python/thread.c - initialized - -Python/thread_pthread.h - condattr_monotonic - -# safe static buffer used during one-time initialization -Python/thread_pthread.h init_condattr ca - +## guards around resource init +Python/thread_pthread.h PyThread__init_thread lib_initialized - ##----------------------- ## other values (not Python-specific) From webhook-mailer at python.org Thu Dec 8 20:16:44 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 01:16:44 -0000 Subject: [Python-checkins] gh-81057: Fix an ifdef in the time module (#100125) Message-ID: <mailman.2873.1670548605.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3e06b5030b18ca9d9d507423b582d13f38d393f2 commit: 3e06b5030b18ca9d9d507423b582d13f38d393f2 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-08T18:16:37-07:00 summary: gh-81057: Fix an ifdef in the time module (#100125) An earlier commit only defined check_ticks_per_second() when HAVE_TIMES is defined. However, we also need it when HAVE_CLOCK is defined. This primarily affects Windows. https://github.com/python/cpython/issues/81057 files: M Modules/timemodule.c diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ba4128c0fdf5..c2bacaae0c03 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -62,8 +62,7 @@ #define SEC_TO_NS (1000 * 1000 * 1000) -#ifdef HAVE_TIMES - +#if defined(HAVE_TIMES) || defined(HAVE_CLOCK) static int check_ticks_per_second(long tps, const char *context) { @@ -75,6 +74,9 @@ check_ticks_per_second(long tps, const char *context) } return 0; } +#endif /* HAVE_TIMES || HAVE_CLOCK */ + +#ifdef HAVE_TIMES # define ticks_per_second _PyRuntime.time.ticks_per_second From webhook-mailer at python.org Thu Dec 8 22:31:25 2022 From: webhook-mailer at python.org (ethanfurman) Date: Fri, 09 Dec 2022 03:31:25 -0000 Subject: [Python-checkins] gh-99087: Add missing newline for prompts in docs (GH-98993) Message-ID: <mailman.2874.1670556687.3313.python-checkins@python.org> https://github.com/python/cpython/commit/286e3c76a9cb8f1adc2a915f0d246a1e2e408733 commit: 286e3c76a9cb8f1adc2a915f0d246a1e2e408733 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-08T19:31:19-08:00 summary: gh-99087: Add missing newline for prompts in docs (GH-98993) Add newline for prompts so copying to REPL does not cause errors. files: M Doc/howto/enum.rst M Doc/library/argparse.rst M Doc/library/bz2.rst M Doc/library/collections.rst M Doc/library/datetime.rst M Doc/library/decimal.rst M Doc/library/doctest.rst M Doc/library/email.policy.rst M Doc/library/enum.rst M Doc/library/functions.rst M Doc/library/hashlib.rst M Doc/library/inspect.rst M Doc/library/re.rst M Doc/library/sqlite3.rst M Doc/library/statistics.rst M Doc/library/stdtypes.rst M Doc/library/unittest.mock.rst M Doc/library/xml.etree.elementtree.rst M Doc/library/zipfile.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.2.rst M Doc/whatsnew/3.3.rst diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 3155c6cb977b..4525acb04503 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -158,6 +158,7 @@ And a function to display the chores for a given day:: ... for chore, days in chores.items(): ... if day in days: ... print(chore) + ... >>> show_chores(chores_for_ethan, Weekday.SATURDAY) answer SO questions @@ -712,6 +713,7 @@ It is also possible to name the combinations:: ... W = 2 ... X = 1 ... RWX = 7 + ... >>> Perm.RWX <Perm.RWX: 7> >>> ~Perm.RWX diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index f8839d0986d0..e6c964864925 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -565,6 +565,7 @@ arguments they contain. For example:: >>> with open('args.txt', 'w', encoding=sys.getfilesystemencoding()) as fp: ... fp.write('-f\nbar') + ... >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') >>> parser.add_argument('-f') >>> parser.parse_args(['-f', 'foo', '@args.txt']) diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ae5a1598f84b..32df99869eb5 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -320,9 +320,11 @@ Writing and reading a bzip2-compressed file in binary mode: >>> with bz2.open("myfile.bz2", "wb") as f: ... # Write compressed data to file ... unused = f.write(data) + ... >>> with bz2.open("myfile.bz2", "rb") as f: ... # Decompress data from file ... content = f.read() + ... >>> content == data # Check equality to original object after round-trip True diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 53b4b69f84b7..2cffc2300a22 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -229,6 +229,7 @@ For example:: >>> cnt = Counter() >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: ... cnt[word] += 1 + ... >>> cnt Counter({'blue': 3, 'red': 2, 'green': 1}) @@ -818,6 +819,7 @@ zero): >>> def constant_factory(value): ... return lambda: value + ... >>> d = defaultdict(constant_factory('<missing>')) >>> d.update(name='John', action='ran') >>> '%(name)s %(action)s to %(object)s' % d diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index f7e2bb3f3c6d..0b39512a2d23 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -765,6 +765,7 @@ Example of counting days to an event:: >>> my_birthday = date(today.year, 6, 24) >>> if my_birthday < today: ... my_birthday = my_birthday.replace(year=today.year + 1) + ... >>> my_birthday datetime.date(2008, 6, 24) >>> time_to_birthday = abs(my_birthday - today) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 260108136df7..fec9b86864c5 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -2057,6 +2057,7 @@ to handle the :meth:`quantize` step: >>> def mul(x, y, fp=TWOPLACES): ... return (x * y).quantize(fp) + ... >>> def div(x, y, fp=TWOPLACES): ... return (x / y).quantize(fp) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index c106d5a3383a..b6dd7f7e1232 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -351,6 +351,7 @@ The fine print: >>> def f(x): ... r'''Backslashes in a raw docstring: m\n''' + ... >>> print(f.__doc__) Backslashes in a raw docstring: m\n @@ -360,6 +361,7 @@ The fine print: >>> def f(x): ... '''Backslashes in a raw docstring: m\\n''' + ... >>> print(f.__doc__) Backslashes in a raw docstring: m\n diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index bf53b9520fc7..2439dee676c9 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -97,6 +97,7 @@ file on disk and pass it to the system ``sendmail`` program on a Unix system: >>> from subprocess import Popen, PIPE >>> with open('mymsg.txt', 'rb') as f: ... msg = message_from_binary_file(f, policy=policy.default) + ... >>> p = Popen(['sendmail', msg['To'].addresses[0]], stdin=PIPE) >>> g = BytesGenerator(p.stdin, policy=msg.policy.clone(linesep='\r\n')) >>> g.flatten(msg) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index f8875440baf5..295152c69082 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -292,6 +292,7 @@ Data Types ... @classmethod ... def today(cls): ... print('today is %s' % cls(date.today().isoweekday()).name) + ... >>> dir(Weekday.SATURDAY) ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] @@ -312,6 +313,7 @@ Data Types ... return (count + 1) * 3 ... FIRST = auto() ... SECOND = auto() + ... >>> PowersOfThree.SECOND.value 6 @@ -336,6 +338,7 @@ Data Types ... if member.value == value: ... return member ... return None + ... >>> Build.DEBUG.value 'debug' >>> Build('deBUG') @@ -353,6 +356,7 @@ Data Types ... def __repr__(self): ... cls_name = self.__class__.__name__ ... return f'{cls_name}.{self.name}' + ... >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" (OtherStyle.ALTERNATE, 'OtherStyle.ALTERNATE', 'OtherStyle.ALTERNATE') @@ -367,6 +371,7 @@ Data Types ... SOMETHING_ELSE = auto() ... def __str__(self): ... return f'{self.name}' + ... >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" (<OtherStyle.ALTERNATE: 1>, 'ALTERNATE', 'ALTERNATE') @@ -381,6 +386,7 @@ Data Types ... SOMETHING_ELSE = auto() ... def __format__(self, spec): ... return f'{self.name}' + ... >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" (<OtherStyle.ALTERNATE: 1>, 'OtherStyle.ALTERNATE', 'ALTERNATE') @@ -403,6 +409,7 @@ Data Types ... ONE = 1 ... TWO = 2 ... THREE = 3 + ... >>> Numbers.THREE <Numbers.THREE: 3> >>> Numbers.ONE + Numbers.TWO @@ -463,6 +470,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> purple = Color.RED | Color.BLUE >>> white = Color.RED | Color.GREEN | Color.BLUE >>> Color.GREEN in purple @@ -570,6 +578,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> Color.RED & 2 <Color: 0> >>> Color.RED | 2 @@ -695,6 +704,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> StrictFlag(2**2 + 2**4) Traceback (most recent call last): ... @@ -712,6 +722,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> ConformFlag(2**2 + 2**4) <ConformFlag.BLUE: 4> @@ -725,6 +736,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> EjectFlag(2**2 + 2**4) 20 @@ -738,6 +750,7 @@ Data Types ... RED = auto() ... GREEN = auto() ... BLUE = auto() + ... >>> KeepFlag(2**2 + 2**4) <KeepFlag.BLUE|16: 20> diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 110e7e5d7fb9..2110990d1889 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -462,6 +462,7 @@ are always available. They are listed here in alphabetical order. >>> class Shape: ... def __dir__(self): ... return ['area', 'perimeter', 'location'] + ... >>> s = Shape() >>> dir(s) ['area', 'location', 'perimeter'] diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 8e47312fe77b..f8d10c0c295c 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -497,6 +497,7 @@ update the hash: >>> h = blake2b() >>> for item in items: ... h.update(item) + ... >>> h.hexdigest() '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 9cb7a6f94e49..6705577551dc 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -715,6 +715,7 @@ function. >>> def test(a, b): ... pass + ... >>> sig = signature(test) >>> new_sig = sig.replace(return_annotation="new return anno") >>> str(new_sig) @@ -1054,6 +1055,7 @@ Classes and functions >>> from inspect import getcallargs >>> def f(a, b=1, *pos, **named): ... pass + ... >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} True >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} diff --git a/Doc/library/re.rst b/Doc/library/re.rst index e6e242320fd8..f7d46586cf75 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -973,6 +973,7 @@ Functions >>> def dashrepl(matchobj): ... if matchobj.group(0) == '-': return ' ' ... else: return '-' + ... >>> re.sub('-{1,2}', dashrepl, 'pro----gram-files') 'pro--gram files' >>> re.sub(r'\sAND\s', ' & ', 'Baked Beans And Spam', flags=re.IGNORECASE) @@ -1672,6 +1673,7 @@ in each word of a sentence except for the first and last characters:: ... inner_word = list(m.group(2)) ... random.shuffle(inner_word) ... return m.group(1) + "".join(inner_word) + m.group(3) + ... >>> text = "Professor Abdolmalek, please report your absences promptly." >>> re.sub(r"(\w)(\w+)(\w)", repl, text) 'Poefsrosr Aealmlobdk, pslaee reorpt your abnseces plmrptoy.' diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 2b6387cbfb52..3622864a4b06 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -397,6 +397,7 @@ Module functions >>> con = sqlite3.connect(":memory:") >>> def evil_trace(stmt): ... 5/0 + ... >>> con.set_trace_callback(evil_trace) >>> def debug(unraisable): ... print(f"{unraisable.exc_value!r} in callback {unraisable.object.__name__}") diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 88a887960edb..f934b0e0319d 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -996,6 +996,7 @@ probability that the Python room will stay within its capacity limits? >>> seed(8675309) >>> def trial(): ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') + ... >>> mean(trial() <= k for i in range(10_000)) 0.8398 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 785b76a11f2f..73debe5ceeaf 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4459,6 +4459,7 @@ can be used interchangeably to index the same dictionary entry. >>> class Counter(dict): ... def __missing__(self, key): ... return 0 + ... >>> c = Counter() >>> c['red'] 0 @@ -4716,6 +4717,7 @@ An example of dictionary view usage:: >>> n = 0 >>> for val in values: ... n += val + ... >>> print(n) 504 diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index b768557e6075..e009f303fef3 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1604,6 +1604,7 @@ decorator: >>> @patch.dict(foo, {'newkey': 'newvalue'}) ... def test(): ... assert foo == {'newkey': 'newvalue'} + ... >>> test() >>> assert foo == {} diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 2fe0d2e082fb..876de29b17ca 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1212,6 +1212,7 @@ Example of changing the attribute "target" of every link in first paragraph:: [<Element 'a' at 0xb77ec2ac>, <Element 'a' at 0xb77ec1cc>] >>> for i in links: # Iterates through all found links ... i.attrib["target"] = "blank" + ... >>> tree.write("output.xhtml") .. _elementtree-qname-objects: diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4dd9fa961a8d..82709dbc9249 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -672,6 +672,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the >>> def notests(s): ... fn = os.path.basename(s) ... return (not (fn == 'test' or fn.startswith('test_'))) + ... >>> zf.writepy('myprog', filterfunc=notests) The :meth:`writepy` method makes archives with file names like diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 276ab63b97f8..810a2cd2537c 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1331,6 +1331,7 @@ changes, or look through the Subversion logs for all the details. >>> from inspect import getcallargs >>> def f(a, b=1, *pos, **named): ... pass + ... >>> getcallargs(f, 1, 2, 3) {'a': 1, 'b': 2, 'pos': (3,), 'named': {}} >>> getcallargs(f, a=2, x=4) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 6037db9f954d..1b1455b72b92 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -468,6 +468,7 @@ Some smaller changes made to the core Python language are: >>> class LowerCasedDict(dict): ... def __getitem__(self, key): ... return dict.__getitem__(self, key.lower()) + ... >>> lcd = LowerCasedDict(part='widgets', quantity=10) >>> 'There are {QUANTITY} {Part} in stock'.format_map(lcd) 'There are 10 widgets in stock' @@ -475,6 +476,7 @@ Some smaller changes made to the core Python language are: >>> class PlaceholderDict(dict): ... def __missing__(self, key): ... return '<{}>'.format(key) + ... >>> 'Hello {name}, welcome to {location}'.format_map(PlaceholderDict()) 'Hello <name>, welcome to <location>' @@ -1886,6 +1888,7 @@ inspect >>> from inspect import getgeneratorstate >>> def gen(): ... yield 'demo' + ... >>> g = gen() >>> getgeneratorstate(g) 'GEN_CREATED' diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 96a632577b2c..9e8d42469b01 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -560,6 +560,7 @@ Example with (non-bound) methods:: >>> class C: ... def meth(self): ... pass + ... >>> C.meth.__name__ 'meth' >>> C.meth.__qualname__ From webhook-mailer at python.org Thu Dec 8 22:52:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 09 Dec 2022 03:52:24 -0000 Subject: [Python-checkins] GH-100113: remove remaining `yield from` usage from `asyncio` tests (#100114) Message-ID: <mailman.2875.1670557945.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0448deac70be94792616c0fb0c9cb524de9a09b8 commit: 0448deac70be94792616c0fb0c9cb524de9a09b8 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-09T09:22:18+05:30 summary: GH-100113: remove remaining `yield from` usage from `asyncio` tests (#100114) files: M Lib/test/test_asyncio/test_tasks.py diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index bb1ffdf2df74..5168b8250ef0 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2092,8 +2092,8 @@ def test_cancel_gather_1(self): async def create(): # The indirection fut->child_coro is needed since otherwise the # gathering task is done at the same time as the child future - def child_coro(): - return (yield from fut) + async def child_coro(): + return await fut gather_future = asyncio.gather(child_coro()) return asyncio.ensure_future(gather_future) gather_task = loop.run_until_complete(create()) From webhook-mailer at python.org Fri Dec 9 05:27:07 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 09 Dec 2022 10:27:07 -0000 Subject: [Python-checkins] GH-100110: Specialize FOR_ITER for tuples (GH-100109) Message-ID: <mailman.2876.1670581628.3313.python-checkins@python.org> https://github.com/python/cpython/commit/748c6c0921ee02a19e01a35f03ce5f4d9cfde5a6 commit: 748c6c0921ee02a19e01a35f03ce5f4d9cfde5a6 branch: main author: Ken Jin <kenjin at python.org> committer: markshannon <mark at hotpy.org> date: 2022-12-09T10:27:01Z summary: GH-100110: Specialize FOR_ITER for tuples (GH-100109) * Specialize FOR_ITER for tuples files: A Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst M Include/internal/pycore_opcode.h M Include/internal/pycore_tuple.h M Include/opcode.h M Lib/opcode.py M Objects/tupleobject.c M Python/bytecodes.c M Python/generated_cases.c.h M Python/opcode_targets.h M Python/specialize.c diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 0d31ca166a7d..da8a272f2fa2 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -125,6 +125,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [FOR_ITER_GEN] = FOR_ITER, [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, + [FOR_ITER_TUPLE] = FOR_ITER, [GET_AITER] = GET_AITER, [GET_ANEXT] = GET_ANEXT, [GET_AWAITABLE] = GET_AWAITABLE, @@ -293,31 +294,31 @@ static const char *const _PyOpcode_OpName[263] = { [FOR_ITER_LIST] = "FOR_ITER_LIST", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [STOPITERATION_ERROR] = "STOPITERATION_ERROR", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -344,7 +345,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -352,7 +353,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -372,9 +373,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -384,24 +385,24 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [170] = "<170>", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [173] = "<173>", @@ -498,7 +499,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 170: \ case 173: \ case 174: \ case 175: \ diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 504c36338d9e..edc70843b575 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -67,6 +67,13 @@ struct _Py_tuple_state { extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); + +typedef struct { + PyObject_HEAD + Py_ssize_t it_index; + PyTupleObject *it_seq; /* Set to NULL when iterator is exhausted */ +} _PyTupleIterObject; + #ifdef __cplusplus } #endif diff --git a/Include/opcode.h b/Include/opcode.h index f284313d2ed7..888250ed37e8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -162,34 +162,35 @@ extern "C" { #define COMPARE_OP_INT_JUMP 57 #define COMPARE_OP_STR_JUMP 58 #define FOR_ITER_LIST 59 -#define FOR_ITER_RANGE 62 -#define FOR_ITER_GEN 64 -#define LOAD_ATTR_CLASS 65 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 72 -#define LOAD_ATTR_PROPERTY 73 -#define LOAD_ATTR_SLOT 76 -#define LOAD_ATTR_WITH_HINT 77 -#define LOAD_ATTR_METHOD_LAZY_DICT 78 -#define LOAD_ATTR_METHOD_NO_DICT 79 -#define LOAD_ATTR_METHOD_WITH_DICT 80 -#define LOAD_ATTR_METHOD_WITH_VALUES 81 -#define LOAD_CONST__LOAD_FAST 86 -#define LOAD_FAST__LOAD_CONST 113 -#define LOAD_FAST__LOAD_FAST 121 -#define LOAD_GLOBAL_BUILTIN 141 -#define LOAD_GLOBAL_MODULE 143 -#define STORE_ATTR_INSTANCE_VALUE 153 -#define STORE_ATTR_SLOT 154 -#define STORE_ATTR_WITH_HINT 158 -#define STORE_FAST__LOAD_FAST 159 -#define STORE_FAST__STORE_FAST 160 -#define STORE_SUBSCR_DICT 161 -#define STORE_SUBSCR_LIST_INT 166 -#define UNPACK_SEQUENCE_LIST 167 -#define UNPACK_SEQUENCE_TUPLE 168 -#define UNPACK_SEQUENCE_TWO_TUPLE 169 +#define FOR_ITER_TUPLE 62 +#define FOR_ITER_RANGE 64 +#define FOR_ITER_GEN 65 +#define LOAD_ATTR_CLASS 66 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 +#define LOAD_ATTR_INSTANCE_VALUE 72 +#define LOAD_ATTR_MODULE 73 +#define LOAD_ATTR_PROPERTY 76 +#define LOAD_ATTR_SLOT 77 +#define LOAD_ATTR_WITH_HINT 78 +#define LOAD_ATTR_METHOD_LAZY_DICT 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_WITH_DICT 81 +#define LOAD_ATTR_METHOD_WITH_VALUES 86 +#define LOAD_CONST__LOAD_FAST 113 +#define LOAD_FAST__LOAD_CONST 121 +#define LOAD_FAST__LOAD_FAST 141 +#define LOAD_GLOBAL_BUILTIN 143 +#define LOAD_GLOBAL_MODULE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 160 +#define STORE_FAST__STORE_FAST 161 +#define STORE_SUBSCR_DICT 166 +#define STORE_SUBSCR_LIST_INT 167 +#define UNPACK_SEQUENCE_LIST 168 +#define UNPACK_SEQUENCE_TUPLE 169 +#define UNPACK_SEQUENCE_TWO_TUPLE 170 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index fa6dbe5d2417..fc57affbac58 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -320,6 +320,7 @@ def pseudo_op(name, op, real_ops): ], "FOR_ITER": [ "FOR_ITER_LIST", + "FOR_ITER_TUPLE", "FOR_ITER_RANGE", "FOR_ITER_GEN", ], diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst new file mode 100644 index 000000000000..e494f44fd42c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-08-12-26-34.gh-issue-100110.ertac-.rst @@ -0,0 +1 @@ +Specialize ``FOR_ITER`` for tuples. diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 4405125d45e7..e1b9953226c0 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -995,14 +995,9 @@ _PyTuple_ClearFreeList(PyInterpreterState *interp) /*********************** Tuple Iterator **************************/ -typedef struct { - PyObject_HEAD - Py_ssize_t it_index; - PyTupleObject *it_seq; /* Set to NULL when iterator is exhausted */ -} tupleiterobject; static void -tupleiter_dealloc(tupleiterobject *it) +tupleiter_dealloc(_PyTupleIterObject *it) { _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); @@ -1010,14 +1005,14 @@ tupleiter_dealloc(tupleiterobject *it) } static int -tupleiter_traverse(tupleiterobject *it, visitproc visit, void *arg) +tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) { Py_VISIT(it->it_seq); return 0; } static PyObject * -tupleiter_next(tupleiterobject *it) +tupleiter_next(_PyTupleIterObject *it) { PyTupleObject *seq; PyObject *item; @@ -1040,7 +1035,7 @@ tupleiter_next(tupleiterobject *it) } static PyObject * -tupleiter_len(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_len(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) { Py_ssize_t len = 0; if (it->it_seq) @@ -1051,7 +1046,7 @@ tupleiter_len(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -tupleiter_reduce(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) { if (it->it_seq) return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), @@ -1061,7 +1056,7 @@ tupleiter_reduce(tupleiterobject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -tupleiter_setstate(tupleiterobject *it, PyObject *state) +tupleiter_setstate(_PyTupleIterObject *it, PyObject *state) { Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) @@ -1089,7 +1084,7 @@ static PyMethodDef tupleiter_methods[] = { PyTypeObject PyTupleIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "tuple_iterator", /* tp_name */ - sizeof(tupleiterobject), /* tp_basicsize */ + sizeof(_PyTupleIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)tupleiter_dealloc, /* tp_dealloc */ @@ -1122,13 +1117,13 @@ PyTypeObject PyTupleIter_Type = { static PyObject * tuple_iter(PyObject *seq) { - tupleiterobject *it; + _PyTupleIterObject *it; if (!PyTuple_Check(seq)) { PyErr_BadInternalCall(); return NULL; } - it = PyObject_GC_New(tupleiterobject, &PyTupleIter_Type); + it = PyObject_GC_New(_PyTupleIterObject, &PyTupleIter_Type); if (it == NULL) return NULL; it->it_index = 0; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7a1bfdcc9e26..5807bd5dc2d2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2523,6 +2523,29 @@ dummy_func( end_for_iter_list: } + // stack effect: ( -- __0) + inst(FOR_ITER_TUPLE) { + assert(cframe.use_tracing == 0); + _PyTupleIterObject *it = (_PyTupleIterObject *)TOP(); + DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq) { + if (it->it_index < PyTuple_GET_SIZE(seq)) { + PyObject *next = PyTuple_GET_ITEM(seq, it->it_index++); + PUSH(Py_NewRef(next)); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); + goto end_for_iter_tuple; // End of this instruction + } + it->it_seq = NULL; + Py_DECREF(seq); + } + STACK_SHRINK(1); + Py_DECREF(it); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + end_for_iter_tuple: + } + // stack effect: ( -- __0) inst(FOR_ITER_RANGE) { assert(cframe.use_tracing == 0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 76eb6085ec57..59e70b722269 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2759,6 +2759,29 @@ DISPATCH(); } + TARGET(FOR_ITER_TUPLE) { + assert(cframe.use_tracing == 0); + _PyTupleIterObject *it = (_PyTupleIterObject *)TOP(); + DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq) { + if (it->it_index < PyTuple_GET_SIZE(seq)) { + PyObject *next = PyTuple_GET_ITEM(seq, it->it_index++); + PUSH(Py_NewRef(next)); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); + goto end_for_iter_tuple; // End of this instruction + } + it->it_seq = NULL; + Py_DECREF(seq); + } + STACK_SHRINK(1); + Py_DECREF(it); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + end_for_iter_tuple: + DISPATCH(); + } + TARGET(FOR_ITER_RANGE) { assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)TOP(); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 3aba4e7556a6..be3ad01c151c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -61,31 +61,31 @@ static void *opcode_targets[256] = { &&TARGET_FOR_ITER_LIST, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_FOR_ITER_RANGE, + &&TARGET_FOR_ITER_TUPLE, &&TARGET_STOPITERATION_ERROR, + &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index cd09b188b7fa..7545a7712493 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2132,6 +2132,10 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) _Py_SET_OPCODE(*instr, FOR_ITER_LIST); goto success; } + else if (tp == &PyTupleIter_Type) { + _Py_SET_OPCODE(*instr, FOR_ITER_TUPLE); + goto success; + } else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { _Py_SET_OPCODE(*instr, FOR_ITER_RANGE); goto success; From webhook-mailer at python.org Fri Dec 9 06:16:21 2022 From: webhook-mailer at python.org (zooba) Date: Fri, 09 Dec 2022 11:16:21 -0000 Subject: [Python-checkins] gh-88267: Avoid DLL exporting functions from static builds on Windows(GH-99888) Message-ID: <mailman.2877.1670584582.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3c5355496b54fa0a4ea0e22344d008528e45682c commit: 3c5355496b54fa0a4ea0e22344d008528e45682c branch: main author: Christian Rendina <christian.rendina at gmail.com> committer: zooba <steve.dower at microsoft.com> date: 2022-12-09T11:16:15Z summary: gh-88267: Avoid DLL exporting functions from static builds on Windows(GH-99888) files: A Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst M Include/exports.h diff --git a/Include/exports.h b/Include/exports.h index fc1a5c5ead62..59373c39ff75 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -2,9 +2,15 @@ #define Py_EXPORTS_H #if defined(_WIN32) || defined(__CYGWIN__) - #define Py_IMPORTED_SYMBOL __declspec(dllimport) - #define Py_EXPORTED_SYMBOL __declspec(dllexport) - #define Py_LOCAL_SYMBOL + #if defined(Py_ENABLE_SHARED) + #define Py_IMPORTED_SYMBOL __declspec(dllimport) + #define Py_EXPORTED_SYMBOL __declspec(dllexport) + #define Py_LOCAL_SYMBOL + #else + #define Py_IMPORTED_SYMBOL + #define Py_EXPORTED_SYMBOL + #define Py_LOCAL_SYMBOL + #endif #else /* * If we only ever used gcc >= 5, we could use __has_attribute(visibility) diff --git a/Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst b/Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst new file mode 100644 index 000000000000..29c3f5a1d763 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-12-08-14-00-04.gh-issue-88267.MqtRbm.rst @@ -0,0 +1 @@ +Avoid exporting Python symbols in linked Windows applications when the core is built as static. From webhook-mailer at python.org Fri Dec 9 07:19:17 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 09 Dec 2022 12:19:17 -0000 Subject: [Python-checkins] GH-98522: Add version number to code objects. (GH-98525) Message-ID: <mailman.2878.1670588358.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fb713b21833a17cba8022af0fa4c486512157d4b commit: fb713b21833a17cba8022af0fa4c486512157d4b branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-09T12:18:45Z summary: GH-98522: Add version number to code objects. (GH-98525) * Add version number to code object for better versioning of functions. * Improves specialization for closures and list comprehensions. files: A Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst M Include/cpython/code.h M Include/internal/pycore_code.h M Objects/codeobject.c M Objects/funcobject.c M Programs/_bootstrap_python.c M Programs/_freeze_module.c M Python/bytecodes.c M Python/generated_cases.c.h M Tools/build/deepfreeze.py diff --git a/Include/cpython/code.h b/Include/cpython/code.h index f11d099e0379..fc7c5ed70243 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -87,6 +87,7 @@ typedef struct { int co_nplaincellvars; /* number of non-arg cell variables */ \ int co_ncellvars; /* total number of cell variables */ \ int co_nfreevars; /* number of free variables */ \ + uint32_t co_version; /* version number */ \ \ PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \ PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 357fc85a95cf..f22fd45f8319 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -474,6 +474,8 @@ typedef struct _PyShimCodeDef { extern PyCodeObject * _Py_MakeShimCode(const _PyShimCodeDef *code); +extern uint32_t _Py_next_func_version; + #ifdef __cplusplus } diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst new file mode 100644 index 000000000000..d923af198f82 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-21-16-10-39.gh-issue-98522.s_SixG.rst @@ -0,0 +1,3 @@ +Add an internal version number to code objects, to give better versioning of +inner functions and comprehensions, and thus better specialization of those +functions. This change is invisible to both Python and C extensions. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0c197d767b0a..c92c7deaf808 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -11,7 +11,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "clinic/codeobject.c.h" - static void notify_code_watchers(PyCodeEvent event, PyCodeObject *co) { @@ -398,7 +397,10 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - + co->co_version = _Py_next_func_version; + if (_Py_next_func_version != 0) { + _Py_next_func_version++; + } /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index bf97edc53ad7..9df06520586a 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -3,7 +3,7 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() -#include "pycore_function.h" // FUNC_MAX_WATCHERS +#include "pycore_code.h" // _Py_next_func_version #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "structmember.h" // PyMemberDef @@ -64,7 +64,6 @@ PyFunction_ClearWatcher(int watcher_id) interp->active_func_watchers &= ~(1 << watcher_id); return 0; } - PyFunctionObject * _PyFunction_FromConstructor(PyFrameConstructor *constr) { diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index bbac0c4e1a8a..6e1593a0b599 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -14,6 +14,8 @@ #include "Python/frozen_modules/importlib._bootstrap_external.h" /* End includes */ +uint32_t _Py_next_func_version = 1; + /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 9e2169f32e92..90fc2dc6e87d 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -9,6 +9,7 @@ Keep this file in sync with Programs/_freeze_module.py. */ + #include <Python.h> #include <marshal.h> #include "pycore_fileutils.h" // _Py_stat_struct @@ -22,6 +23,8 @@ #include <unistd.h> #endif +uint32_t _Py_next_func_version = 1; + /* Empty initializer for deepfrozen modules */ int _Py_Deepfreeze_Init(void) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5807bd5dc2d2..c56f1d3ef9f4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3452,6 +3452,7 @@ dummy_func( func->func_defaults = POP(); } + func->func_version = ((PyCodeObject *)codeobj)->co_version; PUSH((PyObject *)func); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 59e70b722269..45382a466b1c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3693,6 +3693,7 @@ func->func_defaults = POP(); } + func->func_version = ((PyCodeObject *)codeobj)->co_version; PUSH((PyObject *)func); DISPATCH(); } diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 2eef649437a6..7f4e24280133 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -44,6 +44,7 @@ def make_string_literal(b: bytes) -> str: CO_FAST_CELL = 0x40 CO_FAST_FREE = 0x80 +next_code_version = 1 def get_localsplus(code: types.CodeType): a = collections.defaultdict(int) @@ -227,6 +228,7 @@ def generate_unicode(self, name: str, s: str) -> str: def generate_code(self, name: str, code: types.CodeType) -> str: + global next_code_version # The ordering here matches PyCode_NewWithPosOnlyArgs() # (but see below). co_consts = self.generate(name + "_consts", code.co_consts) @@ -268,6 +270,8 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_nplaincellvars = {nplaincellvars},") self.write(f".co_ncellvars = {ncellvars},") self.write(f".co_nfreevars = {nfreevars},") + self.write(f".co_version = {next_code_version},") + next_code_version += 1 self.write(f".co_localsplusnames = {co_localsplusnames},") self.write(f".co_localspluskinds = {co_localspluskinds},") self.write(f".co_filename = {co_filename},") @@ -461,6 +465,7 @@ def generate(args: list[str], output: TextIO) -> None: with printer.block(f"if ({p} < 0)"): printer.write("return -1;") printer.write("return 0;") + printer.write(f"\nuint32_t _Py_next_func_version = {next_code_version};\n") if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") From webhook-mailer at python.org Fri Dec 9 07:47:33 2022 From: webhook-mailer at python.org (zooba) Date: Fri, 09 Dec 2022 12:47:33 -0000 Subject: [Python-checkins] bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) Message-ID: <mailman.2879.1670590054.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1 commit: a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1 branch: main author: Shreyan Avigyan <shreyan.avigyan at gmail.com> committer: zooba <steve.dower at microsoft.com> date: 2022-12-09T12:47:18Z summary: bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) files: A Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst M Lib/test/test_winreg.py M PC/winreg.c diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 8157c2da6efa..769ab67b0f56 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -113,7 +113,6 @@ def _write_test_data(self, root_key, subkeystr="sub_key", "does not close the actual key!") except OSError: pass - def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): # Check we can get default value for this key. val = QueryValue(root_key, test_key_name) @@ -340,6 +339,23 @@ def test_setvalueex_value_range(self): finally: DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_return_value(self): # Test for Issue #16759, return unsigned int from QueryValueEx. # Reg2Py, which gets called by QueryValueEx, was returning a value diff --git a/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst new file mode 100644 index 000000000000..a5975b2d00c7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst @@ -0,0 +1,3 @@ +:meth:`winreg.SetValueEx` now leaves the target value untouched in the case of conversion errors. +Previously, ``-1`` would be written in case of such errors. + diff --git a/PC/winreg.c b/PC/winreg.c index df34e8cf5a77..63b37be526ab 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -561,42 +561,54 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) { Py_ssize_t i,j; switch (typ) { - case REG_DWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD); - if (value == Py_None) { - DWORD zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD)); - } - else { - DWORD d = PyLong_AsUnsignedLong(value); + case REG_DWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLong(value); + if (d == (DWORD)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD)); + *retDataSize = sizeof(DWORD); + break; } - break; - case REG_QWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD64); - if (value == Py_None) { - DWORD64 zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD64)); - } - else { - DWORD64 d = PyLong_AsUnsignedLongLong(value); + case REG_QWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD64 d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLongLong(value); + if (d == (DWORD64)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD64)); + *retDataSize = sizeof(DWORD64); + break; } - break; case REG_SZ: case REG_EXPAND_SZ: { From webhook-mailer at python.org Fri Dec 9 08:13:13 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 09 Dec 2022 13:13:13 -0000 Subject: [Python-checkins] bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) Message-ID: <mailman.2880.1670591593.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3939a4b7d95a403ee587ff5d222db53ec83bec2c commit: 3939a4b7d95a403ee587ff5d222db53ec83bec2c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-09T05:12:43-08:00 summary: bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) (cherry picked from commit a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1) Co-authored-by: Shreyan Avigyan <shreyan.avigyan at gmail.com> files: A Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst M Lib/test/test_winreg.py M PC/winreg.c diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 8157c2da6efa..769ab67b0f56 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -113,7 +113,6 @@ def _write_test_data(self, root_key, subkeystr="sub_key", "does not close the actual key!") except OSError: pass - def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): # Check we can get default value for this key. val = QueryValue(root_key, test_key_name) @@ -340,6 +339,23 @@ def test_setvalueex_value_range(self): finally: DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_return_value(self): # Test for Issue #16759, return unsigned int from QueryValueEx. # Reg2Py, which gets called by QueryValueEx, was returning a value diff --git a/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst new file mode 100644 index 000000000000..a5975b2d00c7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst @@ -0,0 +1,3 @@ +:meth:`winreg.SetValueEx` now leaves the target value untouched in the case of conversion errors. +Previously, ``-1`` would be written in case of such errors. + diff --git a/PC/winreg.c b/PC/winreg.c index 817717a50b3d..f668cf3c19ca 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -564,42 +564,54 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) { Py_ssize_t i,j; switch (typ) { - case REG_DWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD); - if (value == Py_None) { - DWORD zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD)); - } - else { - DWORD d = PyLong_AsUnsignedLong(value); + case REG_DWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLong(value); + if (d == (DWORD)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD)); + *retDataSize = sizeof(DWORD); + break; } - break; - case REG_QWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD64); - if (value == Py_None) { - DWORD64 zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD64)); - } - else { - DWORD64 d = PyLong_AsUnsignedLongLong(value); + case REG_QWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD64 d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLongLong(value); + if (d == (DWORD64)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD64)); + *retDataSize = sizeof(DWORD64); + break; } - break; case REG_SZ: case REG_EXPAND_SZ: { From webhook-mailer at python.org Fri Dec 9 08:17:31 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 09 Dec 2022 13:17:31 -0000 Subject: [Python-checkins] bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) Message-ID: <mailman.2881.1670591851.3313.python-checkins@python.org> https://github.com/python/cpython/commit/580165d01c579c781c9ba0327f6ff7457201cea8 commit: 580165d01c579c781c9ba0327f6ff7457201cea8 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-09T05:17:25-08:00 summary: bpo-43984: Allow winreg.SetValueEx to set -1 without treating it as an error (GH-25775) (cherry picked from commit a29a7b9b786d6b928c4bb4e6e683a3788e3ab1c1) Co-authored-by: Shreyan Avigyan <shreyan.avigyan at gmail.com> files: A Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst M Lib/test/test_winreg.py M PC/winreg.c diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 8157c2da6efa..769ab67b0f56 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -113,7 +113,6 @@ def _write_test_data(self, root_key, subkeystr="sub_key", "does not close the actual key!") except OSError: pass - def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): # Check we can get default value for this key. val = QueryValue(root_key, test_key_name) @@ -340,6 +339,23 @@ def test_setvalueex_value_range(self): finally: DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_return_value(self): # Test for Issue #16759, return unsigned int from QueryValueEx. # Reg2Py, which gets called by QueryValueEx, was returning a value diff --git a/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst new file mode 100644 index 000000000000..a5975b2d00c7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-05-02-15-29-33.bpo-43984.U92jiv.rst @@ -0,0 +1,3 @@ +:meth:`winreg.SetValueEx` now leaves the target value untouched in the case of conversion errors. +Previously, ``-1`` would be written in case of such errors. + diff --git a/PC/winreg.c b/PC/winreg.c index 4fefcdcc942f..29b4029c320e 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -564,42 +564,54 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) { Py_ssize_t i,j; switch (typ) { - case REG_DWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD); - if (value == Py_None) { - DWORD zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD)); - } - else { - DWORD d = PyLong_AsUnsignedLong(value); + case REG_DWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLong(value); + if (d == (DWORD)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD)); + *retDataSize = sizeof(DWORD); + break; } - break; - case REG_QWORD: - if (value != Py_None && !PyLong_Check(value)) - return FALSE; - *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); - if (*retDataBuf == NULL){ - PyErr_NoMemory(); - return FALSE; - } - *retDataSize = sizeof(DWORD64); - if (value == Py_None) { - DWORD64 zero = 0; - memcpy(*retDataBuf, &zero, sizeof(DWORD64)); - } - else { - DWORD64 d = PyLong_AsUnsignedLongLong(value); + case REG_QWORD: + { + if (value != Py_None && !PyLong_Check(value)) { + return FALSE; + } + DWORD64 d; + if (value == Py_None) { + d = 0; + } + else if (PyLong_Check(value)) { + d = PyLong_AsUnsignedLongLong(value); + if (d == (DWORD64)(-1) && PyErr_Occurred()) { + return FALSE; + } + } + *retDataBuf = (BYTE *)PyMem_NEW(DWORD64, 1); + if (*retDataBuf == NULL) { + PyErr_NoMemory(); + return FALSE; + } memcpy(*retDataBuf, &d, sizeof(DWORD64)); + *retDataSize = sizeof(DWORD64); + break; } - break; case REG_SZ: case REG_EXPAND_SZ: { From webhook-mailer at python.org Fri Dec 9 11:14:39 2022 From: webhook-mailer at python.org (iritkatriel) Date: Fri, 09 Dec 2022 16:14:39 -0000 Subject: [Python-checkins] bpo-44512: Fix handling of extrasactions arg to csv.DictWriter with mixed or upper case (#26924) Message-ID: <mailman.2882.1670602479.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d0679c12398579fe9f10e78b6332dded119e4697 commit: d0679c12398579fe9f10e78b6332dded119e4697 branch: main author: andrei kulakov <andrei.avk at gmail.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-09T16:14:33Z summary: bpo-44512: Fix handling of extrasactions arg to csv.DictWriter with mixed or upper case (#26924) files: A Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst M Lib/csv.py M Lib/test/test_csv.py diff --git a/Lib/csv.py b/Lib/csv.py index 309a8f3f4863..4ef8be45ca9e 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -139,7 +139,8 @@ def __init__(self, f, fieldnames, restval="", extrasaction="raise", fieldnames = list(fieldnames) self.fieldnames = fieldnames # list of keys for the dict self.restval = restval # for writing short dicts - if extrasaction.lower() not in ("raise", "ignore"): + extrasaction = extrasaction.lower() + if extrasaction not in ("raise", "ignore"): raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'" % extrasaction) self.extrasaction = extrasaction diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index d64bff13a44e..8289ddb1c3a5 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -762,6 +762,10 @@ def test_write_field_not_in_field_names_raise(self): dictrow = {'f0': 0, 'f1': 1, 'f2': 2, 'f3': 3} self.assertRaises(ValueError, csv.DictWriter.writerow, writer, dictrow) + # see bpo-44512 (differently cased 'raise' should not result in 'ignore') + writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="RAISE") + self.assertRaises(ValueError, csv.DictWriter.writerow, writer, dictrow) + def test_write_field_not_in_field_names_ignore(self): fileobj = StringIO() writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="ignore") @@ -769,6 +773,10 @@ def test_write_field_not_in_field_names_ignore(self): csv.DictWriter.writerow(writer, dictrow) self.assertEqual(fileobj.getvalue(), "1,2\r\n") + # bpo-44512 + writer = csv.DictWriter(fileobj, ['f1', 'f2'], extrasaction="IGNORE") + csv.DictWriter.writerow(writer, dictrow) + def test_dict_reader_fieldnames_accepts_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() diff --git a/Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst b/Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst new file mode 100644 index 000000000000..7f290605934d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-09-10-35-36.bpo-44592.z-P3oe.rst @@ -0,0 +1,2 @@ +Fixes inconsistent handling of case sensitivity of *extrasaction* arg in +:class:`csv.DictWriter`. From webhook-mailer at python.org Fri Dec 9 12:02:41 2022 From: webhook-mailer at python.org (rhettinger) Date: Fri, 09 Dec 2022 17:02:41 -0000 Subject: [Python-checkins] GH-98363: Shrink the physical size as well as the logical size (GH-100138) Message-ID: <mailman.2883.1670605362.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8c215466dbcdfbdedf0faba7c8ae64461069042b commit: 8c215466dbcdfbdedf0faba7c8ae64461069042b branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-09T11:02:35-06:00 summary: GH-98363: Shrink the physical size as well as the logical size (GH-100138) files: M Modules/itertoolsmodule.c diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 60ec11c32d01..c1f1e7320db7 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -182,8 +182,7 @@ batched_next(batchedobject *bo) Py_DECREF(result); return NULL; } - /* Elements in result[i:] are still NULL */ - Py_SET_SIZE(result, i); + _PyTuple_Resize(&result, i); return result; } From webhook-mailer at python.org Fri Dec 9 12:18:14 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 17:18:14 -0000 Subject: [Python-checkins] gh-81057: Fix the wasm32-wasi Buildbot (gh-100139) Message-ID: <mailman.2884.1670606295.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8d0bd93ae28299f9b2cd11d3dd1a49599b62430a commit: 8d0bd93ae28299f9b2cd11d3dd1a49599b62430a branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-09T10:17:54-07:00 summary: gh-81057: Fix the wasm32-wasi Buildbot (gh-100139) The build was broken by gh-100084. https://github.com/python/cpython/issues/81057 files: M Include/internal/pycore_pythread.h M Python/thread_pthread.h diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 4aeb285b89a8..f53921494c15 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -33,7 +33,7 @@ extern "C" { # endif /* _POSIX_THREADS */ #endif /* _POSIX_THREADS */ -#if defined(_POSIX_THREADS) && !defined(HAVE_PTHREAD_STUBS) +#if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) # define _USE_PTHREADS #endif diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index ae312e987bd6..76d6f3bcdf9c 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -132,7 +132,8 @@ init_condattr(void) if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) { condattr_monotonic = &ca; // Use monotonic clock } -#endif +# undef ca +#endif // CONDATTR_MONOTONIC } int From webhook-mailer at python.org Fri Dec 9 12:18:35 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Fri, 09 Dec 2022 17:18:35 -0000 Subject: [Python-checkins] gh-81057: Fix a Reference Leak in the posix Module (gh-100140) Message-ID: <mailman.2885.1670606316.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7a0f3c1d92ef0768e082ace19d970b0ef12e7346 commit: 7a0f3c1d92ef0768e082ace19d970b0ef12e7346 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-09T10:18:29-07:00 summary: gh-81057: Fix a Reference Leak in the posix Module (gh-100140) The leak was introduced in gh-100082. https://github.com/python/cpython/issues/81057 files: M Modules/posixmodule.c diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f5175350e12a..4817973262f4 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2242,6 +2242,7 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } _posixstate *state = get_posix_state(mod); + Py_DECREF(mod); if (state == NULL) { return NULL; } From webhook-mailer at python.org Sat Dec 10 04:16:06 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 09:16:06 -0000 Subject: [Python-checkins] gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (#100052) Message-ID: <mailman.2886.1670663767.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7c0fb71fbfa8682f56c15832e2c793a6180f2ec0 commit: 7c0fb71fbfa8682f56c15832e2c793a6180f2ec0 branch: main author: ram vikram singh <ramvikrams243 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T14:46:00+05:30 summary: gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (#100052) files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 73debe5ceeaf..40f787ffe639 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4743,7 +4743,7 @@ An example of dictionary view usage:: >>> # get back a read-only proxy for the original dictionary >>> values.mapping - mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}) + mappingproxy({'bacon': 1, 'spam': 500}) >>> values.mapping['spam'] 500 From webhook-mailer at python.org Sat Dec 10 04:23:29 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 09:23:29 -0000 Subject: [Python-checkins] gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) Message-ID: <mailman.2887.1670664211.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9aca00341a75b03f08ed232f0a697eddefa5c8eb commit: 9aca00341a75b03f08ed232f0a697eddefa5c8eb branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T01:23:24-08:00 summary: gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) (cherry picked from commit 7c0fb71fbfa8682f56c15832e2c793a6180f2ec0) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 75550d6e4698..03264ebbeab9 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4733,7 +4733,7 @@ An example of dictionary view usage:: >>> # get back a read-only proxy for the original dictionary >>> values.mapping - mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}) + mappingproxy({'bacon': 1, 'spam': 500}) >>> values.mapping['spam'] 500 From webhook-mailer at python.org Sat Dec 10 04:24:51 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 09:24:51 -0000 Subject: [Python-checkins] gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) Message-ID: <mailman.2888.1670664292.3313.python-checkins@python.org> https://github.com/python/cpython/commit/92dd53bf94a2cf7686681c0486f003775692bd4a commit: 92dd53bf94a2cf7686681c0486f003775692bd4a branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T01:24:46-08:00 summary: gh-100049: fix `repr` for `mappingproxy` in dictionary view example doc (GH-100052) (cherry picked from commit 7c0fb71fbfa8682f56c15832e2c793a6180f2ec0) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8f90bd3efa6c..a6a7eea9f071 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4693,7 +4693,7 @@ An example of dictionary view usage:: >>> # get back a read-only proxy for the original dictionary >>> values.mapping - mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}) + mappingproxy({'bacon': 1, 'spam': 500}) >>> values.mapping['spam'] 500 From webhook-mailer at python.org Sat Dec 10 04:36:01 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 09:36:01 -0000 Subject: [Python-checkins] gh-99582: freeze `zipimport` into `_bootstrap_python` (#99583) Message-ID: <mailman.2889.1670664962.3313.python-checkins@python.org> https://github.com/python/cpython/commit/228c92eb5c126130316a32b44a0ce8f28cc5d544 commit: 228c92eb5c126130316a32b44a0ce8f28cc5d544 branch: main author: Kai Zhang <kylerzhang11 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:05:56+05:30 summary: gh-99582: freeze `zipimport` into `_bootstrap_python` (#99583) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst M Makefile.pre.in M Programs/_bootstrap_python.c diff --git a/Makefile.pre.in b/Makefile.pre.in index de42d684f166..815df69ad48c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -976,7 +976,8 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ - Python/frozen_modules/importlib._bootstrap_external.h + Python/frozen_modules/importlib._bootstrap_external.h \ + Python/frozen_modules/zipimport.h Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst new file mode 100644 index 000000000000..320d47cb9cf6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-19-01-11-06.gh-issue-99582.wvOBVy.rst @@ -0,0 +1 @@ +Freeze :mod:`zipimport` module into ``_bootstrap_python``. diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6e1593a0b599..6c388fc7033d 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -12,6 +12,7 @@ /* Includes for frozen modules: */ #include "Python/frozen_modules/importlib._bootstrap.h" #include "Python/frozen_modules/importlib._bootstrap_external.h" +#include "Python/frozen_modules/zipimport.h" /* End includes */ uint32_t _Py_next_func_version = 1; @@ -32,6 +33,7 @@ _Py_Deepfreeze_Fini(void) static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)}, + {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { From webhook-mailer at python.org Sat Dec 10 05:05:29 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 10:05:29 -0000 Subject: [Python-checkins] Fix potential flakiness in `test_run_until_complete_baseexception` (#100148) Message-ID: <mailman.2890.1670666730.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a9bad4d28413666edc57551dd439bca6a6a59dd9 commit: a9bad4d28413666edc57551dd439bca6a6a59dd9 branch: main author: Fantix King <fantix.king at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:35:24+05:30 summary: Fix potential flakiness in `test_run_until_complete_baseexception` (#100148) files: From webhook-mailer at python.org Sat Dec 10 05:12:46 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 10:12:46 -0000 Subject: [Python-checkins] gh-99970 Adding missing `optionflags` parameter in the documentation of `doctest` (#99971) Message-ID: <mailman.2891.1670667167.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e477348f36eff3f63ba509582622ea620fa9ae5b commit: e477348f36eff3f63ba509582622ea620fa9ae5b branch: main author: busywhitespace <busywhitespace at tuta.io> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:42:40+05:30 summary: gh-99970 Adding missing `optionflags` parameter in the documentation of `doctest` (#99971) files: M Doc/library/doctest.rst diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index b6dd7f7e1232..d6e4dca08606 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1057,7 +1057,7 @@ from text files and modules with doctests: from a text file using :func:`DocFileSuite`. -.. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, checker=None) +.. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, optionflags=0, checker=None) Convert doctest tests for a module to a :class:`unittest.TestSuite`. From webhook-mailer at python.org Sat Dec 10 05:20:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 10 Dec 2022 10:20:23 -0000 Subject: [Python-checkins] gh-99728: correct typo in `datetime` format codes documentation (#99750) Message-ID: <mailman.2892.1670667624.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d5f8a2b6ad408368e728a389da918cead3ef7ee9 commit: d5f8a2b6ad408368e728a389da918cead3ef7ee9 branch: main author: Brad Wolfe <brad.wolfe at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-10T15:50:18+05:30 summary: gh-99728: correct typo in `datetime` format codes documentation (#99750) files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 0b39512a2d23..8bfed19d3fd2 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2602,7 +2602,7 @@ Notes: (9) When used with the :meth:`strptime` method, the leading zero is optional - for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. .. rubric:: Footnotes From webhook-mailer at python.org Sat Dec 10 05:28:56 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 10:28:56 -0000 Subject: [Python-checkins] gh-99728: correct typo in `datetime` format codes documentation (GH-99750) Message-ID: <mailman.2893.1670668137.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e6b0bd59481b9bc4570736c1f5ef291dbbe06b8e commit: e6b0bd59481b9bc4570736c1f5ef291dbbe06b8e branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T02:28:51-08:00 summary: gh-99728: correct typo in `datetime` format codes documentation (GH-99750) (cherry picked from commit d5f8a2b6ad408368e728a389da918cead3ef7ee9) Co-authored-by: Brad Wolfe <brad.wolfe at gmail.com> files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c2d7715a52cf..f222ec9bb771 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2570,7 +2570,7 @@ Notes: (9) When used with the :meth:`strptime` method, the leading zero is optional - for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. .. rubric:: Footnotes From webhook-mailer at python.org Sat Dec 10 05:29:10 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 10 Dec 2022 10:29:10 -0000 Subject: [Python-checkins] gh-99728: correct typo in `datetime` format codes documentation (GH-99750) Message-ID: <mailman.2894.1670668151.3313.python-checkins@python.org> https://github.com/python/cpython/commit/606adb4b891b52c8b9a53d29d594e996f117c0b3 commit: 606adb4b891b52c8b9a53d29d594e996f117c0b3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-10T02:29:04-08:00 summary: gh-99728: correct typo in `datetime` format codes documentation (GH-99750) (cherry picked from commit d5f8a2b6ad408368e728a389da918cead3ef7ee9) Co-authored-by: Brad Wolfe <brad.wolfe at gmail.com> files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 8a7d99999f5d..e700b0a1347a 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2589,7 +2589,7 @@ Notes: (9) When used with the :meth:`strptime` method, the leading zero is optional - for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. .. rubric:: Footnotes From webhook-mailer at python.org Sat Dec 10 18:07:11 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 10 Dec 2022 23:07:11 -0000 Subject: [Python-checkins] gh-99941: Ensure that asyncio.Protocol.data_received receives immutable bytes (#100053) Message-ID: <mailman.2895.1670713631.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1bb68ba6d9de6bb7f00aee11d135123163f15887 commit: 1bb68ba6d9de6bb7f00aee11d135123163f15887 branch: main author: DarioDaF <dario.fagotto at gmail.com> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-10T15:07:02-08:00 summary: gh-99941: Ensure that asyncio.Protocol.data_received receives immutable bytes (#100053) files: A Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst M Lib/asyncio/proactor_events.py M Lib/asyncio/streams.py M Lib/test/test_asyncio/test_proactor_events.py diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index c6aab408fc74..1e2a730cf368 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -288,7 +288,8 @@ def _loop_reading(self, fut=None): # we got end-of-file so no need to reschedule a new read return - data = self._data[:length] + # It's a new slice so make it immutable so protocols upstream don't have problems + data = bytes(memoryview(self._data)[:length]) else: # the future will be replaced by next proactor.recv call fut.cancel() diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 3bd99043d096..0f9098b41956 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -688,7 +688,7 @@ async def read(self, n=-1): await self._wait_for_data('read') # This will work right even if buffer is less than n bytes - data = bytes(self._buffer[:n]) + data = bytes(memoryview(self._buffer)[:n]) del self._buffer[:n] self._maybe_resume_transport() @@ -730,7 +730,7 @@ async def readexactly(self, n): data = bytes(self._buffer) self._buffer.clear() else: - data = bytes(self._buffer[:n]) + data = bytes(memoryview(self._buffer)[:n]) del self._buffer[:n] self._maybe_resume_transport() return data diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index ae30185cef77..6cb7dc300c53 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -75,7 +75,10 @@ def test_loop_reading_data(self): called_buf = bytearray(self.buffer_size) called_buf[:len(buf)] = buf self.loop._proactor.recv_into.assert_called_with(self.sock, called_buf) - self.protocol.data_received.assert_called_with(bytearray(buf)) + self.protocol.data_received.assert_called_with(buf) + # assert_called_with maps bytearray and bytes to the same thing so check manually + # regression test for https://github.com/python/cpython/issues/99941 + self.assertIsInstance(self.protocol.data_received.call_args.args[0], bytes) @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") def test_loop_reading_no_data(self): diff --git a/Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst b/Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst new file mode 100644 index 000000000000..a019d7287207 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2022-12-06-11-16-46.gh-issue-99941.GmUQ6o.rst @@ -0,0 +1,2 @@ +Ensure that :func:`asyncio.Protocol.data_received` receives an immutable +:class:`bytes` object (as documented), instead of :class:`bytearray`. From webhook-mailer at python.org Sat Dec 10 19:17:44 2022 From: webhook-mailer at python.org (gpshead) Date: Sun, 11 Dec 2022 00:17:44 -0000 Subject: [Python-checkins] gh-88500: Reduce memory use of `urllib.unquote` (#96763) Message-ID: <mailman.2896.1670717866.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2e279e85fece187b6058718ac7e82d1692461e26 commit: 2e279e85fece187b6058718ac7e82d1692461e26 branch: main author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-10T16:17:39-08:00 summary: gh-88500: Reduce memory use of `urllib.unquote` (#96763) `urllib.unquote_to_bytes` and `urllib.unquote` could both potentially generate `O(len(string))` intermediate `bytes` or `str` objects while computing the unquoted final result depending on the input provided. As Python objects are relatively large, this could consume a lot of ram. This switches the implementation to using an expanding `bytearray` and a generator internally instead of precomputed `split()` style operations. Microbenchmarks with some antagonistic inputs like `mess = "\u0141%%%20a%fe"*1000` show this is 10-20% slower for unquote and unquote_to_bytes and no different for typical inputs that are short or lack much unicode or % escaping. But the functions are already quite fast anyways so not a big deal. The slowdown scales consistently linear with input size as expected. Memory usage observed manually using `/usr/bin/time -v` on `python -m timeit` runs of larger inputs. Unittesting memory consumption is difficult and does not seem worthwhile. Observed memory usage is ~1/2 for `unquote()` and <1/3 for `unquote_to_bytes()` using `python -m timeit -s 'from urllib.parse import unquote, unquote_to_bytes; v="\u0141%01\u0161%20"*500_000' 'unquote_to_bytes(v)'` as a test. files: A Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst M Lib/test/test_urllib.py M Lib/urllib/parse.py diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index f067560ca6ca..2df74f5e6f99 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1104,6 +1104,8 @@ def test_unquoting(self): self.assertEqual(result.count('%'), 1, "using unquote(): not all characters escaped: " "%s" % result) + + def test_unquote_rejects_none_and_tuple(self): self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, None) self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, ()) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 4f6867accbc0..5f95c5ff7f9c 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -600,6 +600,9 @@ def urldefrag(url): def unquote_to_bytes(string): """unquote_to_bytes('abc%20def') -> b'abc def'.""" + return bytes(_unquote_impl(string)) + +def _unquote_impl(string: bytes | bytearray | str) -> bytes | bytearray: # Note: strings are encoded as UTF-8. This is only an issue if it contains # unescaped non-ASCII characters, which URIs should not. if not string: @@ -611,8 +614,8 @@ def unquote_to_bytes(string): bits = string.split(b'%') if len(bits) == 1: return string - res = [bits[0]] - append = res.append + res = bytearray(bits[0]) + append = res.extend # Delay the initialization of the table to not waste memory # if the function is never called global _hextobyte @@ -626,10 +629,20 @@ def unquote_to_bytes(string): except KeyError: append(b'%') append(item) - return b''.join(res) + return res _asciire = re.compile('([\x00-\x7f]+)') +def _generate_unquoted_parts(string, encoding, errors): + previous_match_end = 0 + for ascii_match in _asciire.finditer(string): + start, end = ascii_match.span() + yield string[previous_match_end:start] # Non-ASCII + # The ascii_match[1] group == string[start:end]. + yield _unquote_impl(ascii_match[1]).decode(encoding, errors) + previous_match_end = end + yield string[previous_match_end:] # Non-ASCII tail + def unquote(string, encoding='utf-8', errors='replace'): """Replace %xx escapes by their single-character equivalent. The optional encoding and errors parameters specify how to decode percent-encoded @@ -641,21 +654,16 @@ def unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ if isinstance(string, bytes): - return unquote_to_bytes(string).decode(encoding, errors) + return _unquote_impl(string).decode(encoding, errors) if '%' not in string: + # Is it a string-like object? string.split return string if encoding is None: encoding = 'utf-8' if errors is None: errors = 'replace' - bits = _asciire.split(string) - res = [bits[0]] - append = res.append - for i in range(1, len(bits), 2): - append(unquote_to_bytes(bits[i]).decode(encoding, errors)) - append(bits[i + 1]) - return ''.join(res) + return ''.join(_generate_unquoted_parts(string, encoding, errors)) def parse_qs(qs, keep_blank_values=False, strict_parsing=False, diff --git a/Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst b/Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst new file mode 100644 index 000000000000..ad01f5e16b16 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-09-16-08-21-46.gh-issue-88500.jQ0pCc.rst @@ -0,0 +1,2 @@ +Reduced the memory usage of :func:`urllib.parse.unquote` and +:func:`urllib.parse.unquote_to_bytes` on large values. From webhook-mailer at python.org Sun Dec 11 18:21:31 2022 From: webhook-mailer at python.org (ethanfurman) Date: Sun, 11 Dec 2022 23:21:31 -0000 Subject: [Python-checkins] gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Message-ID: <mailman.2897.1670800892.3313.python-checkins@python.org> https://github.com/python/cpython/commit/868bab0fdc514cfa70ce97e484a689aee8cb5a36 commit: 868bab0fdc514cfa70ce97e484a689aee8cb5a36 branch: main author: Beweeted <Beweeted at users.noreply.github.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-11T15:20:59-08:00 summary: gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Changed from multiples of 3 to powers of 3 to match the class name. files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 295152c69082..25a6e1f0b616 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -310,12 +310,12 @@ Data Types >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): - ... return (count + 1) * 3 + ... return 3 ** (count + 1) ... FIRST = auto() ... SECOND = auto() ... >>> PowersOfThree.SECOND.value - 6 + 9 .. method:: Enum.__init_subclass__(cls, **kwds) From webhook-mailer at python.org Sun Dec 11 18:30:33 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 11 Dec 2022 23:30:33 -0000 Subject: [Python-checkins] gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Message-ID: <mailman.2898.1670801434.3313.python-checkins@python.org> https://github.com/python/cpython/commit/593c5a02470c063e1dcaf5df5f397294067d0a80 commit: 593c5a02470c063e1dcaf5df5f397294067d0a80 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-11T15:30:25-08:00 summary: gh-100174: [Enum] Correct PowersOfThree example. (GH-100178) Changed from multiples of 3 to powers of 3 to match the class name. (cherry picked from commit 868bab0fdc514cfa70ce97e484a689aee8cb5a36) Co-authored-by: Beweeted <Beweeted at users.noreply.github.com> files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index be0686203bd1..b7b4b949790b 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -316,11 +316,11 @@ Data Types >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): - ... return (count + 1) * 3 + ... return 3 ** (count + 1) ... FIRST = auto() ... SECOND = auto() >>> PowersOfThree.SECOND.value - 6 + 9 .. method:: Enum.__init_subclass__(cls, **kwds) From webhook-mailer at python.org Sun Dec 11 18:44:34 2022 From: webhook-mailer at python.org (sweeneyde) Date: Sun, 11 Dec 2022 23:44:34 -0000 Subject: [Python-checkins] gh-99688: Fix outdated tests in test_unary (#99712) Message-ID: <mailman.2899.1670802275.3313.python-checkins@python.org> https://github.com/python/cpython/commit/54289f85b2af1ecf046089ddf535dda1bdf6af24 commit: 54289f85b2af1ecf046089ddf535dda1bdf6af24 branch: main author: Yesung(Isaac) Lee <yy0221ss at gmail.com> committer: sweeneyde <36520290+sweeneyde at users.noreply.github.com> date: 2022-12-11T18:44:29-05:00 summary: gh-99688: Fix outdated tests in test_unary (#99712) * Remove duplicates from "L" suffix removal * test_invert now tests `~`. files: M Lib/test/test_unary.py diff --git a/Lib/test/test_unary.py b/Lib/test/test_unary.py index c3c17cc9f611..a45fbf6bd6bc 100644 --- a/Lib/test/test_unary.py +++ b/Lib/test/test_unary.py @@ -8,7 +8,6 @@ def test_negative(self): self.assertTrue(-2 == 0 - 2) self.assertEqual(-0, 0) self.assertEqual(--2, 2) - self.assertTrue(-2 == 0 - 2) self.assertTrue(-2.0 == 0 - 2.0) self.assertTrue(-2j == 0 - 2j) @@ -16,15 +15,13 @@ def test_positive(self): self.assertEqual(+2, 2) self.assertEqual(+0, 0) self.assertEqual(++2, 2) - self.assertEqual(+2, 2) self.assertEqual(+2.0, 2.0) self.assertEqual(+2j, 2j) def test_invert(self): - self.assertTrue(-2 == 0 - 2) - self.assertEqual(-0, 0) - self.assertEqual(--2, 2) - self.assertTrue(-2 == 0 - 2) + self.assertTrue(~2 == -(2+1)) + self.assertEqual(~0, -1) + self.assertEqual(~~2, 2) def test_no_overflow(self): nines = "9" * 32 From webhook-mailer at python.org Sun Dec 11 23:16:02 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Mon, 12 Dec 2022 04:16:02 -0000 Subject: [Python-checkins] gh-70393: Clarify mention of "middle" scope (#98839) Message-ID: <mailman.2900.1670818563.3313.python-checkins@python.org> https://github.com/python/cpython/commit/70be5e42f6e288de32e0df3c77ac22a9ddf1a74b commit: 70be5e42f6e288de32e0df3c77ac22a9ddf1a74b branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-11T20:15:55-08:00 summary: gh-70393: Clarify mention of "middle" scope (#98839) files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 9ecbf8b87efb..0e5a9402bc50 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -119,12 +119,12 @@ directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contains non-local, but also non-global names + nearest enclosing scope, contain non-local, but also non-global names * the next-to-last scope contains the current module's global names * the outermost scope (searched last) is the namespace containing built-in names If a name is declared global, then all references and assignments go directly to -the middle scope containing the module's global names. To rebind variables +the next-to-last scope containing the module's global names. To rebind variables found outside of the innermost scope, the :keyword:`nonlocal` statement can be used; if not declared nonlocal, those variables are read-only (an attempt to write to such a variable will simply create a *new* local variable in the From webhook-mailer at python.org Sun Dec 11 23:25:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 04:25:11 -0000 Subject: [Python-checkins] gh-70393: Clarify mention of "middle" scope (GH-98839) Message-ID: <mailman.2901.1670819112.3313.python-checkins@python.org> https://github.com/python/cpython/commit/71b032635d757348cb915c2e8d99392f5bbcb059 commit: 71b032635d757348cb915c2e8d99392f5bbcb059 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-11T20:25:06-08:00 summary: gh-70393: Clarify mention of "middle" scope (GH-98839) (cherry picked from commit 70be5e42f6e288de32e0df3c77ac22a9ddf1a74b) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index f27abe48b2d4..d7a24b4893fb 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -119,12 +119,12 @@ directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contains non-local, but also non-global names + nearest enclosing scope, contain non-local, but also non-global names * the next-to-last scope contains the current module's global names * the outermost scope (searched last) is the namespace containing built-in names If a name is declared global, then all references and assignments go directly to -the middle scope containing the module's global names. To rebind variables +the next-to-last scope containing the module's global names. To rebind variables found outside of the innermost scope, the :keyword:`nonlocal` statement can be used; if not declared nonlocal, those variables are read-only (an attempt to write to such a variable will simply create a *new* local variable in the From webhook-mailer at python.org Sun Dec 11 23:26:36 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 04:26:36 -0000 Subject: [Python-checkins] gh-70393: Clarify mention of "middle" scope (GH-98839) Message-ID: <mailman.2902.1670819196.3313.python-checkins@python.org> https://github.com/python/cpython/commit/eece1bdca7d04184a298a7045f389d89f3e5e8a8 commit: eece1bdca7d04184a298a7045f389d89f3e5e8a8 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-11T20:26:30-08:00 summary: gh-70393: Clarify mention of "middle" scope (GH-98839) (cherry picked from commit 70be5e42f6e288de32e0df3c77ac22a9ddf1a74b) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index f27abe48b2d4..d7a24b4893fb 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -119,12 +119,12 @@ directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contains non-local, but also non-global names + nearest enclosing scope, contain non-local, but also non-global names * the next-to-last scope contains the current module's global names * the outermost scope (searched last) is the namespace containing built-in names If a name is declared global, then all references and assignments go directly to -the middle scope containing the module's global names. To rebind variables +the next-to-last scope containing the module's global names. To rebind variables found outside of the innermost scope, the :keyword:`nonlocal` statement can be used; if not declared nonlocal, those variables are read-only (an attempt to write to such a variable will simply create a *new* local variable in the From webhook-mailer at python.org Mon Dec 12 06:40:00 2022 From: webhook-mailer at python.org (hugovk) Date: Mon, 12 Dec 2022 11:40:00 -0000 Subject: [Python-checkins] clarify the 4300-digit limit on int-str conversion (#100175) Message-ID: <mailman.2903.1670845201.3313.python-checkins@python.org> https://github.com/python/cpython/commit/935ef593211a627526b2b869ce1fc2a5e67e6cdd commit: 935ef593211a627526b2b869ce1fc2a5e67e6cdd branch: main author: Ned Batchelder <ned at nedbatchelder.com> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-12T13:39:54+02:00 summary: clarify the 4300-digit limit on int-str conversion (#100175) files: M Doc/library/stdtypes.rst M Objects/longobject.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 40f787ffe639..c785336944f5 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5503,7 +5503,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> _ = int('2' * 5432) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. >>> i = int('2' * 4300) >>> len(str(i)) 4300 @@ -5511,7 +5511,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> len(str(i_squared)) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. >>> len(hex(i_squared)) 7144 >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. diff --git a/Objects/longobject.c b/Objects/longobject.c index c84b4d3f316d..8596ce9797b5 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -36,8 +36,8 @@ medium_value(PyLongObject *x) #define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) -#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" -#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d digits) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" /* If defined, use algorithms from the _pylong.py module */ #define WITH_PYLONG_MODULE 1 From webhook-mailer at python.org Mon Dec 12 06:52:33 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 12 Dec 2022 11:52:33 -0000 Subject: [Python-checkins] gh-100176: remove incorrect version compatibility check from argument clinic (#100190) Message-ID: <mailman.2904.1670845955.3313.python-checkins@python.org> https://github.com/python/cpython/commit/621a1790c4d9e316c7228a13735bf71f2b0f5372 commit: 621a1790c4d9e316c7228a13735bf71f2b0f5372 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-12T17:22:12+05:30 summary: gh-100176: remove incorrect version compatibility check from argument clinic (#100190) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0ece814e8f18..fdf8041e14bb 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5212,10 +5212,6 @@ def state_terminal(self, line): def main(argv): import sys - - if sys.version_info.major < 3 or sys.version_info.minor < 3: - sys.exit("Error: clinic.py requires Python 3.3 or greater.") - import argparse cmdline = argparse.ArgumentParser( description="""Preprocessor for CPython C files. From webhook-mailer at python.org Mon Dec 12 06:59:34 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 12 Dec 2022 11:59:34 -0000 Subject: [Python-checkins] gh-96715 Remove redundant NULL check in `profile_trampoline` function (#96716) Message-ID: <mailman.2905.1670846374.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3221b0de6792145cb4d0d461065a956db82acc93 commit: 3221b0de6792145cb4d0d461065a956db82acc93 branch: main author: chgnrdv <52372310+chgnrdv at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-12T17:29:27+05:30 summary: gh-96715 Remove redundant NULL check in `profile_trampoline` function (#96716) Closes #96715 files: M Python/sysmodule.c diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 88f806e616f2..91f5c487c98f 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -950,10 +950,6 @@ static int profile_trampoline(PyObject *self, PyFrameObject *frame, int what, PyObject *arg) { - if (arg == NULL) { - arg = Py_None; - } - PyThreadState *tstate = _PyThreadState_GET(); PyObject *result = call_trampoline(tstate, self, frame, what, arg); if (result == NULL) { From webhook-mailer at python.org Mon Dec 12 07:06:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:06:55 -0000 Subject: [Python-checkins] clarify the 4300-digit limit on int-str conversion (GH-100175) Message-ID: <mailman.2906.1670846815.3313.python-checkins@python.org> https://github.com/python/cpython/commit/17bc55e0430f2fd08bd006364bd127ae1ff63161 commit: 17bc55e0430f2fd08bd006364bd127ae1ff63161 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:06:48-08:00 summary: clarify the 4300-digit limit on int-str conversion (GH-100175) (cherry picked from commit 935ef593211a627526b2b869ce1fc2a5e67e6cdd) Co-authored-by: Ned Batchelder <ned at nedbatchelder.com> files: M Doc/library/stdtypes.rst M Objects/longobject.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 03264ebbeab9..426eec620566 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5493,7 +5493,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> _ = int('2' * 5432) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. >>> i = int('2' * 4300) >>> len(str(i)) 4300 @@ -5501,7 +5501,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> len(str(i_squared)) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. >>> len(hex(i_squared)) 7144 >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. diff --git a/Objects/longobject.c b/Objects/longobject.c index e93461af7275..1b289d5468bd 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -36,8 +36,8 @@ medium_value(PyLongObject *x) #define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) -#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" -#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d digits) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" +#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit" static inline void _Py_DECREF_INT(PyLongObject *op) From webhook-mailer at python.org Mon Dec 12 07:25:28 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:25:28 -0000 Subject: [Python-checkins] Fix: typo (Indention) (GH-99904) Message-ID: <mailman.2907.1670847929.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8711b59f7ac1803307d340e357e025043fbe2f39 commit: 8711b59f7ac1803307d340e357e025043fbe2f39 branch: main author: jarrodcolburn <jcourtlandcolburn at gmail.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:25:22-08:00 summary: Fix: typo (Indention) (GH-99904) Example needed to be indented. Was trying to call a context manger `pr` (from ` with cProfile.Profile() as pr:`) wot perform ` pr.print_stats()` once it had already exited. Automerge-Triggered-By: GH:AlexWaygood files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 2d95096f4cb8..c2189e02656c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -274,7 +274,7 @@ functions: with cProfile.Profile() as pr: # ... do something ... - pr.print_stats() + pr.print_stats() .. versionchanged:: 3.8 Added context manager support. From webhook-mailer at python.org Mon Dec 12 07:38:45 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:38:45 -0000 Subject: [Python-checkins] Fix: typo (Indention) (GH-99904) Message-ID: <mailman.2908.1670848726.3313.python-checkins@python.org> https://github.com/python/cpython/commit/60383a3cd8739cf386ed64477924d0ccca00fe9a commit: 60383a3cd8739cf386ed64477924d0ccca00fe9a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:38:39-08:00 summary: Fix: typo (Indention) (GH-99904) Example needed to be indented. Was trying to call a context manger `pr` (from ` with cProfile.Profile() as pr:`) wot perform ` pr.print_stats()` once it had already exited. (cherry picked from commit 8711b59f7ac1803307d340e357e025043fbe2f39) Co-authored-by: jarrodcolburn <jcourtlandcolburn at gmail.com> Automerge-Triggered-By: GH:AlexWaygood files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 2d95096f4cb8..c2189e02656c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -274,7 +274,7 @@ functions: with cProfile.Profile() as pr: # ... do something ... - pr.print_stats() + pr.print_stats() .. versionchanged:: 3.8 Added context manager support. From webhook-mailer at python.org Mon Dec 12 07:40:15 2022 From: webhook-mailer at python.org (miss-islington) Date: Mon, 12 Dec 2022 12:40:15 -0000 Subject: [Python-checkins] Fix: typo (Indention) (GH-99904) Message-ID: <mailman.2909.1670848815.3313.python-checkins@python.org> https://github.com/python/cpython/commit/24ed439462bdcbca6a23d0517970d9b5d195b2d6 commit: 24ed439462bdcbca6a23d0517970d9b5d195b2d6 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-12T04:40:09-08:00 summary: Fix: typo (Indention) (GH-99904) Example needed to be indented. Was trying to call a context manger `pr` (from ` with cProfile.Profile() as pr:`) wot perform ` pr.print_stats()` once it had already exited. (cherry picked from commit 8711b59f7ac1803307d340e357e025043fbe2f39) Co-authored-by: jarrodcolburn <jcourtlandcolburn at gmail.com> Automerge-Triggered-By: GH:AlexWaygood files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index cf324a57e79f..6f01d1e027ec 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -273,7 +273,7 @@ functions: with cProfile.Profile() as pr: # ... do something ... - pr.print_stats() + pr.print_stats() .. versionchanged:: 3.8 Added context manager support. From webhook-mailer at python.org Mon Dec 12 08:39:31 2022 From: webhook-mailer at python.org (zooba) Date: Mon, 12 Dec 2022 13:39:31 -0000 Subject: [Python-checkins] gh-79218: Define `MS_WIN64` macro for Mingw-w64 64bit on Windows (GH-100137) Message-ID: <mailman.2910.1670852371.3313.python-checkins@python.org> https://github.com/python/cpython/commit/158b8a07212cea6066afe8bb91f1cd542d922dba commit: 158b8a07212cea6066afe8bb91f1cd542d922dba branch: main author: GalaxySnail <me at glxys.nl> committer: zooba <steve.dower at microsoft.com> date: 2022-12-12T13:39:23Z summary: gh-79218: Define `MS_WIN64` macro for Mingw-w64 64bit on Windows (GH-100137) files: A Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst M PC/pyconfig.h diff --git a/Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst b/Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst new file mode 100644 index 000000000000..e2e6ca3c7796 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2022-12-09-22-47-42.gh-issue-79218.Yiot2e.rst @@ -0,0 +1 @@ +Define ``MS_WIN64`` for Mingw-w64 64bit, fix cython compilation failure. diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 1a33d4c5a1e4..1d8408b363a6 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -209,6 +209,16 @@ typedef int pid_t; #endif /* _MSC_VER */ +/* ------------------------------------------------------------------------*/ +/* mingw and mingw-w64 define __MINGW32__ */ +#ifdef __MINGW32__ + +#ifdef _WIN64 +#define MS_WIN64 +#endif + +#endif /* __MINGW32__*/ + /* ------------------------------------------------------------------------*/ /* egcs/gnu-win32 defines __GNUC__ and _WIN32 */ #if defined(__GNUC__) && defined(_WIN32) From webhook-mailer at python.org Mon Dec 12 09:22:56 2022 From: webhook-mailer at python.org (iritkatriel) Date: Mon, 12 Dec 2022 14:22:56 -0000 Subject: [Python-checkins] gh-99955: standardize return values of functions in compiler's code-gen (#100010) Message-ID: <mailman.2911.1670854977.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e4ea33b17807d99ed737f800d9b0006957c008d2 commit: e4ea33b17807d99ed737f800d9b0006957c008d2 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-12T14:22:15Z summary: gh-99955: standardize return values of functions in compiler's code-gen (#100010) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst M Python/compile.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst new file mode 100644 index 000000000000..e9867b39ad7d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-11-27-54.gh-issue-99955.Ix5Rrg.rst @@ -0,0 +1 @@ +Internal compiler functions (in compile.c) now consistently return -1 on error and 0 on success. diff --git a/Python/compile.c b/Python/compile.c index d6ed6941ac1e..17b164a4d06e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -55,6 +55,14 @@ */ #define STACK_USE_GUIDELINE 30 +#define SUCCESS 0 +#define ERROR -1 + +#define RETURN_IF_ERROR(X) \ + if ((X) == -1) { \ + return ERROR; \ + } + /* If we exceed this limit, it should * be considered a compiler bug. * Currently it should be impossible @@ -498,7 +506,7 @@ static int compiler_annassign(struct compiler *, stmt_ty); static int compiler_subscript(struct compiler *, expr_ty); static int compiler_slice(struct compiler *, expr_ty); -static int are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); +static bool are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); static int compiler_with(struct compiler *, stmt_ty, int); @@ -610,18 +618,18 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, { c->c_const_cache = PyDict_New(); if (!c->c_const_cache) { - return 0; + return ERROR; } c->c_stack = PyList_New(0); if (!c->c_stack) { - return 0; + return ERROR; } c->c_filename = Py_NewRef(filename); c->c_arena = arena; if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { - return 0; + return ERROR; } int merged = c->c_future.ff_features | flags.cf_flags; c->c_future.ff_features = merged; @@ -635,16 +643,16 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, state.ff_features = merged; if (!_PyAST_Optimize(mod, arena, &state)) { - return 0; + return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); if (c->c_st == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_SystemError, "no symtable"); } - return 0; + return ERROR; } - return 1; + return SUCCESS; } static struct compiler* @@ -656,7 +664,7 @@ new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, if (c == NULL) { return NULL; } - if (!compiler_setup(c, mod, filename, flags, optimize, arena)) { + if (compiler_setup(c, mod, filename, flags, optimize, arena) < 0) { compiler_free(c); return NULL; } @@ -800,11 +808,12 @@ cfg_builder_init(cfg_builder *g) { g->g_block_list = NULL; basicblock *block = cfg_builder_new_block(g); - if (block == NULL) - return 0; + if (block == NULL) { + return ERROR; + } g->g_curblock = g->g_entryblock = block; g->g_current_label = NO_LABEL; - return 1; + return SUCCESS; } static void @@ -862,8 +871,10 @@ compiler_set_qualname(struct compiler *c) || u->u_scope_type == COMPILER_SCOPE_CLASS) { assert(u->u_name); mangled = _Py_Mangle(parent->u_private, u->u_name); - if (!mangled) - return 0; + if (!mangled) { + return ERROR; + } + scope = _PyST_GetScope(parent->u_ste, mangled); Py_DECREF(mangled); assert(scope != GLOBAL_IMPLICIT); @@ -879,8 +890,9 @@ compiler_set_qualname(struct compiler *c) _Py_DECLARE_STR(dot_locals, ".<locals>"); base = PyUnicode_Concat(parent->u_qualname, &_Py_STR(dot_locals)); - if (base == NULL) - return 0; + if (base == NULL) { + return ERROR; + } } else { base = Py_NewRef(parent->u_qualname); @@ -892,18 +904,20 @@ compiler_set_qualname(struct compiler *c) _Py_DECLARE_STR(dot, "."); name = PyUnicode_Concat(base, &_Py_STR(dot)); Py_DECREF(base); - if (name == NULL) - return 0; + if (name == NULL) { + return ERROR; + } PyUnicode_Append(&name, u->u_name); - if (name == NULL) - return 0; + if (name == NULL) { + return ERROR; + } } else { name = Py_NewRef(u->u_name); } u->u_qualname = name; - return 1; + return SUCCESS; } static jump_target_label @@ -1304,10 +1318,6 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return stack_effect(opcode, oparg, -1); } -/* Add an opcode with no argument. - Returns 0 on failure, 1 on success. -*/ - static int basicblock_addop(basicblock *b, int opcode, int oparg, location loc) { @@ -1318,7 +1328,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) int off = basicblock_next_instr(b); if (off < 0) { - return 0; + return ERROR; } struct instr *i = &b->b_instr[off]; i->i_opcode = opcode; @@ -1326,7 +1336,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) i->i_target = NULL; i->i_loc = loc; - return 1; + return SUCCESS; } static bool @@ -1522,8 +1532,9 @@ static int compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) { Py_ssize_t arg = compiler_add_const(c, o); - if (arg < 0) - return 0; + if (arg < 0) { + return ERROR; + } return cfg_builder_addop_i(CFG_BUILDER(c), LOAD_CONST, arg, loc); } @@ -1532,8 +1543,9 @@ compiler_addop_o(struct compiler *c, location loc, int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); - if (arg < 0) - return 0; + if (arg < 0) { + return ERROR; + } return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); } @@ -1544,12 +1556,14 @@ compiler_addop_name(struct compiler *c, location loc, Py_ssize_t arg; PyObject *mangled = _Py_Mangle(c->u->u_private, o); - if (!mangled) - return 0; + if (!mangled) { + return ERROR; + } arg = dict_add_o(dict, mangled); Py_DECREF(mangled); - if (arg < 0) - return 0; + if (arg < 0) { + return ERROR; + } if (opcode == LOAD_ATTR) { arg <<= 1; } @@ -1561,9 +1575,7 @@ compiler_addop_name(struct compiler *c, location loc, return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); } -/* Add an opcode with an integer argument. - Returns 0 on failure, 1 on success. -*/ +/* Add an opcode with an integer argument */ static int cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) { @@ -1588,94 +1600,79 @@ cfg_builder_addop_j(cfg_builder *g, location loc, return cfg_builder_addop(g, opcode, target.id, loc); } - -#define ADDOP(C, LOC, OP) { \ - if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) \ - return 0; \ -} +#define ADDOP(C, LOC, OP) \ + RETURN_IF_ERROR(cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) #define ADDOP_IN_SCOPE(C, LOC, OP) { \ - if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) { \ + if (cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC)) < 0) { \ compiler_exit_scope(c); \ - return 0; \ + return -1; \ } \ } -#define ADDOP_LOAD_CONST(C, LOC, O) { \ - if (!compiler_addop_load_const((C), (LOC), (O))) \ - return 0; \ -} +#define ADDOP_LOAD_CONST(C, LOC, O) \ + RETURN_IF_ERROR(compiler_addop_load_const((C), (LOC), (O))) /* Same as ADDOP_LOAD_CONST, but steals a reference. */ #define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ PyObject *__new_const = (O); \ if (__new_const == NULL) { \ - return 0; \ + return ERROR; \ } \ - if (!compiler_addop_load_const((C), (LOC), __new_const)) { \ + if (compiler_addop_load_const((C), (LOC), __new_const) < 0) { \ Py_DECREF(__new_const); \ - return 0; \ + return ERROR; \ } \ Py_DECREF(__new_const); \ } #define ADDOP_N(C, LOC, OP, O, TYPE) { \ assert(!HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (!compiler_addop_o((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) { \ + if (compiler_addop_o((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O)) < 0) { \ Py_DECREF((O)); \ - return 0; \ + return ERROR; \ } \ Py_DECREF((O)); \ } -#define ADDOP_NAME(C, LOC, OP, O, TYPE) { \ - if (!compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) \ - return 0; \ -} +#define ADDOP_NAME(C, LOC, OP, O, TYPE) \ + RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) -#define ADDOP_I(C, LOC, OP, O) { \ - if (!cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) \ - return 0; \ -} +#define ADDOP_I(C, LOC, OP, O) \ + RETURN_IF_ERROR(cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) -#define ADDOP_JUMP(C, LOC, OP, O) { \ - if (!cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) \ - return 0; \ -} +#define ADDOP_JUMP(C, LOC, OP, O) \ + RETURN_IF_ERROR(cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) -#define ADDOP_COMPARE(C, LOC, CMP) { \ - if (!compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) \ - return 0; \ -} +#define ADDOP_COMPARE(C, LOC, CMP) \ + RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) #define ADDOP_BINARY(C, LOC, BINOP) \ - RETURN_IF_FALSE(addop_binary((C), (LOC), (BINOP), false)) + RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false)) #define ADDOP_INPLACE(C, LOC, BINOP) \ - RETURN_IF_FALSE(addop_binary((C), (LOC), (BINOP), true)) + RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true)) #define ADD_YIELD_FROM(C, LOC, await) \ - RETURN_IF_FALSE(compiler_add_yield_from((C), (LOC), (await))) + RETURN_IF_ERROR(compiler_add_yield_from((C), (LOC), (await))) #define POP_EXCEPT_AND_RERAISE(C, LOC) \ - RETURN_IF_FALSE(compiler_pop_except_and_reraise((C), (LOC))) + RETURN_IF_ERROR(compiler_pop_except_and_reraise((C), (LOC))) #define ADDOP_YIELD(C, LOC) \ - RETURN_IF_FALSE(addop_yield((C), (LOC))) + RETURN_IF_ERROR(addop_yield((C), (LOC))) /* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. */ -#define VISIT(C, TYPE, V) {\ - if (!compiler_visit_ ## TYPE((C), (V))) \ - return 0; \ -} +#define VISIT(C, TYPE, V) \ + RETURN_IF_ERROR(compiler_visit_ ## TYPE((C), (V))); #define VISIT_IN_SCOPE(C, TYPE, V) {\ - if (!compiler_visit_ ## TYPE((C), (V))) { \ + if (compiler_visit_ ## TYPE((C), (V)) < 0) { \ compiler_exit_scope(c); \ - return 0; \ + return ERROR; \ } \ } @@ -1684,8 +1681,8 @@ cfg_builder_addop_j(cfg_builder *g, location loc, asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - if (!compiler_visit_ ## TYPE((C), elt)) \ - return 0; \ + if (compiler_visit_ ## TYPE((C), elt) < 0) \ + return ERROR; \ } \ } @@ -1694,17 +1691,13 @@ cfg_builder_addop_j(cfg_builder *g, location loc, asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - if (!compiler_visit_ ## TYPE((C), elt)) { \ + if (compiler_visit_ ## TYPE((C), elt) < 0) { \ compiler_exit_scope(c); \ - return 0; \ + return ERROR; \ } \ } \ } -#define RETURN_IF_FALSE(X) \ - if (!(X)) { \ - return 0; \ - } static int compiler_enter_scope(struct compiler *c, identifier name, @@ -1718,7 +1711,7 @@ compiler_enter_scope(struct compiler *c, identifier name, struct compiler_unit)); if (!u) { PyErr_NoMemory(); - return 0; + return ERROR; } u->u_scope_type = scope_type; u->u_argcount = 0; @@ -1727,14 +1720,14 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_ste = PySymtable_Lookup(c->c_st, key); if (!u->u_ste) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_name = Py_NewRef(name); u->u_varnames = list2dict(u->u_ste->ste_varnames); u->u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, 0, 0); if (!u->u_varnames || !u->u_cellvars) { compiler_unit_free(u); - return 0; + return ERROR; } if (u->u_ste->ste_needs_class_closure) { /* Cook up an implicit __class__ cell. */ @@ -1745,7 +1738,7 @@ compiler_enter_scope(struct compiler *c, identifier name, _PyLong_GetZero()); if (res < 0) { compiler_unit_free(u); - return 0; + return ERROR; } } @@ -1753,7 +1746,7 @@ compiler_enter_scope(struct compiler *c, identifier name, PyDict_GET_SIZE(u->u_cellvars)); if (!u->u_freevars) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_nfblocks = 0; @@ -1761,12 +1754,12 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_consts = PyDict_New(); if (!u->u_consts) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_names = PyDict_New(); if (!u->u_names) { compiler_unit_free(u); - return 0; + return ERROR; } u->u_private = NULL; @@ -1777,7 +1770,7 @@ compiler_enter_scope(struct compiler *c, identifier name, if (!capsule || PyList_Append(c->c_stack, capsule) < 0) { Py_XDECREF(capsule); compiler_unit_free(u); - return 0; + return ERROR; } Py_DECREF(capsule); u->u_private = Py_XNewRef(c->u->u_private); @@ -1787,23 +1780,20 @@ compiler_enter_scope(struct compiler *c, identifier name, c->c_nestlevel++; cfg_builder *g = CFG_BUILDER(c); - if (!cfg_builder_init(g)) { - return 0; - } + RETURN_IF_ERROR(cfg_builder_init(g)); if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = 0; } else { - if (!compiler_set_qualname(c)) - return 0; + RETURN_IF_ERROR(compiler_set_qualname(c)); } ADDOP_I(c, loc, RESUME, 0); if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = -1; } - return 1; + return SUCCESS; } static void @@ -1837,7 +1827,7 @@ compiler_exit_scope(struct compiler *c) /* Search if variable annotations are present statically in a block. */ -static int +static bool find_ann(asdl_stmt_seq *stmts) { int i, j, res = 0; @@ -1847,7 +1837,7 @@ find_ann(asdl_stmt_seq *stmts) st = (stmt_ty)asdl_seq_GET(stmts, i); switch (st->kind) { case AnnAssign_kind: - return 1; + return true; case For_kind: res = find_ann(st->v.For.body) || find_ann(st->v.For.orelse); @@ -1875,7 +1865,7 @@ find_ann(asdl_stmt_seq *stmts) excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( st->v.Try.handlers, j); if (find_ann(handler->v.ExceptHandler.body)) { - return 1; + return true; } } res = find_ann(st->v.Try.body) || @@ -1887,7 +1877,7 @@ find_ann(asdl_stmt_seq *stmts) excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( st->v.TryStar.handlers, j); if (find_ann(handler->v.ExceptHandler.body)) { - return 1; + return true; } } res = find_ann(st->v.TryStar.body) || @@ -1895,7 +1885,7 @@ find_ann(asdl_stmt_seq *stmts) find_ann(st->v.TryStar.orelse); break; default: - res = 0; + res = false; } if (res) { break; @@ -1922,7 +1912,7 @@ compiler_push_fblock(struct compiler *c, location loc, f->fb_block = block_label; f->fb_exit = exit; f->fb_datum = datum; - return 1; + return SUCCESS; } static void @@ -1942,7 +1932,7 @@ compiler_call_exit_with_nones(struct compiler *c, location loc) ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_I(c, loc, CALL, 2); - return 1; + return SUCCESS; } static int @@ -1966,7 +1956,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) ADDOP(c, loc, CLEANUP_THROW); USE_LABEL(c, exit); - return 1; + return SUCCESS; } static int @@ -1982,7 +1972,7 @@ compiler_pop_except_and_reraise(struct compiler *c, location loc) ADDOP_I(c, loc, COPY, 3); ADDOP(c, loc, POP_EXCEPT); ADDOP_I(c, loc, RERAISE, 1); - return 1; + return SUCCESS; } /* Unwind a frame block. If preserve_tos is true, the TOS before @@ -1999,7 +1989,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, case EXCEPTION_HANDLER: case EXCEPTION_GROUP_HANDLER: case ASYNC_COMPREHENSION_GENERATOR: - return 1; + return SUCCESS; case FOR_LOOP: /* Pop the iterator */ @@ -2007,19 +1997,18 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, ADDOP_I(c, *ploc, SWAP, 2); } ADDOP(c, *ploc, POP_TOP); - return 1; + return SUCCESS; case TRY_EXCEPT: ADDOP(c, *ploc, POP_BLOCK); - return 1; + return SUCCESS; case FINALLY_TRY: /* This POP_BLOCK gets the line number of the unwinding statement */ ADDOP(c, *ploc, POP_BLOCK); if (preserve_tos) { - if (!compiler_push_fblock(c, *ploc, POP_VALUE, NO_LABEL, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, *ploc, POP_VALUE, NO_LABEL, NO_LABEL, NULL)); } /* Emit the finally block */ VISIT_SEQ(c, stmt, info->fb_datum); @@ -2030,7 +2019,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, * statement causing the unwinding, so make the unwinding * instruction artificial */ *ploc = NO_LOCATION; - return 1; + return SUCCESS; case FINALLY_END: if (preserve_tos) { @@ -2042,7 +2031,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, } ADDOP(c, *ploc, POP_BLOCK); ADDOP(c, *ploc, POP_EXCEPT); - return 1; + return SUCCESS; case WITH: case ASYNC_WITH: @@ -2051,9 +2040,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, if (preserve_tos) { ADDOP_I(c, *ploc, SWAP, 2); } - if(!compiler_call_exit_with_nones(c, *ploc)) { - return 0; - } + RETURN_IF_ERROR(compiler_call_exit_with_nones(c, *ploc)); if (info->fb_type == ASYNC_WITH) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, *ploc, Py_None); @@ -2064,7 +2051,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, * statement causing the unwinding, so make the unwinding * instruction artificial */ *ploc = NO_LOCATION; - return 1; + return SUCCESS; case HANDLER_CLEANUP: { if (info->fb_datum) { @@ -2077,17 +2064,17 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, ADDOP(c, *ploc, POP_EXCEPT); if (info->fb_datum) { ADDOP_LOAD_CONST(c, *ploc, Py_None); - compiler_nameop(c, *ploc, info->fb_datum, Store); - compiler_nameop(c, *ploc, info->fb_datum, Del); + RETURN_IF_ERROR(compiler_nameop(c, *ploc, info->fb_datum, Store)); + RETURN_IF_ERROR(compiler_nameop(c, *ploc, info->fb_datum, Del)); } - return 1; + return SUCCESS; } case POP_VALUE: { if (preserve_tos) { ADDOP_I(c, *ploc, SWAP, 2); } ADDOP(c, *ploc, POP_TOP); - return 1; + return SUCCESS; } } Py_UNREACHABLE(); @@ -2099,7 +2086,7 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, int preserve_tos, struct fblockinfo **loop) { if (c->u->u_nfblocks == 0) { - return 1; + return SUCCESS; } struct fblockinfo *top = &c->u->u_fblock[c->u->u_nfblocks-1]; if (top->fb_type == EXCEPTION_GROUP_HANDLER) { @@ -2108,19 +2095,15 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, } if (loop != NULL && (top->fb_type == WHILE_LOOP || top->fb_type == FOR_LOOP)) { *loop = top; - return 1; + return SUCCESS; } struct fblockinfo copy = *top; c->u->u_nfblocks--; - if (!compiler_unwind_fblock(c, ploc, ©, preserve_tos)) { - return 0; - } - if (!compiler_unwind_fblock_stack(c, ploc, preserve_tos, loop)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock(c, ploc, ©, preserve_tos)); + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, ploc, preserve_tos, loop)); c->u->u_fblock[c->u->u_nfblocks] = copy; c->u->u_nfblocks++; - return 1; + return SUCCESS; } /* Compile a sequence of statements, checking for a docstring @@ -2145,8 +2128,9 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) if (find_ann(stmts)) { ADDOP(c, loc, SETUP_ANNOTATIONS); } - if (!asdl_seq_LEN(stmts)) - return 1; + if (!asdl_seq_LEN(stmts)) { + return SUCCESS; + } /* if not -OO mode, set docstring */ if (c->c_optimize < 2) { docstring = _PyAST_GetDocString(stmts); @@ -2155,29 +2139,29 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) st = (stmt_ty)asdl_seq_GET(stmts, 0); assert(st->kind == Expr_kind); VISIT(c, expr, st->v.Expr.value); - if (!compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)) - return 0; + RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } - for (; i < asdl_seq_LEN(stmts); i++) + for (; i < asdl_seq_LEN(stmts); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); - return 1; + } + return SUCCESS; } static int compiler_codegen(struct compiler *c, mod_ty mod) { _Py_DECLARE_STR(anon_module, "<module>"); - if (!compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, + mod, 1)); + location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { case Module_kind: - if (!compiler_body(c, loc, mod->v.Module.body)) { + if (compiler_body(c, loc, mod->v.Module.body) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } break; case Interactive_kind: @@ -2194,16 +2178,16 @@ compiler_codegen(struct compiler *c, mod_ty mod) PyErr_Format(PyExc_SystemError, "module kind %d should not be possible", mod->kind); - return 0; + return ERROR; } - return 1; + return SUCCESS; } static PyCodeObject * compiler_mod(struct compiler *c, mod_ty mod) { int addNone = mod->kind != Expression_kind; - if (!compiler_codegen(c, mod)) { + if (compiler_codegen(c, mod) < 0) { return NULL; } PyCodeObject *co = assemble(c, addNone); @@ -2270,7 +2254,7 @@ compiler_make_closure(struct compiler *c, location loc, */ int reftype = get_ref_type(c, name); if (reftype == -1) { - return 0; + return ERROR; } int arg; if (reftype == CELL) { @@ -2293,7 +2277,7 @@ compiler_make_closure(struct compiler *c, location loc, co->co_name, freevars); Py_DECREF(freevars); - return 0; + return ERROR; } ADDOP_I(c, loc, LOAD_CLOSURE, arg); } @@ -2302,34 +2286,34 @@ compiler_make_closure(struct compiler *c, location loc, } ADDOP_LOAD_CONST(c, loc, (PyObject*)co); ADDOP_I(c, loc, MAKE_FUNCTION, flags); - return 1; + return SUCCESS; } static int compiler_decorators(struct compiler *c, asdl_expr_seq* decos) { - int i; - - if (!decos) - return 1; + if (!decos) { + return SUCCESS; + } - for (i = 0; i < asdl_seq_LEN(decos); i++) { + for (Py_ssize_t i = 0; i < asdl_seq_LEN(decos); i++) { VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i)); } - return 1; + return SUCCESS; } static int compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) { - if (!decos) - return 1; + if (!decos) { + return SUCCESS; + } for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); ADDOP_I(c, loc, CALL, 0); } - return 1; + return SUCCESS; } static int @@ -2338,7 +2322,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, { /* Push a dict of keyword-only default values. - Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed. + Return -1 on error, 0 if no dict pushed, 1 if a dict is pushed. */ int i; PyObject *keys = NULL; @@ -2355,7 +2339,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, keys = PyList_New(1); if (keys == NULL) { Py_DECREF(mangled); - return 0; + return ERROR; } PyList_SET_ITEM(keys, 0, mangled); } @@ -2366,7 +2350,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, goto error; } } - if (!compiler_visit_expr(c, default_)) { + if (compiler_visit_expr(c, default_) < 0) { goto error; } } @@ -2381,12 +2365,12 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, return 1; } else { - return -1; + return 0; } error: Py_XDECREF(keys); - return 0; + return ERROR; } static int @@ -2394,7 +2378,7 @@ compiler_visit_annexpr(struct compiler *c, expr_ty annotation) { location loc = LOC(annotation); ADDOP_LOAD_CONST_NEW(c, loc, _PyAST_ExprAsUnicode(annotation)); - return 1; + return SUCCESS; } static int @@ -2402,11 +2386,11 @@ compiler_visit_argannotation(struct compiler *c, identifier id, expr_ty annotation, Py_ssize_t *annotations_len, location loc) { if (!annotation) { - return 1; + return SUCCESS; } PyObject *mangled = _Py_Mangle(c->u->u_private, id); if (!mangled) { - return 0; + return ERROR; } ADDOP_LOAD_CONST(c, loc, mangled); Py_DECREF(mangled); @@ -2428,7 +2412,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, } } *annotations_len += 2; - return 1; + return SUCCESS; } static int @@ -2438,15 +2422,15 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, int i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (!compiler_visit_argannotation( + RETURN_IF_ERROR( + compiler_visit_argannotation( c, arg->arg, arg->annotation, annotations_len, - loc)) - return 0; + loc)); } - return 1; + return SUCCESS; } static int @@ -2456,36 +2440,40 @@ compiler_visit_annotations(struct compiler *c, location loc, /* Push arg annotation names and values. The expressions are evaluated out-of-order wrt the source code. - Return 0 on error, -1 if no annotations pushed, 1 if a annotations is pushed. + Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed. */ Py_ssize_t annotations_len = 0; - if (!compiler_visit_argannotations(c, args->args, &annotations_len, loc)) - return 0; - if (!compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)) - return 0; - if (args->vararg && args->vararg->annotation && - !compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len, loc)) - return 0; - if (!compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)) - return 0; - if (args->kwarg && args->kwarg->annotation && - !compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len, loc)) - return 0; + RETURN_IF_ERROR( + compiler_visit_argannotations(c, args->args, &annotations_len, loc)); - if (!compiler_visit_argannotation(c, &_Py_ID(return), returns, - &annotations_len, loc)) { - return 0; + RETURN_IF_ERROR( + compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)); + + if (args->vararg && args->vararg->annotation) { + RETURN_IF_ERROR( + compiler_visit_argannotation(c, args->vararg->arg, + args->vararg->annotation, &annotations_len, loc)); } + RETURN_IF_ERROR( + compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)); + + if (args->kwarg && args->kwarg->annotation) { + RETURN_IF_ERROR( + compiler_visit_argannotation(c, args->kwarg->arg, + args->kwarg->annotation, &annotations_len, loc)); + } + + RETURN_IF_ERROR( + compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc)); + if (annotations_len) { ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); return 1; } - return -1; + return 0; } static int @@ -2494,7 +2482,7 @@ compiler_visit_defaults(struct compiler *c, arguments_ty args, { VISIT_SEQ(c, expr, args->defaults); ADDOP_I(c, loc, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); - return 1; + return SUCCESS; } static Py_ssize_t @@ -2503,47 +2491,45 @@ compiler_default_arguments(struct compiler *c, location loc, { Py_ssize_t funcflags = 0; if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { - if (!compiler_visit_defaults(c, args, loc)) - return -1; + RETURN_IF_ERROR(compiler_visit_defaults(c, args, loc)); funcflags |= 0x01; } if (args->kwonlyargs) { int res = compiler_visit_kwonlydefaults(c, loc, args->kwonlyargs, args->kw_defaults); - if (res == 0) { - return -1; - } - else if (res > 0) { + RETURN_IF_ERROR(res); + if (res > 0) { funcflags |= 0x02; } } return funcflags; } -static int +static bool forbidden_name(struct compiler *c, location loc, identifier name, expr_context_ty ctx) { if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { compiler_error(c, loc, "cannot assign to __debug__"); - return 1; + return true; } if (ctx == Del && _PyUnicode_EqualToASCIIString(name, "__debug__")) { compiler_error(c, loc, "cannot delete __debug__"); - return 1; + return true; } - return 0; + return false; } static int compiler_check_debug_one_arg(struct compiler *c, arg_ty arg) { if (arg != NULL) { - if (forbidden_name(c, LOC(arg), arg->arg, Store)) - return 0; + if (forbidden_name(c, LOC(arg), arg->arg, Store)) { + return ERROR; + } } - return 1; + return SUCCESS; } static int @@ -2551,39 +2537,32 @@ compiler_check_debug_args_seq(struct compiler *c, asdl_arg_seq *args) { if (args != NULL) { for (Py_ssize_t i = 0, n = asdl_seq_LEN(args); i < n; i++) { - if (!compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))) - return 0; + RETURN_IF_ERROR( + compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))); } } - return 1; + return SUCCESS; } static int compiler_check_debug_args(struct compiler *c, arguments_ty args) { - if (!compiler_check_debug_args_seq(c, args->posonlyargs)) - return 0; - if (!compiler_check_debug_args_seq(c, args->args)) - return 0; - if (!compiler_check_debug_one_arg(c, args->vararg)) - return 0; - if (!compiler_check_debug_args_seq(c, args->kwonlyargs)) - return 0; - if (!compiler_check_debug_one_arg(c, args->kwarg)) - return 0; - return 1; + RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->posonlyargs)); + RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->args)); + RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->vararg)); + RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->kwonlyargs)); + RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->kwarg)); + return SUCCESS; } static inline int insert_instruction(basicblock *block, int pos, struct instr *instr) { - if (basicblock_next_instr(block) < 0) { - return -1; - } + RETURN_IF_ERROR(basicblock_next_instr(block)); for (int i = block->b_iused - 1; i > pos; i--) { block->b_instr[i] = block->b_instr[i-1]; } block->b_instr[pos] = *instr; - return 0; + return SUCCESS; } static int @@ -2598,16 +2577,15 @@ wrap_in_stopiteration_handler(struct compiler *c) .i_loc = NO_LOCATION, .i_target = NULL, }; - if (insert_instruction(c->u->u_cfg_builder.g_entryblock, 0, &setup)) { - return 0; - } + RETURN_IF_ERROR( + insert_instruction(c->u->u_cfg_builder.g_entryblock, 0, &setup)); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); ADDOP(c, NO_LOCATION, RETURN_VALUE); USE_LABEL(c, handler); ADDOP(c, NO_LOCATION, STOPITERATION_ERROR); ADDOP_I(c, NO_LOCATION, RERAISE, 1); - return 1; + return SUCCESS; } static int @@ -2647,11 +2625,8 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) scope_type = COMPILER_SCOPE_FUNCTION; } - if (!compiler_check_debug_args(c, args)) - return 0; - - if (!compiler_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_check_debug_args(c, args)); + RETURN_IF_ERROR(compiler_decorators(c, decos)); firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2661,19 +2636,16 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) location loc = LOC(s); funcflags = compiler_default_arguments(c, loc, args); if (funcflags == -1) { - return 0; + return ERROR; } annotations = compiler_visit_annotations(c, loc, args, returns); - if (annotations == 0) { - return 0; - } - else if (annotations > 0) { + RETURN_IF_ERROR(annotations); + if (annotations > 0) { funcflags |= 0x04; } - if (!compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)); /* if not -OO mode, add docstring */ if (c->c_optimize < 2) { @@ -2681,7 +2653,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } c->u->u_argcount = asdl_seq_LEN(args->args); @@ -2691,9 +2663,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } if (c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator) { - if (!wrap_in_stopiteration_handler(c)) { + if (wrap_in_stopiteration_handler(c) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } } co = assemble(c, 1); @@ -2702,18 +2674,17 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (co == NULL) { Py_XDECREF(qualname); Py_XDECREF(co); - return 0; + return ERROR; } - if (!compiler_make_closure(c, loc, co, funcflags, qualname)) { + if (compiler_make_closure(c, loc, co, funcflags, qualname) < 0) { Py_DECREF(qualname); Py_DECREF(co); - return 0; + return ERROR; } Py_DECREF(qualname); Py_DECREF(co); - if (!compiler_apply_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); return compiler_nameop(c, loc, name, Store); } @@ -2724,8 +2695,7 @@ compiler_class(struct compiler *c, stmt_ty s) int i, firstlineno; asdl_expr_seq *decos = s->v.ClassDef.decorator_list; - if (!compiler_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_decorators(c, decos)); firstlineno = s->lineno; if (asdl_seq_LEN(decos)) { @@ -2743,35 +2713,35 @@ compiler_class(struct compiler *c, stmt_ty s) This borrows from compiler_call. */ /* 1. compile the class body into a code object */ - if (!compiler_enter_scope(c, s->v.ClassDef.name, - COMPILER_SCOPE_CLASS, (void *)s, firstlineno)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, s->v.ClassDef.name, + COMPILER_SCOPE_CLASS, (void *)s, firstlineno)); + /* this block represents what we do in the new scope */ { location loc = LOCATION(firstlineno, firstlineno, 0, 0); /* use the class name for name mangling */ Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); /* load (global) __name__ ... */ - if (!compiler_nameop(c, loc, &_Py_ID(__name__), Load)) { + if (compiler_nameop(c, loc, &_Py_ID(__name__), Load) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } /* ... and store it as __module__ */ - if (!compiler_nameop(c, loc, &_Py_ID(__module__), Store)) { + if (compiler_nameop(c, loc, &_Py_ID(__module__), Store) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } assert(c->u->u_qualname); ADDOP_LOAD_CONST(c, loc, c->u->u_qualname); - if (!compiler_nameop(c, loc, &_Py_ID(__qualname__), Store)) { + if (compiler_nameop(c, loc, &_Py_ID(__qualname__), Store) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } /* compile the body proper */ - if (!compiler_body(c, loc, s->v.ClassDef.body)) { + if (compiler_body(c, loc, s->v.ClassDef.body) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } /* The following code is artificial */ /* Return __classcell__ if it is referenced, otherwise return None */ @@ -2780,14 +2750,14 @@ compiler_class(struct compiler *c, stmt_ty s) i = compiler_lookup_arg(c->u->u_cellvars, &_Py_ID(__class__)); if (i < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } assert(i == 0); ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); ADDOP_I(c, NO_LOCATION, COPY, 1); - if (!compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store)) { + if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store) < 0) { compiler_exit_scope(c); - return 0; + return ERROR; } } else { @@ -2801,8 +2771,9 @@ compiler_class(struct compiler *c, stmt_ty s) } /* leave the new scope */ compiler_exit_scope(c); - if (co == NULL) - return 0; + if (co == NULL) { + return ERROR; + } location loc = LOC(s); /* 2. load the 'build_class' function */ @@ -2810,9 +2781,9 @@ compiler_class(struct compiler *c, stmt_ty s) ADDOP(c, loc, LOAD_BUILD_CLASS); /* 3. load a function (or closure) made from the code object */ - if (!compiler_make_closure(c, loc, co, 0, NULL)) { + if (compiler_make_closure(c, loc, co, 0, NULL) < 0) { Py_DECREF(co); - return 0; + return ERROR; } Py_DECREF(co); @@ -2820,27 +2791,25 @@ compiler_class(struct compiler *c, stmt_ty s) ADDOP_LOAD_CONST(c, loc, s->v.ClassDef.name); /* 5. generate the rest of the code for the call */ - if (!compiler_call_helper(c, loc, 2, - s->v.ClassDef.bases, - s->v.ClassDef.keywords)) - return 0; + RETURN_IF_ERROR(compiler_call_helper(c, loc, 2, + s->v.ClassDef.bases, + s->v.ClassDef.keywords)); + /* 6. apply decorators */ - if (!compiler_apply_decorators(c, decos)) - return 0; + RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); /* 7. store into <name> */ - if (!compiler_nameop(c, loc, s->v.ClassDef.name, Store)) - return 0; - return 1; + RETURN_IF_ERROR(compiler_nameop(c, loc, s->v.ClassDef.name, Store)); + return SUCCESS; } -/* Return 0 if the expression is a constant value except named singletons. - Return 1 otherwise. */ -static int +/* Return false if the expression is a constant value except named singletons. + Return true otherwise. */ +static bool check_is_arg(expr_ty e) { if (e->kind != Constant_kind) { - return 1; + return true; } PyObject *value = e->v.Constant.value; return (value == Py_None @@ -2849,19 +2818,18 @@ check_is_arg(expr_ty e) || value == Py_Ellipsis); } -/* Check operands of identity chacks ("is" and "is not"). +/* Check operands of identity checks ("is" and "is not"). Emit a warning if any operand is a constant except named singletons. - Return 0 on error. */ static int check_compare(struct compiler *c, expr_ty e) { Py_ssize_t i, n; - int left = check_is_arg(e->v.Compare.left); + bool left = check_is_arg(e->v.Compare.left); n = asdl_seq_LEN(e->v.Compare.ops); for (i = 0; i < n; i++) { cmpop_ty op = (cmpop_ty)asdl_seq_GET(e->v.Compare.ops, i); - int right = check_is_arg((expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); + bool right = check_is_arg((expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); if (op == Is || op == IsNot) { if (!right || !left) { const char *msg = (op == Is) @@ -2872,7 +2840,7 @@ check_compare(struct compiler *c, expr_ty e) } left = right; } - return 1; + return SUCCESS; } static int compiler_addcompare(struct compiler *c, location loc, @@ -2900,21 +2868,21 @@ static int compiler_addcompare(struct compiler *c, location loc, break; case Is: ADDOP_I(c, loc, IS_OP, 0); - return 1; + return SUCCESS; case IsNot: ADDOP_I(c, loc, IS_OP, 1); - return 1; + return SUCCESS; case In: ADDOP_I(c, loc, CONTAINS_OP, 0); - return 1; + return SUCCESS; case NotIn: ADDOP_I(c, loc, CONTAINS_OP, 1); - return 1; + return SUCCESS; default: Py_UNREACHABLE(); } ADDOP_I(c, loc, COMPARE_OP, cmp); - return 1; + return SUCCESS; } @@ -2941,43 +2909,36 @@ compiler_jump_if(struct compiler *c, location loc, next2 = new_next2; } for (i = 0; i < n; ++i) { - if (!compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)) { - return 0; - } - } - if (!compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)) { - return 0; + RETURN_IF_ERROR( + compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); } + RETURN_IF_ERROR( + compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); if (!SAME_LABEL(next2, next)) { USE_LABEL(c, next2); } - return 1; + return SUCCESS; } case IfExp_kind: { NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next2); - if (!compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)) { - return 0; - } - if (!compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)); + RETURN_IF_ERROR( + compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)); ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, next2); - if (!compiler_jump_if(c, loc, e->v.IfExp.orelse, next, cond)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); USE_LABEL(c, end); - return 1; + return SUCCESS; } case Compare_kind: { Py_ssize_t n = asdl_seq_LEN(e->v.Compare.ops) - 1; if (n > 0) { - if (!check_compare(c, e)) { - return 0; - } + RETURN_IF_ERROR(check_compare(c, e)); NEW_JUMP_TARGET_LABEL(c, cleanup); VISIT(c, expr, e->v.Compare.left); for (Py_ssize_t i = 0; i < n; i++) { @@ -3001,7 +2962,7 @@ compiler_jump_if(struct compiler *c, location loc, } USE_LABEL(c, end); - return 1; + return SUCCESS; } /* fallback to general implementation */ break; @@ -3014,7 +2975,7 @@ compiler_jump_if(struct compiler *c, location loc, /* general implementation */ VISIT(c, expr, e); ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); - return 1; + return SUCCESS; } static int @@ -3024,9 +2985,9 @@ compiler_ifexp(struct compiler *c, expr_ty e) NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next); - if (!compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); + VISIT(c, expr, e->v.IfExp.body); ADDOP_JUMP(c, NO_LOCATION, JUMP, end); @@ -3034,7 +2995,7 @@ compiler_ifexp(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.IfExp.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3046,8 +3007,7 @@ compiler_lambda(struct compiler *c, expr_ty e) arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); - if (!compiler_check_debug_args(c, args)) - return 0; + RETURN_IF_ERROR(compiler_check_debug_args(c, args)); location loc = LOC(e); funcflags = compiler_default_arguments(c, loc, args); @@ -3056,14 +3016,13 @@ compiler_lambda(struct compiler *c, expr_ty e) } _Py_DECLARE_STR(anon_lambda, "<lambda>"); - if (!compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, - (void *)e, e->lineno)) { - return 0; - } + RETURN_IF_ERROR( + compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, + (void *)e, e->lineno)); + /* Make None the first constant, so the lambda can't have a docstring. */ - if (compiler_add_const(c, Py_None) < 0) - return 0; + RETURN_IF_ERROR(compiler_add_const(c, Py_None)); c->u->u_argcount = asdl_seq_LEN(args->args); c->u->u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); @@ -3081,18 +3040,18 @@ compiler_lambda(struct compiler *c, expr_ty e) compiler_exit_scope(c); if (co == NULL) { Py_DECREF(qualname); - return 0; + return ERROR; } - if (!compiler_make_closure(c, loc, co, funcflags, qualname)) { + if (compiler_make_closure(c, loc, co, funcflags, qualname) < 0) { Py_DECREF(qualname); Py_DECREF(co); - return 0; + return ERROR; } Py_DECREF(qualname); Py_DECREF(co); - return 1; + return SUCCESS; } static int @@ -3108,9 +3067,9 @@ compiler_if(struct compiler *c, stmt_ty s) else { next = end; } - if (!compiler_jump_if(c, LOC(s), s->v.If.test, next, 0)) { - return 0; - } + RETURN_IF_ERROR( + compiler_jump_if(c, LOC(s), s->v.If.test, next, 0)); + VISIT_SEQ(c, stmt, s->v.If.body); if (asdl_seq_LEN(s->v.If.orelse)) { ADDOP_JUMP(c, NO_LOCATION, JUMP, end); @@ -3120,7 +3079,7 @@ compiler_if(struct compiler *c, stmt_ty s) } USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3132,9 +3091,8 @@ compiler_for(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup); NEW_JUMP_TARGET_LABEL(c, end); - if (!compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); + VISIT(c, expr, s->v.For.iter); ADDOP(c, loc, GET_ITER); @@ -3155,7 +3113,7 @@ compiler_for(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.For.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } @@ -3177,9 +3135,8 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, loc, GET_AITER); USE_LABEL(c, start); - if (!compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); + /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); @@ -3207,7 +3164,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.For.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3219,18 +3176,13 @@ compiler_while(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, anchor); USE_LABEL(c, loop); - if (!compiler_push_fblock(c, LOC(s), WHILE_LOOP, loop, end, NULL)) { - return 0; - } - if (!compiler_jump_if(c, LOC(s), s->v.While.test, anchor, 0)) { - return 0; - } + + RETURN_IF_ERROR(compiler_push_fblock(c, LOC(s), WHILE_LOOP, loop, end, NULL)); + RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); USE_LABEL(c, body); VISIT_SEQ(c, stmt, s->v.While.body); - if (!compiler_jump_if(c, LOC(s), s->v.While.test, body, 1)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, body, 1)); compiler_pop_fblock(c, WHILE_LOOP, loop); @@ -3240,7 +3192,7 @@ compiler_while(struct compiler *c, stmt_ty s) } USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3249,13 +3201,13 @@ compiler_return(struct compiler *c, stmt_ty s) location loc = LOC(s); int preserve_tos = ((s->v.Return.value != NULL) && (s->v.Return.value->kind != Constant_kind)); - if (c->u->u_ste->ste_type != FunctionBlock) + if (c->u->u_ste->ste_type != FunctionBlock) { return compiler_error(c, loc, "'return' outside function"); + } if (s->v.Return.value != NULL && c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) { - return compiler_error( - c, loc, "'return' with value in async generator"); + return compiler_error(c, loc, "'return' with value in async generator"); } if (preserve_tos) { @@ -3272,8 +3224,7 @@ compiler_return(struct compiler *c, stmt_ty s) ADDOP(c, loc, NOP); } - if (!compiler_unwind_fblock_stack(c, &loc, preserve_tos, NULL)) - return 0; + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, preserve_tos, NULL)); if (s->v.Return.value == NULL) { ADDOP_LOAD_CONST(c, loc, Py_None); } @@ -3282,7 +3233,7 @@ compiler_return(struct compiler *c, stmt_ty s) } ADDOP(c, loc, RETURN_VALUE); - return 1; + return SUCCESS; } static int @@ -3291,17 +3242,13 @@ compiler_break(struct compiler *c, location loc) struct fblockinfo *loop = NULL; /* Emit instruction with line number */ ADDOP(c, loc, NOP); - if (!compiler_unwind_fblock_stack(c, &loc, 0, &loop)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, 0, &loop)); if (loop == NULL) { return compiler_error(c, loc, "'break' outside loop"); } - if (!compiler_unwind_fblock(c, &loc, loop, 0)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock(c, &loc, loop, 0)); ADDOP_JUMP(c, loc, JUMP, loop->fb_exit); - return 1; + return SUCCESS; } static int @@ -3310,14 +3257,12 @@ compiler_continue(struct compiler *c, location loc) struct fblockinfo *loop = NULL; /* Emit instruction with line number */ ADDOP(c, loc, NOP); - if (!compiler_unwind_fblock_stack(c, &loc, 0, &loop)) { - return 0; - } + RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, 0, &loop)); if (loop == NULL) { return compiler_error(c, loc, "'continue' not properly in loop"); } ADDOP_JUMP(c, loc, JUMP, loop->fb_block); - return 1; + return SUCCESS; } @@ -3376,11 +3321,12 @@ compiler_try_finally(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, end); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, FINALLY_TRY, body, end, s->v.Try.finalbody)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_TRY, body, end, + s->v.Try.finalbody)); + if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { - if (!compiler_try_except(c, s)) - return 0; + RETURN_IF_ERROR(compiler_try_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.Try.body); @@ -3397,8 +3343,8 @@ compiler_try_finally(struct compiler *c, stmt_ty s) loc = NO_LOCATION; ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); - if (!compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)); VISIT_SEQ(c, stmt, s->v.Try.finalbody); loc = location_of_last_executing_statement(s->v.Try.finalbody); compiler_pop_fblock(c, FINALLY_END, end); @@ -3409,7 +3355,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) POP_EXCEPT_AND_RERAISE(c, loc); USE_LABEL(c, exit); - return 1; + return SUCCESS; } static int @@ -3425,13 +3371,12 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, end); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, FINALLY_TRY, body, end, s->v.TryStar.finalbody)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_TRY, body, end, + s->v.TryStar.finalbody)); + if (s->v.TryStar.handlers && asdl_seq_LEN(s->v.TryStar.handlers)) { - if (!compiler_try_star_except(c, s)) { - return 0; - } + RETURN_IF_ERROR(compiler_try_star_except(c, s)); } else { VISIT_SEQ(c, stmt, s->v.TryStar.body); @@ -3448,9 +3393,9 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) loc = NO_LOCATION; ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); - if (!compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)); + VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); loc = location_of_last_executing_statement(s->v.Try.finalbody); @@ -3461,7 +3406,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) POP_EXCEPT_AND_RERAISE(c, loc); USE_LABEL(c, exit); - return 1; + return SUCCESS; } @@ -3507,8 +3452,8 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, except); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)); VISIT_SEQ(c, stmt, s->v.Try.body); compiler_pop_fblock(c, TRY_EXCEPT, body); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -3522,9 +3467,11 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); + /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, loc, EXCEPTION_HANDLER, NO_LABEL, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, EXCEPTION_HANDLER, NO_LABEL, NO_LABEL, NULL)); + for (i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( s->v.Try.handlers, i); @@ -3543,7 +3490,8 @@ compiler_try_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup_end); NEW_JUMP_TARGET_LABEL(c, cleanup_body); - compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store); + RETURN_IF_ERROR( + compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store)); /* try: @@ -3560,10 +3508,9 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, - NO_LABEL, handler->v.ExceptHandler.name)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, + NO_LABEL, handler->v.ExceptHandler.name)); /* second # body */ VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); @@ -3573,8 +3520,10 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP(c, NO_LOCATION, POP_BLOCK); ADDOP(c, NO_LOCATION, POP_EXCEPT); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); ADDOP_JUMP(c, NO_LOCATION, JUMP, end); /* except: */ @@ -3582,8 +3531,10 @@ compiler_try_except(struct compiler *c, stmt_ty s) /* name = None; del name; # artificial */ ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); ADDOP_I(c, NO_LOCATION, RERAISE, 1); } @@ -3593,8 +3544,10 @@ compiler_try_except(struct compiler *c, stmt_ty s) ADDOP(c, loc, POP_TOP); /* exc_value */ USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, NO_LABEL, NULL)) - return 0; + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, + NO_LABEL, NULL)); + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -3612,7 +3565,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); USE_LABEL(c, end); - return 1; + return SUCCESS; } /* @@ -3679,9 +3632,8 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_FINALLY, except); USE_LABEL(c, body); - if (!compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)); VISIT_SEQ(c, stmt, s->v.TryStar.body); compiler_pop_fblock(c, TRY_EXCEPT, body); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -3692,11 +3644,12 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); + /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, loc, EXCEPTION_GROUP_HANDLER, - NO_LABEL, NO_LABEL, "except handler")) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, EXCEPTION_GROUP_HANDLER, + NO_LABEL, NO_LABEL, "except handler")); + for (Py_ssize_t i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( s->v.TryStar.handlers, i); @@ -3736,7 +3689,8 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup_body); if (handler->v.ExceptHandler.name) { - compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store); + RETURN_IF_ERROR( + compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store)); } else { ADDOP(c, loc, POP_TOP); // match @@ -3756,9 +3710,9 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, NO_LABEL, handler->v.ExceptHandler.name)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, + NO_LABEL, handler->v.ExceptHandler.name)); /* second # body */ VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); @@ -3767,8 +3721,10 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP(c, NO_LOCATION, POP_BLOCK); if (handler->v.ExceptHandler.name) { ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); } ADDOP_JUMP(c, NO_LOCATION, JUMP, except); @@ -3778,8 +3734,10 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) /* name = None; del name; # artificial */ if (handler->v.ExceptHandler.name) { ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); } /* add exception raised to the res list */ @@ -3823,7 +3781,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.TryStar.orelse); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -3857,19 +3815,22 @@ compiler_import_as(struct compiler *c, location loc, */ Py_ssize_t len = PyUnicode_GET_LENGTH(name); Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, len, 1); - if (dot == -2) - return 0; + if (dot == -2) { + return ERROR; + } if (dot != -1) { /* Consume the base module name to get the first attribute */ while (1) { Py_ssize_t pos = dot + 1; PyObject *attr; dot = PyUnicode_FindChar(name, '.', pos, len, 1); - if (dot == -2) - return 0; + if (dot == -2) { + return ERROR; + } attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); - if (!attr) - return 0; + if (!attr) { + return ERROR; + } ADDOP_N(c, loc, IMPORT_FROM, attr, names); if (dot == -1) { break; @@ -3877,11 +3838,9 @@ compiler_import_as(struct compiler *c, location loc, ADDOP_I(c, loc, SWAP, 2); ADDOP(c, loc, POP_TOP); } - if (!compiler_nameop(c, loc, asname, Store)) { - return 0; - } + RETURN_IF_ERROR(compiler_nameop(c, loc, asname, Store)); ADDOP(c, loc, POP_TOP); - return 1; + return SUCCESS; } return compiler_nameop(c, loc, asname, Store); } @@ -3910,8 +3869,7 @@ compiler_import(struct compiler *c, stmt_ty s) if (alias->asname) { r = compiler_import_as(c, loc, alias->name, alias->asname); - if (!r) - return r; + RETURN_IF_ERROR(r); } else { identifier tmp = alias->name; @@ -3919,18 +3877,18 @@ compiler_import(struct compiler *c, stmt_ty s) alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1); if (dot != -1) { tmp = PyUnicode_Substring(alias->name, 0, dot); - if (tmp == NULL) - return 0; + if (tmp == NULL) { + return ERROR; + } } r = compiler_nameop(c, loc, tmp, Store); if (dot != -1) { Py_DECREF(tmp); } - if (!r) - return r; + RETURN_IF_ERROR(r); } } - return 1; + return SUCCESS; } static int @@ -3942,7 +3900,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) PyObject *names = PyTuple_New(n); if (!names) { - return 0; + return ERROR; } /* build up the names */ @@ -3975,7 +3933,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') { assert(n == 1); ADDOP(c, LOC(s), IMPORT_STAR); - return 1; + return SUCCESS; } ADDOP_NAME(c, LOC(s), IMPORT_FROM, alias->name, names); @@ -3984,13 +3942,11 @@ compiler_from_import(struct compiler *c, stmt_ty s) store_name = alias->asname; } - if (!compiler_nameop(c, LOC(s), store_name, Store)) { - return 0; - } + RETURN_IF_ERROR(compiler_nameop(c, LOC(s), store_name, Store)); } /* remove imported module */ ADDOP(c, LOC(s), POP_TOP); - return 1; + return SUCCESS; } static int @@ -4003,19 +3959,15 @@ compiler_assert(struct compiler *c, stmt_ty s) PyTuple_Check(s->v.Assert.test->v.Constant.value) && PyTuple_Size(s->v.Assert.test->v.Constant.value) > 0)) { - if (!compiler_warn(c, LOC(s), "assertion is always true, " - "perhaps remove parentheses?")) - { - return 0; - } + RETURN_IF_ERROR( + compiler_warn(c, LOC(s), "assertion is always true, " + "perhaps remove parentheses?")); } if (c->c_optimize) { - return 1; + return SUCCESS; } NEW_JUMP_TARGET_LABEL(c, end); - if (!compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); ADDOP(c, LOC(s), LOAD_ASSERTION_ERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); @@ -4024,7 +3976,7 @@ compiler_assert(struct compiler *c, stmt_ty s) ADDOP_I(c, LOC(s), RAISE_VARARGS, 1); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -4033,18 +3985,18 @@ compiler_stmt_expr(struct compiler *c, location loc, expr_ty value) if (c->c_interactive && c->c_nestlevel <= 1) { VISIT(c, expr, value); ADDOP(c, loc, PRINT_EXPR); - return 1; + return SUCCESS; } if (value->kind == Constant_kind) { /* ignore constant statement */ ADDOP(c, loc, NOP); - return 1; + return SUCCESS; } VISIT(c, expr, value); ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ - return 1; + return SUCCESS; } static int @@ -4140,7 +4092,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) return compiler_async_for(c, s); } - return 1; + return SUCCESS; } static int @@ -4210,10 +4162,10 @@ addop_binary(struct compiler *c, location loc, operator_ty binop, default: PyErr_Format(PyExc_SystemError, "%s op %d should not be possible", inplace ? "inplace" : "binary", binop); - return 0; + return ERROR; } ADDOP_I(c, loc, BINARY_OP, oparg); - return 1; + return SUCCESS; } @@ -4224,7 +4176,7 @@ addop_yield(struct compiler *c, location loc) { } ADDOP_I(c, loc, YIELD_VALUE, 0); ADDOP_I(c, loc, RESUME, 1); - return 1; + return SUCCESS; } static int @@ -4242,12 +4194,14 @@ compiler_nameop(struct compiler *c, location loc, !_PyUnicode_EqualToASCIIString(name, "True") && !_PyUnicode_EqualToASCIIString(name, "False")); - if (forbidden_name(c, loc, name, ctx)) - return 0; + if (forbidden_name(c, loc, name, ctx)) { + return ERROR; + } mangled = _Py_Mangle(c->u->u_private, name); - if (!mangled) - return 0; + if (!mangled) { + return ERROR; + } op = 0; optype = OP_NAME; @@ -4297,7 +4251,7 @@ compiler_nameop(struct compiler *c, location loc, case Del: op = DELETE_FAST; break; } ADDOP_N(c, loc, op, mangled, varnames); - return 1; + return SUCCESS; case OP_GLOBAL: switch (ctx) { case Load: op = LOAD_GLOBAL; break; @@ -4318,7 +4272,7 @@ compiler_nameop(struct compiler *c, location loc, arg = dict_add_o(dict, mangled); Py_DECREF(mangled); if (arg < 0) { - return 0; + return ERROR; } if (op == LOAD_GLOBAL) { arg <<= 1; @@ -4353,7 +4307,7 @@ compiler_boolop(struct compiler *c, expr_ty e) VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n)); USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -4365,7 +4319,7 @@ starunpack_helper(struct compiler *c, location loc, if (n > 2 && are_all_items_const(elts, 0, n)) { PyObject *folded = PyTuple_New(n); if (folded == NULL) { - return 0; + return ERROR; } PyObject *val; for (Py_ssize_t i = 0; i < n; i++) { @@ -4378,7 +4332,7 @@ starunpack_helper(struct compiler *c, location loc, if (add == SET_ADD) { Py_SETREF(folded, PyFrozenSet_New(folded)); if (folded == NULL) { - return 0; + return ERROR; } } ADDOP_I(c, loc, build, pushed); @@ -4388,7 +4342,7 @@ starunpack_helper(struct compiler *c, location loc, ADDOP(c, loc, LIST_TO_TUPLE); } } - return 1; + return SUCCESS; } int big = n+pushed > STACK_USE_GUIDELINE; @@ -4410,7 +4364,7 @@ starunpack_helper(struct compiler *c, location loc, } else { ADDOP_I(c, loc, build, n+pushed); } - return 1; + return SUCCESS; } int sequence_built = 0; if (big) { @@ -4438,7 +4392,7 @@ starunpack_helper(struct compiler *c, location loc, if (tuple) { ADDOP(c, loc, LIST_TO_TUPLE); } - return 1; + return SUCCESS; } static int @@ -4450,10 +4404,11 @@ unpack_helper(struct compiler *c, location loc, asdl_expr_seq *elts) expr_ty elt = asdl_seq_GET(elts, i); if (elt->kind == Starred_kind && !seen_star) { if ((i >= (1 << 8)) || - (n-i-1 >= (INT_MAX >> 8))) + (n-i-1 >= (INT_MAX >> 8))) { return compiler_error(c, loc, "too many expressions in " "star-unpacking assignment"); + } ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); seen_star = 1; } @@ -4465,19 +4420,19 @@ unpack_helper(struct compiler *c, location loc, asdl_expr_seq *elts) if (!seen_star) { ADDOP_I(c, loc, UNPACK_SEQUENCE, n); } - return 1; + return SUCCESS; } static int assignment_helper(struct compiler *c, location loc, asdl_expr_seq *elts) { Py_ssize_t n = asdl_seq_LEN(elts); - RETURN_IF_FALSE(unpack_helper(c, loc, elts)); + RETURN_IF_ERROR(unpack_helper(c, loc, elts)); for (Py_ssize_t i = 0; i < n; i++) { expr_ty elt = asdl_seq_GET(elts, i); VISIT(c, expr, elt->kind != Starred_kind ? elt : elt->v.Starred.value); } - return 1; + return SUCCESS; } static int @@ -4492,9 +4447,10 @@ compiler_list(struct compiler *c, expr_ty e) return starunpack_helper(c, loc, elts, 0, BUILD_LIST, LIST_APPEND, LIST_EXTEND, 0); } - else + else { VISIT_SEQ(c, expr, elts); - return 1; + } + return SUCCESS; } static int @@ -4509,9 +4465,10 @@ compiler_tuple(struct compiler *c, expr_ty e) return starunpack_helper(c, loc, elts, 0, BUILD_LIST, LIST_APPEND, LIST_EXTEND, 1); } - else + else { VISIT_SEQ(c, expr, elts); - return 1; + } + return SUCCESS; } static int @@ -4522,16 +4479,16 @@ compiler_set(struct compiler *c, expr_ty e) BUILD_SET, SET_ADD, SET_UPDATE, 0); } -static int +static bool are_all_items_const(asdl_expr_seq *seq, Py_ssize_t begin, Py_ssize_t end) { - Py_ssize_t i; - for (i = begin; i < end; i++) { + for (Py_ssize_t i = begin; i < end; i++) { expr_ty key = (expr_ty)asdl_seq_GET(seq, i); - if (key == NULL || key->kind != Constant_kind) - return 0; + if (key == NULL || key->kind != Constant_kind) { + return false; + } } - return 1; + return true; } static int @@ -4547,7 +4504,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end } keys = PyTuple_New(n); if (keys == NULL) { - return 0; + return SUCCESS; } for (i = begin; i < end; i++) { key = ((expr_ty)asdl_seq_GET(e->v.Dict.keys, i))->v.Constant.value; @@ -4555,7 +4512,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end } ADDOP_LOAD_CONST_NEW(c, loc, keys); ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return 1; + return SUCCESS; } if (big) { ADDOP_I(c, loc, BUILD_MAP, 0); @@ -4570,7 +4527,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end if (!big) { ADDOP_I(c, loc, BUILD_MAP, n); } - return 1; + return SUCCESS; } static int @@ -4587,9 +4544,7 @@ compiler_dict(struct compiler *c, expr_ty e) is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; if (is_unpacking) { if (elements) { - if (!compiler_subdict(c, e, i - elements, i)) { - return 0; - } + RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4605,9 +4560,7 @@ compiler_dict(struct compiler *c, expr_ty e) } else { if (elements*2 > STACK_USE_GUIDELINE) { - if (!compiler_subdict(c, e, i - elements, i + 1)) { - return 0; - } + RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i + 1)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4620,9 +4573,7 @@ compiler_dict(struct compiler *c, expr_ty e) } } if (elements) { - if (!compiler_subdict(c, e, n - elements, n)) { - return 0; - } + RETURN_IF_ERROR(compiler_subdict(c, e, n - elements, n)); if (have_dict) { ADDOP_I(c, loc, DICT_UPDATE, 1); } @@ -4631,7 +4582,7 @@ compiler_dict(struct compiler *c, expr_ty e) if (!have_dict) { ADDOP_I(c, loc, BUILD_MAP, 0); } - return 1; + return SUCCESS; } static int @@ -4640,9 +4591,7 @@ compiler_compare(struct compiler *c, expr_ty e) location loc = LOC(e); Py_ssize_t i, n; - if (!check_compare(c, e)) { - return 0; - } + RETURN_IF_ERROR(check_compare(c, e)); VISIT(c, expr, e->v.Compare.left); assert(asdl_seq_LEN(e->v.Compare.ops) > 0); n = asdl_seq_LEN(e->v.Compare.ops) - 1; @@ -4671,7 +4620,7 @@ compiler_compare(struct compiler *c, expr_ty e) USE_LABEL(c, end); } - return 1; + return SUCCESS; } static PyTypeObject * @@ -4724,7 +4673,7 @@ check_caller(struct compiler *c, expr_ty e) infer_type(e)->tp_name); } default: - return 1; + return SUCCESS; } } @@ -4740,7 +4689,7 @@ check_subscripter(struct compiler *c, expr_ty e) PyLong_Check(v) || PyFloat_Check(v) || PyComplex_Check(v) || PyAnySet_Check(v))) { - return 1; + return SUCCESS; } /* fall through */ case Set_kind: @@ -4753,7 +4702,7 @@ check_subscripter(struct compiler *c, expr_ty e) infer_type(e)->tp_name); } default: - return 1; + return SUCCESS; } } @@ -4766,14 +4715,14 @@ check_index(struct compiler *c, expr_ty e, expr_ty s) if (index_type == NULL || PyType_FastSubclass(index_type, Py_TPFLAGS_LONG_SUBCLASS) || index_type == &PySlice_Type) { - return 1; + return SUCCESS; } switch (e->kind) { case Constant_kind: v = e->v.Constant.value; if (!(PyUnicode_Check(v) || PyBytes_Check(v) || PyTuple_Check(v))) { - return 1; + return SUCCESS; } /* fall through */ case Tuple_kind: @@ -4789,7 +4738,7 @@ check_index(struct compiler *c, expr_ty e, expr_ty s) index_type->tp_name); } default: - return 1; + return SUCCESS; } } @@ -4838,7 +4787,7 @@ update_start_location_to_match_attr(struct compiler *c, location loc, return loc; } -// Return 1 if the method call was optimized, -1 if not, and 0 on error. +// Return 1 if the method call was optimized, 0 if not, and -1 on error. static int maybe_optimize_method_call(struct compiler *c, expr_ty e) { @@ -4849,32 +4798,32 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) /* Check that the call node is an attribute access */ if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load) { - return -1; + return 0; } /* Check that the base object is not something that is imported */ if (is_import_originated(c, meth->v.Attribute.value)) { - return -1; + return 0; } /* Check that there aren't too many arguments */ argsl = asdl_seq_LEN(args); kwdsl = asdl_seq_LEN(kwds); if (argsl + kwdsl + (kwdsl != 0) >= STACK_USE_GUIDELINE) { - return -1; + return 0; } /* Check that there are no *varargs types of arguments. */ for (i = 0; i < argsl; i++) { expr_ty elt = asdl_seq_GET(args, i); if (elt->kind == Starred_kind) { - return -1; + return 0; } } for (i = 0; i < kwdsl; i++) { keyword_ty kw = asdl_seq_GET(kwds, i); if (kw->arg == NULL) { - return -1; + return 0; } } /* Alright, we can optimize the code. */ @@ -4886,9 +4835,8 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) if (kwdsl) { VISIT_SEQ(c, keyword, kwds); - if (!compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)) { - return 0; - }; + RETURN_IF_ERROR( + compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)); } loc = update_start_location_to_match_attr(c, LOC(e), meth); ADDOP_I(c, loc, CALL, argsl + kwdsl); @@ -4906,32 +4854,31 @@ validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) } location loc = LOC(key); if (forbidden_name(c, loc, key->arg, Store)) { - return -1; + return ERROR; } for (Py_ssize_t j = i + 1; j < nkeywords; j++) { keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { compiler_error(c, LOC(other), "keyword argument repeated: %U", key->arg); - return -1; + return ERROR; } } } - return 0; + return SUCCESS; } static int compiler_call(struct compiler *c, expr_ty e) { - if (validate_keywords(c, e->v.Call.keywords) == -1) { - return 0; - } + RETURN_IF_ERROR(validate_keywords(c, e->v.Call.keywords)); int ret = maybe_optimize_method_call(c, e); - if (ret >= 0) { - return ret; + if (ret < 0) { + return ERROR; } - if (!check_caller(c, e->v.Call.func)) { - return 0; + if (ret == 1) { + return SUCCESS; } + RETURN_IF_ERROR(check_caller(c, e->v.Call.func)); location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); VISIT(c, expr, e->v.Call.func); @@ -4963,7 +4910,7 @@ compiler_joined_str(struct compiler *c, expr_ty e) ADDOP_I(c, loc, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); } } - return 1; + return SUCCESS; } /* Used to implement f-strings. Format a single value. */ @@ -4998,7 +4945,7 @@ compiler_formatted_value(struct compiler *c, expr_ty e) default: PyErr_Format(PyExc_SystemError, "Unrecognized conversion character %d", conversion); - return 0; + return ERROR; } if (e->v.FormattedValue.format_spec) { /* Evaluate the format spec, and update our opcode arg. */ @@ -5010,7 +4957,7 @@ compiler_formatted_value(struct compiler *c, expr_ty e) location loc = LOC(e); ADDOP_I(c, loc, FORMAT_VALUE, oparg); - return 1; + return SUCCESS; } static int @@ -5030,7 +4977,7 @@ compiler_subkwargs(struct compiler *c, location loc, } keys = PyTuple_New(n); if (keys == NULL) { - return 0; + return ERROR; } for (i = begin; i < end; i++) { key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg; @@ -5038,7 +4985,7 @@ compiler_subkwargs(struct compiler *c, location loc, } ADDOP_LOAD_CONST_NEW(c, loc, keys); ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return 1; + return SUCCESS; } if (big) { ADDOP_I(c, NO_LOCATION, BUILD_MAP, 0); @@ -5054,12 +5001,11 @@ compiler_subkwargs(struct compiler *c, location loc, if (!big) { ADDOP_I(c, loc, BUILD_MAP, n); } - return 1; + return SUCCESS; } /* Used by compiler_call_helper and maybe_optimize_method_call to emit * KW_NAMES before CALL. - * Returns 1 on success, 0 on error. */ static int compiler_call_simple_kw_helper(struct compiler *c, location loc, @@ -5068,7 +5014,7 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, PyObject *names; names = PyTuple_New(nkwelts); if (names == NULL) { - return 0; + return ERROR; } for (int i = 0; i < nkwelts; i++) { keyword_ty kw = asdl_seq_GET(keywords, i); @@ -5076,11 +5022,11 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, } Py_ssize_t arg = compiler_add_const(c, names); if (arg < 0) { - return 0; + return ERROR; } Py_DECREF(names); ADDOP_I(c, loc, KW_NAMES, arg); - return 1; + return SUCCESS; } @@ -5093,9 +5039,7 @@ compiler_call_helper(struct compiler *c, location loc, { Py_ssize_t i, nseen, nelts, nkwelts; - if (validate_keywords(c, keywords) == -1) { - return 0; - } + RETURN_IF_ERROR(validate_keywords(c, keywords)); nelts = asdl_seq_LEN(args); nkwelts = asdl_seq_LEN(keywords); @@ -5124,12 +5068,11 @@ compiler_call_helper(struct compiler *c, location loc, } if (nkwelts) { VISIT_SEQ(c, keyword, keywords); - if (!compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)) { - return 0; - }; + RETURN_IF_ERROR( + compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)); } ADDOP_I(c, loc, CALL, n + nelts + nkwelts); - return 1; + return SUCCESS; ex_call: @@ -5138,8 +5081,8 @@ compiler_call_helper(struct compiler *c, location loc, VISIT(c, expr, ((expr_ty)asdl_seq_GET(args, 0))->v.Starred.value); } else if (starunpack_helper(c, loc, args, n, BUILD_LIST, - LIST_APPEND, LIST_EXTEND, 1) == 0) { - return 0; + LIST_APPEND, LIST_EXTEND, 1) < 0) { + return ERROR; } /* Then keyword arguments */ if (nkwelts) { @@ -5152,9 +5095,7 @@ compiler_call_helper(struct compiler *c, location loc, if (kw->arg == NULL) { /* A keyword argument unpacking. */ if (nseen) { - if (!compiler_subkwargs(c, loc, keywords, i - nseen, i)) { - return 0; - } + RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, i - nseen, i)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5174,9 +5115,7 @@ compiler_call_helper(struct compiler *c, location loc, } if (nseen) { /* Pack up any trailing keyword arguments. */ - if (!compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)) { - return 0; - } + RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); if (have_dict) { ADDOP_I(c, loc, DICT_MERGE, 1); } @@ -5185,7 +5124,7 @@ compiler_call_helper(struct compiler *c, location loc, assert(have_dict); } ADDOP_I(c, loc, CALL_FUNCTION_EX, nkwelts > 0); - return 1; + return SUCCESS; } @@ -5280,17 +5219,14 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - if (!compiler_jump_if(c, loc, e, if_cleanup, 0)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); } if (++gen_index < asdl_seq_LEN(generators)) { - if (!compiler_comprehension_generator(c, loc, - generators, gen_index, depth, - elt, val, type)) { - return 0; - } + RETURN_IF_ERROR( + compiler_comprehension_generator(c, loc, + generators, gen_index, depth, + elt, val, type)); } location elt_loc = LOC(elt); @@ -5324,7 +5260,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); break; default: - return 0; + return ERROR; } } @@ -5336,7 +5272,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, ADDOP(c, NO_LOCATION, END_FOR); } - return 1; + return SUCCESS; } static int @@ -5365,10 +5301,9 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, USE_LABEL(c, start); /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, loc, ASYNC_COMPREHENSION_GENERATOR, - start, NO_LABEL, NULL)) { - return 0; - } + RETURN_IF_ERROR( + compiler_push_fblock(c, loc, ASYNC_COMPREHENSION_GENERATOR, + start, NO_LABEL, NULL)); ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); @@ -5380,18 +5315,15 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, Py_ssize_t n = asdl_seq_LEN(gen->ifs); for (Py_ssize_t i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - if (!compiler_jump_if(c, loc, e, if_cleanup, 0)) { - return 0; - } + RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); } depth++; if (++gen_index < asdl_seq_LEN(generators)) { - if (!compiler_comprehension_generator(c, loc, - generators, gen_index, depth, - elt, val, type)) { - return 0; - } + RETURN_IF_ERROR( + compiler_comprehension_generator(c, loc, + generators, gen_index, depth, + elt, val, type)); } location elt_loc = LOC(elt); @@ -5424,7 +5356,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); break; default: - return 0; + return ERROR; } } @@ -5437,7 +5369,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, ADDOP(c, loc, END_ASYNC_FOR); - return 1; + return SUCCESS; } static int @@ -5453,8 +5385,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); outermost = (comprehension_ty) asdl_seq_GET(generators, 0); - if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno)) + if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, + (void *)e, e->lineno) < 0) { goto error; } @@ -5493,8 +5425,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP_I(c, loc, op, 0); } - if (!compiler_comprehension_generator(c, loc, generators, 0, 0, - elt, val, type)) { + if (compiler_comprehension_generator(c, loc, generators, 0, 0, + elt, val, type) < 0) { goto error_in_scope; } @@ -5502,7 +5434,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP(c, LOC(e), RETURN_VALUE); } if (type == COMP_GENEXP) { - if (!wrap_in_stopiteration_handler(c)) { + if (wrap_in_stopiteration_handler(c) < 0) { goto error_in_scope; } } @@ -5513,11 +5445,12 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, if (is_top_level_await && is_async_generator){ c->u->u_ste->ste_coroutine = 1; } - if (co == NULL) + if (co == NULL) { goto error; + } loc = LOC(e); - if (!compiler_make_closure(c, loc, co, 0, qualname)) { + if (compiler_make_closure(c, loc, co, 0, qualname) < 0) { goto error; } Py_DECREF(qualname); @@ -5540,13 +5473,13 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADD_YIELD_FROM(c, loc, 1); } - return 1; + return SUCCESS; error_in_scope: compiler_exit_scope(c); error: Py_XDECREF(qualname); Py_XDECREF(co); - return 0; + return ERROR; } static int @@ -5595,7 +5528,7 @@ static int compiler_visit_keyword(struct compiler *c, keyword_ty k) { VISIT(c, expr, k->value); - return 1; + return SUCCESS; } @@ -5618,7 +5551,7 @@ compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); USE_LABEL(c, exit); - return 1; + return SUCCESS; } /* @@ -5675,15 +5608,13 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* SETUP_WITH pushes a finally block. */ USE_LABEL(c, block); - if (!compiler_push_fblock(c, loc, ASYNC_WITH, block, final, s)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, ASYNC_WITH, block, final, s)); if (item->optional_vars) { VISIT(c, expr, item->optional_vars); } else { - /* Discard result from context.__aenter__() */ + /* Discard result from context.__aenter__() */ ADDOP(c, loc, POP_TOP); } @@ -5692,8 +5623,8 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* BLOCK code */ VISIT_SEQ(c, stmt, s->v.AsyncWith.body) } - else if (!compiler_async_with(c, s, pos)) { - return 0; + else { + RETURN_IF_ERROR(compiler_async_with(c, s, pos)); } compiler_pop_fblock(c, ASYNC_WITH, block); @@ -5704,8 +5635,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - if(!compiler_call_exit_with_nones(c, loc)) - return 0; + RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -5723,10 +5653,10 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); - compiler_with_except_finish(c, cleanup); + RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); USE_LABEL(c, exit); - return 1; + return SUCCESS; } @@ -5772,9 +5702,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* SETUP_WITH pushes a finally block. */ USE_LABEL(c, block); - if (!compiler_push_fblock(c, loc, WITH, block, final, s)) { - return 0; - } + RETURN_IF_ERROR(compiler_push_fblock(c, loc, WITH, block, final, s)); if (item->optional_vars) { VISIT(c, expr, item->optional_vars); @@ -5785,11 +5713,13 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) } pos++; - if (pos == asdl_seq_LEN(s->v.With.items)) + if (pos == asdl_seq_LEN(s->v.With.items)) { /* BLOCK code */ VISIT_SEQ(c, stmt, s->v.With.body) - else if (!compiler_with(c, s, pos)) - return 0; + } + else { + RETURN_IF_ERROR(compiler_with(c, s, pos)); + } ADDOP(c, NO_LOCATION, POP_BLOCK); compiler_pop_fblock(c, WITH, block); @@ -5800,8 +5730,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) * call __exit__(None, None, None) */ loc = LOC(s); - if (!compiler_call_exit_with_nones(c, loc)) - return 0; + RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); ADDOP(c, loc, POP_TOP); ADDOP_JUMP(c, loc, JUMP, exit); @@ -5811,10 +5740,10 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); - compiler_with_except_finish(c, cleanup); + RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); USE_LABEL(c, exit); - return 1; + return SUCCESS; } static int @@ -5855,8 +5784,9 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case DictComp_kind: return compiler_dictcomp(c, e); case Yield_kind: - if (c->u->u_ste->ste_type != FunctionBlock) + if (c->u->u_ste->ste_type != FunctionBlock) { return compiler_error(c, loc, "'yield' outside function"); + } if (e->v.Yield.value) { VISIT(c, expr, e->v.Yield.value); } @@ -5866,12 +5796,12 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP_YIELD(c, loc); break; case YieldFrom_kind: - if (c->u->u_ste->ste_type != FunctionBlock) + if (c->u->u_ste->ste_type != FunctionBlock) { return compiler_error(c, loc, "'yield' outside function"); - - if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) + } + if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) { return compiler_error(c, loc, "'yield from' inside async function"); - + } VISIT(c, expr, e->v.YieldFrom.value); ADDOP(c, loc, GET_YIELD_FROM_ITER); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5884,7 +5814,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) } if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){ + c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) { return compiler_error(c, loc, "'await' outside async function"); } } @@ -5916,7 +5846,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) break; case Store: if (forbidden_name(c, loc, e->v.Attribute.attr, e->v.Attribute.ctx)) { - return 0; + return ERROR; } ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); break; @@ -5942,9 +5872,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case Slice_kind: { int n = compiler_slice(c, e); - if (n == 0) { - return 0; - } + RETURN_IF_ERROR(n); ADDOP_I(c, loc, BUILD_SLICE, n); break; } @@ -5956,7 +5884,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case Tuple_kind: return compiler_tuple(c, e); } - return 1; + return SUCCESS; } static int @@ -5991,9 +5919,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) case Subscript_kind: VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice)) { - if (!compiler_slice(c, e->v.Subscript.slice)) { - return 0; - } + RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); @@ -6007,14 +5933,13 @@ compiler_augassign(struct compiler *c, stmt_ty s) } break; case Name_kind: - if (!compiler_nameop(c, loc, e->v.Name.id, Load)) - return 0; + RETURN_IF_ERROR(compiler_nameop(c, loc, e->v.Name.id, Load)); break; default: PyErr_Format(PyExc_SystemError, "invalid node type (%d) for augmented assignment", e->kind); - return 0; + return ERROR; } loc = LOC(s); @@ -6048,7 +5973,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) default: Py_UNREACHABLE(); } - return 1; + return SUCCESS; } static int @@ -6056,7 +5981,7 @@ check_ann_expr(struct compiler *c, expr_ty e) { VISIT(c, expr, e); ADDOP(c, LOC(e), POP_TOP); - return 1; + return SUCCESS; } static int @@ -6065,7 +5990,7 @@ check_annotation(struct compiler *c, stmt_ty s) /* Annotations of complex targets does not produce anything under annotations future */ if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - return 1; + return SUCCESS; } /* Annotations are only evaluated in a module or class. */ @@ -6073,7 +5998,7 @@ check_annotation(struct compiler *c, stmt_ty s) c->u->u_scope_type == COMPILER_SCOPE_CLASS) { return check_ann_expr(c, s->v.AnnAssign.annotation); } - return 1; + return SUCCESS; } static int @@ -6082,26 +6007,24 @@ check_ann_subscr(struct compiler *c, expr_ty e) /* We check that everything in a subscript is defined at runtime. */ switch (e->kind) { case Slice_kind: - if (e->v.Slice.lower && !check_ann_expr(c, e->v.Slice.lower)) { - return 0; + if (e->v.Slice.lower && check_ann_expr(c, e->v.Slice.lower) < 0) { + return ERROR; } - if (e->v.Slice.upper && !check_ann_expr(c, e->v.Slice.upper)) { - return 0; + if (e->v.Slice.upper && check_ann_expr(c, e->v.Slice.upper) < 0) { + return ERROR; } - if (e->v.Slice.step && !check_ann_expr(c, e->v.Slice.step)) { - return 0; + if (e->v.Slice.step && check_ann_expr(c, e->v.Slice.step) < 0) { + return ERROR; } - return 1; + return SUCCESS; case Tuple_kind: { /* extended slice */ asdl_expr_seq *elts = e->v.Tuple.elts; Py_ssize_t i, n = asdl_seq_LEN(elts); for (i = 0; i < n; i++) { - if (!check_ann_subscr(c, asdl_seq_GET(elts, i))) { - return 0; - } + RETURN_IF_ERROR(check_ann_subscr(c, asdl_seq_GET(elts, i))); } - return 1; + return SUCCESS; } default: return check_ann_expr(c, e); @@ -6124,8 +6047,9 @@ compiler_annassign(struct compiler *c, stmt_ty s) } switch (targ->kind) { case Name_kind: - if (forbidden_name(c, loc, targ->v.Name.id, Store)) - return 0; + if (forbidden_name(c, loc, targ->v.Name.id, Store)) { + return ERROR; + } /* If we have a simple name in a module or class, store annotation. */ if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || @@ -6143,31 +6067,32 @@ compiler_annassign(struct compiler *c, stmt_ty s) } break; case Attribute_kind: - if (forbidden_name(c, loc, targ->v.Attribute.attr, Store)) - return 0; + if (forbidden_name(c, loc, targ->v.Attribute.attr, Store)) { + return ERROR; + } if (!s->v.AnnAssign.value && - !check_ann_expr(c, targ->v.Attribute.value)) { - return 0; + check_ann_expr(c, targ->v.Attribute.value) < 0) { + return ERROR; } break; case Subscript_kind: if (!s->v.AnnAssign.value && - (!check_ann_expr(c, targ->v.Subscript.value) || - !check_ann_subscr(c, targ->v.Subscript.slice))) { - return 0; + (check_ann_expr(c, targ->v.Subscript.value) < 0 || + check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { + return ERROR; } break; default: PyErr_Format(PyExc_SystemError, "invalid node type (%d) for annotated assignment", targ->kind); - return 0; + return ERROR; } /* Annotation is evaluated last. */ - if (!s->v.AnnAssign.simple && !check_annotation(c, s)) { - return 0; + if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + return ERROR; } - return 1; + return SUCCESS; } /* Raises a SyntaxError and returns 0. @@ -6183,7 +6108,7 @@ compiler_error(struct compiler *c, location loc, PyObject *msg = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg == NULL) { - return 0; + return ERROR; } PyObject *loc_obj = PyErr_ProgramTextObject(c->c_filename, loc.lineno); if (loc_obj == NULL) { @@ -6200,7 +6125,7 @@ compiler_error(struct compiler *c, location loc, exit: Py_DECREF(loc_obj); Py_XDECREF(args); - return 0; + return ERROR; } /* Emits a SyntaxWarning and returns 1 on success. @@ -6216,7 +6141,7 @@ compiler_warn(struct compiler *c, location loc, PyObject *msg = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg == NULL) { - return 0; + return ERROR; } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, c->c_filename, loc.lineno, NULL, NULL) < 0) @@ -6229,10 +6154,10 @@ compiler_warn(struct compiler *c, location loc, compiler_error(c, loc, PyUnicode_AsUTF8(msg)); } Py_DECREF(msg); - return 0; + return ERROR; } Py_DECREF(msg); - return 1; + return SUCCESS; } static int @@ -6243,19 +6168,13 @@ compiler_subscript(struct compiler *c, expr_ty e) int op = 0; if (ctx == Load) { - if (!check_subscripter(c, e->v.Subscript.value)) { - return 0; - } - if (!check_index(c, e->v.Subscript.value, e->v.Subscript.slice)) { - return 0; - } + RETURN_IF_ERROR(check_subscripter(c, e->v.Subscript.value)); + RETURN_IF_ERROR(check_index(c, e->v.Subscript.value, e->v.Subscript.slice)); } VISIT(c, expr, e->v.Subscript.value); if (is_two_element_slice(e->v.Subscript.slice) && ctx != Del) { - if (!compiler_slice(c, e->v.Subscript.slice)) { - return 0; - } + RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); if (ctx == Load) { ADDOP(c, loc, BINARY_SLICE); } @@ -6274,11 +6193,11 @@ compiler_subscript(struct compiler *c, expr_ty e) assert(op); ADDOP(c, loc, op); } - return 1; + return SUCCESS; } /* Returns the number of the values emitted, - * thus are needed to build the slice, or 0 if there is an error. */ + * thus are needed to build the slice, or -1 if there is an error. */ static int compiler_slice(struct compiler *c, expr_ty s) { @@ -6336,20 +6255,20 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) { Py_ssize_t size = n + 1; if (size <= pc->fail_pop_size) { - return 1; + return SUCCESS; } Py_ssize_t needed = sizeof(jump_target_label) * size; jump_target_label *resized = PyObject_Realloc(pc->fail_pop, needed); if (resized == NULL) { PyErr_NoMemory(); - return 0; + return ERROR; } pc->fail_pop = resized; while (pc->fail_pop_size < size) { NEW_JUMP_TARGET_LABEL(c, new_block); pc->fail_pop[pc->fail_pop_size++] = new_block; } - return 1; + return SUCCESS; } // Use op to jump to the correct fail_pop block. @@ -6360,9 +6279,9 @@ jump_to_fail_pop(struct compiler *c, location loc, // Pop any items on the top of the stack, plus any objects we were going to // capture on success: Py_ssize_t pops = pc->on_top + PyList_GET_SIZE(pc->stores); - RETURN_IF_FALSE(ensure_fail_pop(c, pc, pops)); + RETURN_IF_ERROR(ensure_fail_pop(c, pc, pops)); ADDOP_JUMP(c, loc, op, pc->fail_pop[pops]); - return 1; + return SUCCESS; } // Build all of the fail_pop blocks and reset fail_pop. @@ -6372,21 +6291,21 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, { if (!pc->fail_pop_size) { assert(pc->fail_pop == NULL); - return 1; + return SUCCESS; } while (--pc->fail_pop_size) { USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); - if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc)) { + if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; - return 0; + return ERROR; } } USE_LABEL(c, pc->fail_pop[0]); PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; - return 1; + return SUCCESS; } static int @@ -6403,7 +6322,7 @@ pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) while (1 < count) { ADDOP_I(c, loc, SWAP, count--); } - return 1; + return SUCCESS; } static int @@ -6412,23 +6331,22 @@ pattern_helper_store_name(struct compiler *c, location loc, { if (n == NULL) { ADDOP(c, loc, POP_TOP); - return 1; + return SUCCESS; } if (forbidden_name(c, loc, n, Store)) { - return 0; + return ERROR; } // Can't assign to the same name twice: int duplicate = PySequence_Contains(pc->stores, n); - if (duplicate < 0) { - return 0; - } + RETURN_IF_ERROR(duplicate); if (duplicate) { return compiler_error_duplicate_store(c, loc, n); } // Rotate this object underneath any items we need to preserve: Py_ssize_t rotations = pc->on_top + PyList_GET_SIZE(pc->stores) + 1; - RETURN_IF_FALSE(pattern_helper_rotate(c, loc, rotations)); - return !PyList_Append(pc->stores, n); + RETURN_IF_ERROR(pattern_helper_rotate(c, loc, rotations)); + RETURN_IF_ERROR(PyList_Append(pc->stores, n)); + return SUCCESS; } @@ -6442,10 +6360,11 @@ pattern_unpack_helper(struct compiler *c, location loc, pattern_ty elt = asdl_seq_GET(elts, i); if (elt->kind == MatchStar_kind && !seen_star) { if ((i >= (1 << 8)) || - (n-i-1 >= (INT_MAX >> 8))) + (n-i-1 >= (INT_MAX >> 8))) { return compiler_error(c, loc, "too many expressions in " "star-unpacking sequence pattern"); + } ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); seen_star = 1; } @@ -6457,7 +6376,7 @@ pattern_unpack_helper(struct compiler *c, location loc, if (!seen_star) { ADDOP_I(c, loc, UNPACK_SEQUENCE, n); } - return 1; + return SUCCESS; } static int @@ -6465,7 +6384,7 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, asdl_pattern_seq *patterns, Py_ssize_t star, pattern_context *pc) { - RETURN_IF_FALSE(pattern_unpack_helper(c, loc, patterns)); + RETURN_IF_ERROR(pattern_unpack_helper(c, loc, patterns)); Py_ssize_t size = asdl_seq_LEN(patterns); // We've now got a bunch of new subjects on the stack. They need to remain // there after each subpattern match: @@ -6474,9 +6393,9 @@ pattern_helper_sequence_unpack(struct compiler *c, location loc, // One less item to keep track of each time we loop through: pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } - return 1; + return SUCCESS; } // Like pattern_helper_sequence_unpack, but uses BINARY_SUBSCR instead of @@ -6511,12 +6430,12 @@ pattern_helper_sequence_subscr(struct compiler *c, location loc, ADDOP_BINARY(c, loc, Sub); } ADDOP(c, loc, BINARY_SUBSCR); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } // Pop the subject, we're done with it: pc->on_top--; ADDOP(c, loc, POP_TOP); - return 1; + return SUCCESS; } // Like compiler_pattern, but turn off checks for irrefutability. @@ -6526,9 +6445,9 @@ compiler_pattern_subpattern(struct compiler *c, { int allow_irrefutable = pc->allow_irrefutable; pc->allow_irrefutable = 1; - RETURN_IF_FALSE(compiler_pattern(c, p, pc)); + RETURN_IF_ERROR(compiler_pattern(c, p, pc)); pc->allow_irrefutable = allow_irrefutable; - return 1; + return SUCCESS; } static int @@ -6550,19 +6469,20 @@ compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to make a copy for (possibly) storing later: pc->on_top++; ADDOP_I(c, LOC(p), COPY, 1); - RETURN_IF_FALSE(compiler_pattern(c, p->v.MatchAs.pattern, pc)); + RETURN_IF_ERROR(compiler_pattern(c, p->v.MatchAs.pattern, pc)); // Success! Store it: pc->on_top--; - RETURN_IF_FALSE(pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); - return 1; + RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); + return SUCCESS; } static int compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchStar_kind); - RETURN_IF_FALSE(pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); - return 1; + RETURN_IF_ERROR( + pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); + return SUCCESS; } static int @@ -6575,18 +6495,18 @@ validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_ identifier attr = ((identifier)asdl_seq_GET(attrs, i)); location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); if (forbidden_name(c, loc, attr, Store)) { - return -1; + return ERROR; } for (Py_ssize_t j = i + 1; j < nattrs; j++) { identifier other = ((identifier)asdl_seq_GET(attrs, j)); if (!PyUnicode_Compare(attr, other)) { location loc = LOC((pattern_ty) asdl_seq_GET(patterns, j)); compiler_error(c, loc, "attribute name repeated in class pattern: %U", attr); - return -1; + return ERROR; } } } - return 0; + return SUCCESS; } static int @@ -6610,11 +6530,13 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) return compiler_error(c, LOC(p), e, p->v.MatchClass.cls); } if (nattrs) { - RETURN_IF_FALSE(!validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); + RETURN_IF_ERROR(validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); } VISIT(c, expr, p->v.MatchClass.cls); - PyObject *attr_names; - RETURN_IF_FALSE(attr_names = PyTuple_New(nattrs)); + PyObject *attr_names = PyTuple_New(nattrs); + if (attr_names == NULL) { + return ERROR; + } Py_ssize_t i; for (i = 0; i < nattrs; i++) { PyObject *name = asdl_seq_GET(kwd_attrs, i); @@ -6627,7 +6549,7 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) ADDOP_I(c, LOC(p), IS_OP, 1); // TOS is now a tuple of (nargs + nattrs) attributes (or None): pc->on_top++; - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, nargs + nattrs); pc->on_top += nargs + nattrs - 1; for (i = 0; i < nargs + nattrs; i++) { @@ -6645,10 +6567,10 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) ADDOP(c, LOC(p), POP_TOP); continue; } - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } // Success! Pop the tuple of attributes: - return 1; + return SUCCESS; } static int @@ -6671,19 +6593,19 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, // We need to keep the subject on top during the mapping and length checks: pc->on_top++; ADDOP(c, LOC(p), MATCH_MAPPING); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); if (!size && !star_target) { // If the pattern is just "{}", we're done! Pop the subject: pc->on_top--; ADDOP(c, LOC(p), POP_TOP); - return 1; + return SUCCESS; } if (size) { // If the pattern has any keys in it, perform a length check: ADDOP(c, LOC(p), GET_LEN); ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); ADDOP_COMPARE(c, LOC(p), GtE); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); } if (INT_MAX < size - 1) { return compiler_error(c, LOC(p), "too many sub-patterns in mapping pattern"); @@ -6695,7 +6617,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, // SyntaxError in the case of duplicates. PyObject *seen = PySet_New(NULL); if (seen == NULL) { - return 0; + return ERROR; } // NOTE: goto error on failure in the loop below to avoid leaking `seen` @@ -6729,7 +6651,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, compiler_error(c, LOC(p), e); goto error; } - if (!compiler_visit_expr(c, key)) { + if (compiler_visit_expr(c, key) < 0) { goto error; } } @@ -6744,7 +6666,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, ADDOP_I(c, LOC(p), COPY, 1); ADDOP_LOAD_CONST(c, LOC(p), Py_None); ADDOP_I(c, LOC(p), IS_OP, 1); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); // So far so good. Use that tuple of values on the stack to match // sub-patterns against: ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, size); @@ -6752,7 +6674,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, for (Py_ssize_t i = 0; i < size; i++) { pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); } // If we get this far, it's a match! Whatever happens next should consume // the tuple of keys and the subject: @@ -6773,17 +6695,17 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, ADDOP_I(c, LOC(p), SWAP, 2); // [copy, keys..., copy, key] ADDOP(c, LOC(p), DELETE_SUBSCR); // [copy, keys...] } - RETURN_IF_FALSE(pattern_helper_store_name(c, LOC(p), star_target, pc)); + RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), star_target, pc)); } else { ADDOP(c, LOC(p), POP_TOP); // Tuple of keys. ADDOP(c, LOC(p), POP_TOP); // Subject. } - return 1; + return SUCCESS; error: Py_DECREF(seen); - return 0; + return ERROR; } static int @@ -6813,8 +6735,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; - if (!cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, LOC(alt)) || - !compiler_pattern(c, alt, pc)) { + if (cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, LOC(alt)) < 0 || + compiler_pattern(c, alt, pc) < 0) { goto error; } // Success! @@ -6868,7 +6790,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Do the same thing to the stack, using several // rotations: while (rotations--) { - if (!pattern_helper_rotate(c, LOC(alt), icontrol + 1)){ + if (pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { goto error; } } @@ -6876,8 +6798,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) } } assert(control); - if (!cfg_builder_addop_j(CFG_BUILDER(c), LOC(alt), JUMP, end) || - !emit_and_reset_fail_pop(c, LOC(alt), pc)) + if (cfg_builder_addop_j(CFG_BUILDER(c), LOC(alt), JUMP, end) < 0 || + emit_and_reset_fail_pop(c, LOC(alt), pc) < 0) { goto error; } @@ -6888,8 +6810,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to NULL this for the PyObject_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: - if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, LOC(p)) || - !jump_to_fail_pop(c, LOC(p), pc, JUMP)) { + if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, LOC(p)) < 0 || + jump_to_fail_pop(c, LOC(p), pc, JUMP) < 0) { goto error; } @@ -6904,7 +6826,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_ssize_t nrots = nstores + 1 + pc->on_top + PyList_GET_SIZE(pc->stores); for (Py_ssize_t i = 0; i < nstores; i++) { // Rotate this capture to its proper place on the stack: - if (!pattern_helper_rotate(c, LOC(p), nrots)) { + if (pattern_helper_rotate(c, LOC(p), nrots) < 0) { goto error; } // Update the list of previous stores with this new name, checking for @@ -6927,14 +6849,14 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // NOTE: Returning macros are safe again. // Pop the copy of the subject: ADDOP(c, LOC(p), POP_TOP); - return 1; + return SUCCESS; diff: compiler_error(c, LOC(p), "alternative patterns bind different names"); error: PyObject_Free(old_pc.fail_pop); Py_DECREF(old_pc.stores); Py_XDECREF(control); - return 0; + return ERROR; } @@ -6966,20 +6888,20 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, // We need to keep the subject on top during the sequence and length checks: pc->on_top++; ADDOP(c, LOC(p), MATCH_SEQUENCE); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); if (star < 0) { // No star: len(subject) == size ADDOP(c, LOC(p), GET_LEN); ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); ADDOP_COMPARE(c, LOC(p), Eq); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); } else if (size > 1) { // Star: len(subject) >= size - 1 ADDOP(c, LOC(p), GET_LEN); ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size - 1)); ADDOP_COMPARE(c, LOC(p), GtE); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); } // Whatever comes next should consume the subject: pc->on_top--; @@ -6988,12 +6910,12 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, ADDOP(c, LOC(p), POP_TOP); } else if (star_wildcard) { - RETURN_IF_FALSE(pattern_helper_sequence_subscr(c, LOC(p), patterns, star, pc)); + RETURN_IF_ERROR(pattern_helper_sequence_subscr(c, LOC(p), patterns, star, pc)); } else { - RETURN_IF_FALSE(pattern_helper_sequence_unpack(c, LOC(p), patterns, star, pc)); + RETURN_IF_ERROR(pattern_helper_sequence_unpack(c, LOC(p), patterns, star, pc)); } - return 1; + return SUCCESS; } static int @@ -7007,8 +6929,8 @@ compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) } VISIT(c, expr, value); ADDOP_COMPARE(c, LOC(p), Eq); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - return 1; + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + return SUCCESS; } static int @@ -7017,8 +6939,8 @@ compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc assert(p->kind == MatchSingleton_kind); ADDOP_LOAD_CONST(c, LOC(p), p->v.MatchSingleton.value); ADDOP_COMPARE(c, LOC(p), Is); - RETURN_IF_FALSE(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - return 1; + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + return SUCCESS; } static int @@ -7063,32 +6985,35 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) if (i != cases - has_default - 1) { ADDOP_I(c, LOC(m->pattern), COPY, 1); } - RETURN_IF_FALSE(pc->stores = PyList_New(0)); + pc->stores = PyList_New(0); + if (pc->stores == NULL) { + return ERROR; + } // Irrefutable cases must be either guarded, last, or both: pc->allow_irrefutable = m->guard != NULL || i == cases - 1; pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; // NOTE: Can't use returning macros here (they'll leak pc->stores)! - if (!compiler_pattern(c, m->pattern, pc)) { + if (compiler_pattern(c, m->pattern, pc) < 0) { Py_DECREF(pc->stores); - return 0; + return ERROR; } assert(!pc->on_top); // It's a match! Store all of the captured names (they're on the stack). Py_ssize_t nstores = PyList_GET_SIZE(pc->stores); for (Py_ssize_t n = 0; n < nstores; n++) { PyObject *name = PyList_GET_ITEM(pc->stores, n); - if (!compiler_nameop(c, LOC(m->pattern), name, Store)) { + if (compiler_nameop(c, LOC(m->pattern), name, Store) < 0) { Py_DECREF(pc->stores); - return 0; + return ERROR; } } Py_DECREF(pc->stores); // NOTE: Returning macros are safe again. if (m->guard) { - RETURN_IF_FALSE(ensure_fail_pop(c, pc, 0)); - RETURN_IF_FALSE(compiler_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); + RETURN_IF_ERROR(ensure_fail_pop(c, pc, 0)); + RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); } // Success! Pop the subject off, we're done with it: if (i != cases - has_default - 1) { @@ -7099,7 +7024,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) // If the pattern fails to match, we want the line number of the // cleanup to be associated with the failed pattern, not the last line // of the body - RETURN_IF_FALSE(emit_and_reset_fail_pop(c, LOC(m->pattern), pc)); + RETURN_IF_ERROR(emit_and_reset_fail_pop(c, LOC(m->pattern), pc)); } if (has_default) { // A trailing "case _" is common, and lets us save a bit of redundant @@ -7114,12 +7039,12 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) ADDOP(c, LOC(m->pattern), NOP); } if (m->guard) { - RETURN_IF_FALSE(compiler_jump_if(c, LOC(m->pattern), m->guard, end, 0)); + RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, end, 0)); } VISIT_SEQ(c, stmt, m->body); } USE_LABEL(c, end); - return 1; + return SUCCESS; } static int @@ -8792,6 +8717,19 @@ prepare_localsplus(struct compiler* c, int code_flags) return nlocalsplus; } +static int +add_return_at_end_of_block(struct compiler *c, int addNone) +{ + /* Make sure every block that falls off the end returns None. */ + if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { + if (addNone) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + } + ADDOP(c, NO_LOCATION, RETURN_VALUE); + } + return SUCCESS; +} + static PyCodeObject * assemble(struct compiler *c, int addNone) { @@ -8805,12 +8743,8 @@ assemble(struct compiler *c, int addNone) return NULL; } - /* Make sure every block that falls off the end returns None. */ - if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { - if (addNone) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - } - ADDOP(c, NO_LOCATION, RETURN_VALUE); + if (add_return_at_end_of_block(c, addNone) < 0) { + return NULL; } int nblocks = 0; @@ -9992,7 +9926,7 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) if (PyErr_Occurred()) { return -1; } - if (!cfg_builder_addop(g, opcode, oparg, loc)) { + if (cfg_builder_addop(g, opcode, oparg, loc) < 0) { return -1; } } @@ -10077,7 +10011,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - if (!compiler_codegen(c, mod)) { + if (compiler_codegen(c, mod) < 0) { goto finally; } From webhook-mailer at python.org Mon Dec 12 09:51:05 2022 From: webhook-mailer at python.org (markshannon) Date: Mon, 12 Dec 2022 14:51:05 -0000 Subject: [Python-checkins] GH-100143: Improve collecting pystats for parts of runs (GH-100144) Message-ID: <mailman.2912.1670856667.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1583c6e326a8454d3c806763620e1329bf6b7cbe commit: 1583c6e326a8454d3c806763620e1329bf6b7cbe branch: main author: Michael Droettboom <mdboom at gmail.com> committer: markshannon <mark at hotpy.org> date: 2022-12-12T14:50:43Z summary: GH-100143: Improve collecting pystats for parts of runs (GH-100144) * pystats off by default * Add -Xpystats flag * Always dump pystats, even if turned off files: A Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst M Python/initconfig.c M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst new file mode 100644 index 000000000000..20a25f8b03d1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-09-14-27-36.gh-issue-100143.5g9rb4.rst @@ -0,0 +1,3 @@ +When built with ``--enable-pystats``, stats collection is now off by +default. To enable it early at startup, pass the ``-Xpystats`` flag. Stats +are now always dumped, even if switched off. diff --git a/Python/initconfig.c b/Python/initconfig.c index 64ae987b3f34..d05099cd9977 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -129,7 +129,14 @@ The following implementation-specific options are available:\n\ \n\ -X int_max_str_digits=number: limit the size of int<->str conversions.\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ - The default is sys.int_info.default_max_str_digits. 0 disables."; + The default is sys.int_info.default_max_str_digits. 0 disables." + +#ifdef Py_STATS +"\n\ +\n\ +-X pystats: Enable pystats collection at startup." +#endif +; /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = @@ -2186,6 +2193,12 @@ config_read(PyConfig *config, int compute_path_config) config->show_ref_count = 1; } +#ifdef Py_STATS + if (config_get_xoption(config, L"pystats")) { + _py_stats = &_py_stats_struct; + } +#endif + status = config_read_complex_options(config); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Python/specialize.c b/Python/specialize.c index 7545a7712493..785088eac8c5 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -18,7 +18,7 @@ #ifdef Py_STATS PyStats _py_stats_struct = { 0 }; -PyStats *_py_stats = &_py_stats_struct; +PyStats *_py_stats = NULL; #define ADD_STAT_TO_DICT(res, field) \ do { \ @@ -205,9 +205,6 @@ _Py_StatsClear(void) void _Py_PrintSpecializationStats(int to_file) { - if (_py_stats == NULL) { - return; - } FILE *out = stderr; if (to_file) { /* Write to a file instead of stderr. */ @@ -238,7 +235,7 @@ _Py_PrintSpecializationStats(int to_file) else { fprintf(out, "Specialization stats:\n"); } - print_stats(out, _py_stats); + print_stats(out, &_py_stats_struct); if (out != stderr) { fclose(out); } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index c15501bdc761..c30a60e9514b 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -34,6 +34,16 @@ TOTAL = "specialization.deferred", "specialization.hit", "specialization.miss", "execution_count" +def format_ratio(num, den): + """ + Format a ratio as a percentage. When the denominator is 0, returns the empty + string. + """ + if den == 0: + return "" + else: + return f"{num/den:.01%}" + def join_rows(a_rows, b_rows): """ Joins two tables together, side-by-side, where the first column in each is a @@ -87,7 +97,7 @@ def calculate_specialization_stats(family_stats, total): continue else: label = key - rows.append((f"{label:>12}", f"{family_stats[key]:>12}", f"{100*family_stats[key]/total:0.1f}%")) + rows.append((f"{label:>12}", f"{family_stats[key]:>12}", format_ratio(family_stats[key], total))) return rows def calculate_specialization_success_failure(family_stats): @@ -100,7 +110,7 @@ def calculate_specialization_success_failure(family_stats): label = key[len("specialization."):] label = label[0].upper() + label[1:] val = family_stats.get(key, 0) - rows.append((label, val, f"{100*val/total_attempts:0.1f}%")) + rows.append((label, val, format_ratio(val, total_attempts))) return rows def calculate_specialization_failure_kinds(name, family_stats, defines): @@ -118,7 +128,7 @@ def calculate_specialization_failure_kinds(name, family_stats, defines): for value, index in failures: if not value: continue - rows.append((kind_to_text(index, defines, name), value, f"{100*value/total_failures:0.1f}%")) + rows.append((kind_to_text(index, defines, name), value, format_ratio(value, total_failures))) return rows def print_specialization_stats(name, family_stats, defines): @@ -318,11 +328,11 @@ def calculate_execution_counts(opcode_stats, total): for (count, name, miss) in counts: cumulative += count if miss: - miss = f"{100*miss/count:0.1f}%" + miss = format_ratio(miss, count) else: miss = "" - rows.append((name, count, f"{100*count/total:0.1f}%", - f"{100*cumulative/total:0.1f}%", miss)) + rows.append((name, count, format_ratio(count, total), + format_ratio(cumulative, total), miss)) return rows def emit_execution_counts(opcode_stats, total): @@ -386,9 +396,9 @@ def emit_comparative_specialization_stats(base_opcode_stats, head_opcode_stats): def calculate_specialization_effectiveness(opcode_stats, total): basic, not_specialized, specialized = categorized_counts(opcode_stats) return [ - ("Basic", basic, f"{basic*100/total:0.1f}%"), - ("Not specialized", not_specialized, f"{not_specialized*100/total:0.1f}%"), - ("Specialized", specialized, f"{specialized*100/total:0.1f}%"), + ("Basic", basic, format_ratio(basic, total)), + ("Not specialized", not_specialized, format_ratio(not_specialized, total)), + ("Specialized", specialized, format_ratio(specialized, total)), ] def emit_specialization_overview(opcode_stats, total): @@ -405,7 +415,7 @@ def emit_specialization_overview(opcode_stats, total): counts.sort(reverse=True) if total: with Section(f"{title} by instruction", 3): - rows = [ (name, count, f"{100*count/total:0.1f}%") for (count, name) in counts[:10] ] + rows = [ (name, count, format_ratio(count, total)) for (count, name) in counts[:10] ] emit_table(("Name", "Count:", "Ratio:"), rows) def emit_comparative_specialization_overview(base_opcode_stats, base_total, head_opcode_stats, head_total): @@ -432,15 +442,15 @@ def calculate_call_stats(stats): rows = [] for key, value in stats.items(): if "Calls to" in key: - rows.append((key, value, f"{100*value/total:0.1f}%")) + rows.append((key, value, format_ratio(value, total))) elif key.startswith("Calls "): name, index = key[:-1].split("[") index = int(index) label = name + " (" + pretty(defines[index][0]) + ")" - rows.append((label, value, f"{100*value/total:0.1f}%")) + rows.append((label, value, format_ratio(value, total))) for key, value in stats.items(): if key.startswith("Frame"): - rows.append((key, value, f"{100*value/total:0.1f}%")) + rows.append((key, value, format_ratio(value, total))) return rows def emit_call_stats(stats): @@ -468,13 +478,13 @@ def calculate_object_stats(stats): for key, value in stats.items(): if key.startswith("Object"): if "materialize" in key: - ratio = f"{100*value/total_materializations:0.1f}%" + ratio = format_ratio(value, total_materializations) elif "allocations" in key: - ratio = f"{100*value/total_allocations:0.1f}%" + ratio = format_ratio(value, total_allocations) elif "increfs" in key: - ratio = f"{100*value/total_increfs:0.1f}%" + ratio = format_ratio(value, total_increfs) elif "decrefs" in key: - ratio = f"{100*value/total_decrefs:0.1f}%" + ratio = format_ratio(value, total_decrefs) else: ratio = "" label = key[6:].strip() @@ -517,8 +527,8 @@ def emit_pair_counts(opcode_stats, total): for (count, pair) in itertools.islice(pair_counts, 100): i, j = pair cumulative += count - rows.append((opname[i] + " " + opname[j], count, f"{100*count/total:0.1f}%", - f"{100*cumulative/total:0.1f}%")) + rows.append((opname[i] + " " + opname[j], count, format_ratio(count, total), + format_ratio(cumulative, total))) emit_table(("Pair", "Count:", "Self:", "Cumulative:"), rows ) From webhook-mailer at python.org Mon Dec 12 10:44:31 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 15:44:31 -0000 Subject: [Python-checkins] gh-81057: Move tracemalloc Globals to _PyRuntimeState (gh-100151) Message-ID: <mailman.2913.1670859873.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8790d4d31fcd3abaccf31d27f72a8684adfc9dee commit: 8790d4d31fcd3abaccf31d27f72a8684adfc9dee branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T08:44:23-07:00 summary: gh-81057: Move tracemalloc Globals to _PyRuntimeState (gh-100151) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_tracemalloc.h M Include/internal/pycore_pymem.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M Modules/_tracemalloc.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 5749af7465f6..4cc953d8d779 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -90,28 +90,6 @@ PyAPI_FUNC(int) _PyMem_GetAllocatorName( PYMEM_ALLOCATOR_NOT_SET does nothing. */ PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator); -struct _PyTraceMalloc_Config { - /* Module initialized? - Variable protected by the GIL */ - enum { - TRACEMALLOC_NOT_INITIALIZED, - TRACEMALLOC_INITIALIZED, - TRACEMALLOC_FINALIZED - } initialized; - - /* Is tracemalloc tracing memory allocations? - Variable protected by the GIL */ - int tracing; - - /* limit of the number of frames in a traceback, 1 by default. - Variable protected by the GIL. */ - int max_nframe; -}; - -#define _PyTraceMalloc_Config_INIT \ - {.initialized = TRACEMALLOC_NOT_INITIALIZED, \ - .tracing = 0, \ - .max_nframe = 1} #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index fe2de5feb47a..99ec6fc8862b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -23,6 +23,7 @@ extern "C" { #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state #include "pycore_time.h" // struct _time_runtime_state +#include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -137,11 +138,9 @@ typedef struct pyruntimestate { struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; struct _getargs_runtime_state getargs; - struct { - struct _PyTraceMalloc_Config config; - } tracemalloc; struct _dtoa_runtime_state dtoa; struct _fileutils_state fileutils; + struct _tracemalloc_runtime_state tracemalloc; PyPreConfig preconfig; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b569e5833f1d..029357dedf3e 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -50,13 +50,11 @@ extern "C" { in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ }, \ - .tracemalloc = { \ - .config = _PyTraceMalloc_Config_INIT, \ - }, \ .dtoa = _dtoa_runtime_state_INIT(runtime), \ .fileutils = { \ .force_ascii = -1, \ }, \ + .tracemalloc = _tracemalloc_runtime_state_INIT, \ .float_state = { \ .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h new file mode 100644 index 000000000000..08d7d1096c78 --- /dev/null +++ b/Include/internal/pycore_tracemalloc.h @@ -0,0 +1,121 @@ +#ifndef Py_INTERNAL_TRACEMALLOC_H +#define Py_INTERNAL_TRACEMALLOC_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_hashtable.h" // _Py_hashtable_t + + +/* Trace memory blocks allocated by PyMem_RawMalloc() */ +#define TRACE_RAW_MALLOC + + +struct _PyTraceMalloc_Config { + /* Module initialized? + Variable protected by the GIL */ + enum { + TRACEMALLOC_NOT_INITIALIZED, + TRACEMALLOC_INITIALIZED, + TRACEMALLOC_FINALIZED + } initialized; + + /* Is tracemalloc tracing memory allocations? + Variable protected by the GIL */ + int tracing; + + /* limit of the number of frames in a traceback, 1 by default. + Variable protected by the GIL. */ + int max_nframe; +}; + + +/* Pack the frame_t structure to reduce the memory footprint on 64-bit + architectures: 12 bytes instead of 16. */ +struct +#ifdef __GNUC__ +__attribute__((packed)) +#elif defined(_MSC_VER) +#pragma pack(push, 4) +#endif +tracemalloc_frame { + /* filename cannot be NULL: "<unknown>" is used if the Python frame + filename is NULL */ + PyObject *filename; + unsigned int lineno; +}; +#ifdef _MSC_VER +#pragma pack(pop) +#endif + +struct tracemalloc_traceback { + Py_uhash_t hash; + /* Number of frames stored */ + uint16_t nframe; + /* Total number of frames the traceback had */ + uint16_t total_nframe; + struct tracemalloc_frame frames[1]; +}; + + +struct _tracemalloc_runtime_state { + struct _PyTraceMalloc_Config config; + + /* Protected by the GIL */ + struct { + PyMemAllocatorEx mem; + PyMemAllocatorEx raw; + PyMemAllocatorEx obj; + } allocators; + +#if defined(TRACE_RAW_MALLOC) + PyThread_type_lock tables_lock; +#endif + /* Size in bytes of currently traced memory. + Protected by TABLES_LOCK(). */ + size_t traced_memory; + /* Peak size in bytes of traced memory. + Protected by TABLES_LOCK(). */ + size_t peak_traced_memory; + /* Hash table used as a set to intern filenames: + PyObject* => PyObject*. + Protected by the GIL */ + _Py_hashtable_t *filenames; + /* Buffer to store a new traceback in traceback_new(). + Protected by the GIL. */ + struct tracemalloc_traceback *traceback; + /* Hash table used as a set to intern tracebacks: + traceback_t* => traceback_t* + Protected by the GIL */ + _Py_hashtable_t *tracebacks; + /* pointer (void*) => trace (trace_t*). + Protected by TABLES_LOCK(). */ + _Py_hashtable_t *traces; + /* domain (unsigned int) => traces (_Py_hashtable_t). + Protected by TABLES_LOCK(). */ + _Py_hashtable_t *domains; + + struct tracemalloc_traceback empty_traceback; + + Py_tss_t reentrant_key; +}; + +#define _tracemalloc_runtime_state_INIT \ + { \ + .config = { \ + .initialized = TRACEMALLOC_NOT_INITIALIZED, \ + .tracing = 0, \ + .max_nframe = 1, \ + }, \ + .reentrant_key = Py_tss_NEEDS_INIT, \ + } + + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_TRACEMALLOC_H diff --git a/Makefile.pre.in b/Makefile.pre.in index 815df69ad48c..9577f9dee6da 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1678,6 +1678,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ + $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tuple.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 0d70f0cf34c8..ac16626f2101 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -20,9 +20,6 @@ module _tracemalloc _Py_DECLARE_STR(anon_unknown, "<unknown>"); -/* Trace memory blocks allocated by PyMem_RawMalloc() */ -#define TRACE_RAW_MALLOC - /* Forward declaration */ static void tracemalloc_stop(void); static void* raw_malloc(size_t size); @@ -35,19 +32,14 @@ static void raw_free(void *ptr); #define TO_PTR(key) ((const void *)(uintptr_t)(key)) #define FROM_PTR(key) ((uintptr_t)(key)) -/* Protected by the GIL */ -static struct { - PyMemAllocatorEx mem; - PyMemAllocatorEx raw; - PyMemAllocatorEx obj; -} allocators; +#define allocators _PyRuntime.tracemalloc.allocators #if defined(TRACE_RAW_MALLOC) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ -static PyThread_type_lock tables_lock; +# define tables_lock _PyRuntime.tracemalloc.tables_lock # define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) # define TABLES_UNLOCK() PyThread_release_lock(tables_lock) #else @@ -59,33 +51,8 @@ static PyThread_type_lock tables_lock; #define DEFAULT_DOMAIN 0 -/* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. */ -typedef struct -#ifdef __GNUC__ -__attribute__((packed)) -#elif defined(_MSC_VER) -#pragma pack(push, 4) -#endif -{ - /* filename cannot be NULL: "<unknown>" is used if the Python frame - filename is NULL */ - PyObject *filename; - unsigned int lineno; -} frame_t; -#ifdef _MSC_VER -#pragma pack(pop) -#endif - - -typedef struct { - Py_uhash_t hash; - /* Number of frames stored */ - uint16_t nframe; - /* Total number of frames the traceback had */ - uint16_t total_nframe; - frame_t frames[1]; -} traceback_t; +typedef struct tracemalloc_frame frame_t; +typedef struct tracemalloc_traceback traceback_t; #define TRACEBACK_SIZE(NFRAME) \ (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1)) @@ -96,7 +63,8 @@ typedef struct { static const unsigned long MAX_NFRAME = Py_MIN(UINT16_MAX, ((SIZE_MAX - sizeof(traceback_t)) / sizeof(frame_t) + 1)); -static traceback_t tracemalloc_empty_traceback; +#define tracemalloc_empty_traceback _PyRuntime.tracemalloc.empty_traceback + /* Trace of a memory block */ typedef struct { @@ -108,35 +76,13 @@ typedef struct { } trace_t; -/* Size in bytes of currently traced memory. - Protected by TABLES_LOCK(). */ -static size_t tracemalloc_traced_memory = 0; - -/* Peak size in bytes of traced memory. - Protected by TABLES_LOCK(). */ -static size_t tracemalloc_peak_traced_memory = 0; - -/* Hash table used as a set to intern filenames: - PyObject* => PyObject*. - Protected by the GIL */ -static _Py_hashtable_t *tracemalloc_filenames = NULL; - -/* Buffer to store a new traceback in traceback_new(). - Protected by the GIL. */ -static traceback_t *tracemalloc_traceback = NULL; - -/* Hash table used as a set to intern tracebacks: - traceback_t* => traceback_t* - Protected by the GIL */ -static _Py_hashtable_t *tracemalloc_tracebacks = NULL; - -/* pointer (void*) => trace (trace_t*). - Protected by TABLES_LOCK(). */ -static _Py_hashtable_t *tracemalloc_traces = NULL; - -/* domain (unsigned int) => traces (_Py_hashtable_t). - Protected by TABLES_LOCK(). */ -static _Py_hashtable_t *tracemalloc_domains = NULL; +#define tracemalloc_traced_memory _PyRuntime.tracemalloc.traced_memory +#define tracemalloc_peak_traced_memory _PyRuntime.tracemalloc.peak_traced_memory +#define tracemalloc_filenames _PyRuntime.tracemalloc.filenames +#define tracemalloc_traceback _PyRuntime.tracemalloc.traceback +#define tracemalloc_tracebacks _PyRuntime.tracemalloc.tracebacks +#define tracemalloc_traces _PyRuntime.tracemalloc.traces +#define tracemalloc_domains _PyRuntime.tracemalloc.domains #ifdef TRACE_DEBUG @@ -157,7 +103,7 @@ tracemalloc_error(const char *format, ...) #if defined(TRACE_RAW_MALLOC) #define REENTRANT_THREADLOCAL -static Py_tss_t tracemalloc_reentrant_key = Py_tss_NEEDS_INIT; +#define tracemalloc_reentrant_key _PyRuntime.tracemalloc.reentrant_key /* Any non-NULL pointer can be used */ #define REENTRANT Py_True diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 35fbff320f46..25572d6ebcd5 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -259,6 +259,7 @@ <ClInclude Include="..\Include\internal\pycore_time.h" /> <ClInclude Include="..\Include\internal\pycore_token.h" /> <ClInclude Include="..\Include\internal\pycore_traceback.h" /> + <ClInclude Include="..\Include\internal\pycore_tracemalloc.h" /> <ClInclude Include="..\Include\internal\pycore_tuple.h" /> <ClInclude Include="..\Include\internal\pycore_typeobject.h" /> <ClInclude Include="..\Include\internal\pycore_ucnhash.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 19cb5cf1c807..d45b50c5d62d 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -678,6 +678,9 @@ <ClInclude Include="..\Include\internal\pycore_traceback.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_tracemalloc.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_tuple.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 94e9831db1fd..5dcd396c5487 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -377,17 +377,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## state -Modules/_tracemalloc.c - allocators - -Modules/_tracemalloc.c - tables_lock - -Modules/_tracemalloc.c - tracemalloc_empty_traceback - -Modules/_tracemalloc.c - tracemalloc_traced_memory - -Modules/_tracemalloc.c - tracemalloc_peak_traced_memory - -Modules/_tracemalloc.c - tracemalloc_filenames - -Modules/_tracemalloc.c - tracemalloc_traceback - -Modules/_tracemalloc.c - tracemalloc_tracebacks - -Modules/_tracemalloc.c - tracemalloc_traces - -Modules/_tracemalloc.c - tracemalloc_domains - -Modules/_tracemalloc.c - tracemalloc_reentrant_key - Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/signalmodule.c - is_tripped - Modules/signalmodule.c - signal_global_state - From webhook-mailer at python.org Mon Dec 12 11:59:26 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 16:59:26 -0000 Subject: [Python-checkins] gh-81057: Move faulthandler Globals to _PyRuntimeState (gh-100152) Message-ID: <mailman.2914.1670864368.3313.python-checkins@python.org> https://github.com/python/cpython/commit/53d9cd95cd91f1a291a3923acb95e0e86942291a commit: 53d9cd95cd91f1a291a3923acb95e0e86942291a branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T09:58:46-07:00 summary: gh-81057: Move faulthandler Globals to _PyRuntimeState (gh-100152) https://github.com/python/cpython/issues/81057 files: A Include/internal/pycore_faulthandler.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Makefile.pre.in M Modules/faulthandler.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h new file mode 100644 index 000000000000..e6aec7745a64 --- /dev/null +++ b/Include/internal/pycore_faulthandler.h @@ -0,0 +1,99 @@ +#ifndef Py_INTERNAL_FAULTHANDLER_H +#define Py_INTERNAL_FAULTHANDLER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef HAVE_SIGACTION +# include <signal.h> +#endif + + +#ifndef MS_WINDOWS + /* register() is useless on Windows, because only SIGSEGV, SIGABRT and + SIGILL can be handled by the process, and these signals can only be used + with enable(), not using register() */ +# define FAULTHANDLER_USER +#endif + + +#ifdef HAVE_SIGACTION +/* Using an alternative stack requires sigaltstack() + and sigaction() SA_ONSTACK */ +# ifdef HAVE_SIGALTSTACK +# define FAULTHANDLER_USE_ALT_STACK +# endif +typedef struct sigaction _Py_sighandler_t; +#else +typedef PyOS_sighandler_t _Py_sighandler_t; +#endif // HAVE_SIGACTION + + +#ifdef FAULTHANDLER_USER +struct faulthandler_user_signal { + int enabled; + PyObject *file; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; + PyInterpreterState *interp; +}; +#endif /* FAULTHANDLER_USER */ + + +struct _faulthandler_runtime_state { + struct { + int enabled; + PyObject *file; + int fd; + int all_threads; + PyInterpreterState *interp; +#ifdef MS_WINDOWS + void *exc_handler; +#endif + } fatal_error; + + struct { + PyObject *file; + int fd; + PY_TIMEOUT_T timeout_us; /* timeout in microseconds */ + int repeat; + PyInterpreterState *interp; + int exit; + char *header; + size_t header_len; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + PyThread_type_lock cancel_event; + /* released by child thread when joined */ + PyThread_type_lock running; + } thread; + +#ifdef FAULTHANDLER_USER + struct faulthandler_user_signal *user_signals; +#endif + +#ifdef FAULTHANDLER_USE_ALT_STACK + stack_t stack; + stack_t old_stack; +#endif +}; + +#define _faulthandler_runtime_state_INIT \ + { \ + .fatal_error = { \ + .fd = -1, \ + }, \ + } + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FAULTHANDLER_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 99ec6fc8862b..b9ed8f593156 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -13,6 +13,7 @@ extern "C" { #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_dtoa.h" // struct _dtoa_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state +#include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_function.h" // struct _func_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state @@ -140,6 +141,7 @@ typedef struct pyruntimestate { struct _getargs_runtime_state getargs; struct _dtoa_runtime_state dtoa; struct _fileutils_state fileutils; + struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; PyPreConfig preconfig; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 029357dedf3e..9677a727c446 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -54,6 +54,7 @@ extern "C" { .fileutils = { \ .force_ascii = -1, \ }, \ + .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ .float_state = { \ .float_format = _py_float_format_unknown, \ diff --git a/Makefile.pre.in b/Makefile.pre.in index 9577f9dee6da..dd6c3fbd1c64 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1634,6 +1634,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dtoa.h \ $(srcdir)/Include/internal/pycore_exceptions.h \ + $(srcdir)/Include/internal/pycore_faulthandler.h \ $(srcdir)/Include/internal/pycore_fileutils.h \ $(srcdir)/Include/internal/pycore_floatobject.h \ $(srcdir)/Include/internal/pycore_format.h \ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 341a03a244cd..5309a3728c5e 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -18,12 +18,6 @@ # include <sys/resource.h> #endif -/* Using an alternative stack requires sigaltstack() - and sigaction() SA_ONSTACK */ -#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) -# define FAULTHANDLER_USE_ALT_STACK -#endif - #if defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_LINUX_AUXVEC_H) && defined(HAVE_SYS_AUXV_H) # include <linux/auxvec.h> // AT_MINSIGSTKSZ # include <sys/auxv.h> // getauxval() @@ -32,13 +26,6 @@ /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) -#ifndef MS_WINDOWS - /* register() is useless on Windows, because only SIGSEGV, SIGABRT and - SIGILL can be handled by the process, and these signals can only be used - with enable(), not using register() */ -# define FAULTHANDLER_USER -#endif - #define PUTS(fd, str) _Py_write_noraise(fd, str, strlen(str)) @@ -58,12 +45,6 @@ #endif -#ifdef HAVE_SIGACTION -typedef struct sigaction _Py_sighandler_t; -#else -typedef PyOS_sighandler_t _Py_sighandler_t; -#endif - typedef struct { int signum; int enabled; @@ -72,47 +53,12 @@ typedef struct { int all_threads; } fault_handler_t; -static struct { - int enabled; - PyObject *file; - int fd; - int all_threads; - PyInterpreterState *interp; -#ifdef MS_WINDOWS - void *exc_handler; -#endif -} fatal_error = {0, NULL, -1, 0}; - -static struct { - PyObject *file; - int fd; - PY_TIMEOUT_T timeout_us; /* timeout in microseconds */ - int repeat; - PyInterpreterState *interp; - int exit; - char *header; - size_t header_len; - /* The main thread always holds this lock. It is only released when - faulthandler_thread() is interrupted before this thread exits, or at - Python exit. */ - PyThread_type_lock cancel_event; - /* released by child thread when joined */ - PyThread_type_lock running; -} thread; +#define fatal_error _PyRuntime.faulthandler.fatal_error +#define thread _PyRuntime.faulthandler.thread #ifdef FAULTHANDLER_USER -typedef struct { - int enabled; - PyObject *file; - int fd; - int all_threads; - int chain; - _Py_sighandler_t previous; - PyInterpreterState *interp; -} user_signal_t; - -static user_signal_t *user_signals; - +#define user_signals _PyRuntime.faulthandler.user_signals +typedef struct faulthandler_user_signal user_signal_t; static void faulthandler_user(int signum); #endif /* FAULTHANDLER_USER */ @@ -134,8 +80,8 @@ static const size_t faulthandler_nsignals = \ Py_ARRAY_LENGTH(faulthandler_handlers); #ifdef FAULTHANDLER_USE_ALT_STACK -static stack_t stack; -static stack_t old_stack; +# define stack _PyRuntime.faulthandler.stack +# define old_stack _PyRuntime.faulthandler.old_stack #endif @@ -1094,7 +1040,7 @@ faulthandler_fatal_error_thread(void *plock) static PyObject * faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) { - long thread; + long tid; PyThread_type_lock lock; faulthandler_suppress_crash_report(); @@ -1105,8 +1051,8 @@ faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) PyThread_acquire_lock(lock, WAIT_LOCK); - thread = PyThread_start_new_thread(faulthandler_fatal_error_thread, lock); - if (thread == -1) { + tid = PyThread_start_new_thread(faulthandler_fatal_error_thread, lock); + if (tid == -1) { PyThread_free_lock(lock); PyErr_SetString(PyExc_RuntimeError, "unable to start the thread"); return NULL; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 25572d6ebcd5..bb2aaae3317b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -214,6 +214,7 @@ <ClInclude Include="..\Include\internal\pycore_dict_state.h" /> <ClInclude Include="..\Include\internal\pycore_dtoa.h" /> <ClInclude Include="..\Include\internal\pycore_exceptions.h" /> + <ClInclude Include="..\Include\internal\pycore_faulthandler.h" /> <ClInclude Include="..\Include\internal\pycore_fileutils.h" /> <ClInclude Include="..\Include\internal\pycore_floatobject.h" /> <ClInclude Include="..\Include\internal\pycore_format.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d45b50c5d62d..339e7cc4937a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -549,6 +549,9 @@ <ClInclude Include="..\Include\internal\pycore_exceptions.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_faulthandler.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_fileutils.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -558,6 +561,9 @@ <ClInclude Include="..\Include\internal\pycore_format.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_function.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_gc.h"> <Filter>Include\internal</Filter> </ClInclude> diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 5dcd396c5487..eb57f95abd17 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -361,15 +361,6 @@ Modules/itertoolsmodule.c - tee_type - Modules/itertoolsmodule.c - teedataobject_type - Modules/itertoolsmodule.c - ziplongest_type - -##----------------------- -## state - -Modules/faulthandler.c - fatal_error - -Modules/faulthandler.c - thread - -Modules/faulthandler.c - user_signals - -Modules/faulthandler.c - stack - -Modules/faulthandler.c - old_stack - - ################################## ## global non-objects to fix in builtin modules @@ -377,7 +368,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## state -Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/signalmodule.c - is_tripped - Modules/signalmodule.c - signal_global_state - Modules/signalmodule.c - wakeup - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7d6ff0ba6d64..b03198627f42 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -151,6 +151,7 @@ Python/sysmodule.c - _preinit_xoptions - # thread-safety # XXX need race protection? +Modules/faulthandler.c faulthandler_dump_traceback reentrant - Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - From webhook-mailer at python.org Mon Dec 12 18:50:36 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 23:50:36 -0000 Subject: [Python-checkins] gh-81057: Move Signal-Related Globals to _PyRuntimeState (gh-100085) Message-ID: <mailman.2915.1670889037.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5eb28bca9fc963607189e3b3e1403f341dbf640a commit: 5eb28bca9fc963607189e3b3e1403f341dbf640a branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T16:50:19-07:00 summary: gh-81057: Move Signal-Related Globals to _PyRuntimeState (gh-100085) https://github.com/python/cpython/issues/81057 files: M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Include/internal/pycore_signal.h M Modules/signalmodule.c M Python/pylifecycle.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index b9ed8f593156..92ed45956c99 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -23,6 +23,7 @@ extern "C" { #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_signal.h" // struct _signals_runtime_state #include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids @@ -93,13 +94,9 @@ typedef struct pyruntimestate { struct _pymem_allocators allocators; struct _obmalloc_state obmalloc; struct pyhash_runtime_state pyhash_state; - struct { - /* True if the main interpreter thread exited due to an unhandled - * KeyboardInterrupt exception, suggesting the user pressed ^C. */ - int unhandled_keyboard_interrupt; - } signals; struct _time_runtime_state time; struct _pythread_runtime_state threads; + struct _signals_runtime_state signals; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 9677a727c446..1431096e2d24 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -26,6 +26,7 @@ extern "C" { }, \ .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ .pyhash_state = pyhash_state_INIT, \ + .signals = _signals_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ diff --git a/Include/internal/pycore_signal.h b/Include/internal/pycore_signal.h index b921dd170e9f..ca3f69d09fc0 100644 --- a/Include/internal/pycore_signal.h +++ b/Include/internal/pycore_signal.h @@ -10,8 +10,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_atomic.h" // _Py_atomic_address + #include <signal.h> // NSIG + #ifdef _SIG_MAXSIG // gh-91145: On FreeBSD, <signal.h> defines NSIG as 32: it doesn't include // realtime signals: [SIGRTMIN,SIGRTMAX]. Use _SIG_MAXSIG instead. For @@ -29,6 +32,66 @@ extern "C" { # define Py_NSIG 64 // Use a reasonable default value #endif +#define INVALID_FD (-1) + +struct _signals_runtime_state { + volatile struct { + _Py_atomic_int tripped; + /* func is atomic to ensure that PyErr_SetInterrupt is async-signal-safe + * (even though it would probably be otherwise, anyway). + */ + _Py_atomic_address func; + } handlers[Py_NSIG]; + + volatile struct { +#ifdef MS_WINDOWS + /* This would be "SOCKET fd" if <winsock2.h> were always included. + It isn't so we must cast to SOCKET where appropriate. */ + volatile int fd; +#elif defined(__VXWORKS__) + int fd; +#else + sig_atomic_t fd; +#endif + + int warn_on_full_buffer; +#ifdef MS_WINDOWS + int use_send; +#endif + } wakeup; + + /* Speed up sigcheck() when none tripped */ + _Py_atomic_int is_tripped; + + /* These objects necessarily belong to the main interpreter. */ + PyObject *default_handler; + PyObject *ignore_handler; + +#ifdef MS_WINDOWS + /* This would be "HANDLE sigint_event" if <windows.h> were always included. + It isn't so we must cast to HANDLE everywhere "sigint_event" is used. */ + void *sigint_event; +#endif + + /* True if the main interpreter thread exited due to an unhandled + * KeyboardInterrupt exception, suggesting the user pressed ^C. */ + int unhandled_keyboard_interrupt; +}; + +#ifdef MS_WINDOWS +# define _signals_WAKEUP_INIT \ + {.fd = INVALID_FD, .warn_on_full_buffer = 1, .use_send = 0} +#else +# define _signals_WAKEUP_INIT \ + {.fd = INVALID_FD, .warn_on_full_buffer = 1} +#endif + +#define _signals_RUNTIME_INIT \ + { \ + .wakeup = _signals_WAKEUP_INIT, \ + } + + #ifdef __cplusplus } #endif diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index c539787e5829..538a7e85bc95 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,7 +13,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_signal.h" // Py_NSIG +#include "pycore_signal.h" #ifndef MS_WINDOWS # include "posixmodule.h" @@ -23,12 +23,13 @@ #endif #ifdef MS_WINDOWS -# include <windows.h> # ifdef HAVE_PROCESS_H # include <process.h> # endif #endif +#include "pycore_signal.h" // Py_NSIG + #ifdef HAVE_SIGNAL_H # include <signal.h> #endif @@ -100,47 +101,13 @@ class sigset_t_converter(CConverter): may not be the thread that received the signal. */ -static volatile struct { - _Py_atomic_int tripped; - /* func is atomic to ensure that PyErr_SetInterrupt is async-signal-safe - * (even though it would probably be otherwise, anyway). - */ - _Py_atomic_address func; -} Handlers[Py_NSIG]; - -#ifdef MS_WINDOWS -#define INVALID_FD ((SOCKET_T)-1) - -static volatile struct { - SOCKET_T fd; - int warn_on_full_buffer; - int use_send; -} wakeup = {.fd = INVALID_FD, .warn_on_full_buffer = 1, .use_send = 0}; -#else -#define INVALID_FD (-1) -static volatile struct { -#ifdef __VXWORKS__ - int fd; -#else - sig_atomic_t fd; -#endif - int warn_on_full_buffer; -} wakeup = {.fd = INVALID_FD, .warn_on_full_buffer = 1}; -#endif - -/* Speed up sigcheck() when none tripped */ -static _Py_atomic_int is_tripped; - -typedef struct { - PyObject *default_handler; - PyObject *ignore_handler; -#ifdef MS_WINDOWS - HANDLE sigint_event; -#endif -} signal_state_t; +#define Handlers _PyRuntime.signals.handlers +#define wakeup _PyRuntime.signals.wakeup +#define is_tripped _PyRuntime.signals.is_tripped // State shared by all Python interpreters -static signal_state_t signal_global_state = {0}; +typedef struct _signals_runtime_state signal_state_t; +#define signal_global_state _PyRuntime.signals #if defined(HAVE_GETITIMER) || defined(HAVE_SETITIMER) # define PYHAVE_ITIMER_ERROR @@ -331,13 +298,7 @@ trip_signal(int sig_num) See bpo-30038 for more details. */ - int fd; -#ifdef MS_WINDOWS - fd = Py_SAFE_DOWNCAST(wakeup.fd, SOCKET_T, int); -#else - fd = wakeup.fd; -#endif - + int fd = wakeup.fd; if (fd != INVALID_FD) { unsigned char byte = (unsigned char)sig_num; #ifdef MS_WINDOWS @@ -407,7 +368,7 @@ signal_handler(int sig_num) #ifdef MS_WINDOWS if (sig_num == SIGINT) { signal_state_t *state = &signal_global_state; - SetEvent(state->sigint_event); + SetEvent((HANDLE)state->sigint_event); } #endif } @@ -822,7 +783,7 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) } old_sockfd = wakeup.fd; - wakeup.fd = sockfd; + wakeup.fd = Py_SAFE_DOWNCAST(sockfd, SOCKET_T, int); wakeup.warn_on_full_buffer = warn_on_full_buffer; wakeup.use_send = is_socket; @@ -873,11 +834,7 @@ PySignal_SetWakeupFd(int fd) fd = -1; } -#ifdef MS_WINDOWS - int old_fd = Py_SAFE_DOWNCAST(wakeup.fd, SOCKET_T, int); -#else int old_fd = wakeup.fd; -#endif wakeup.fd = fd; wakeup.warn_on_full_buffer = 1; return old_fd; @@ -1654,6 +1611,8 @@ signal_module_exec(PyObject *m) signal_state_t *state = &signal_global_state; _signal_module_state *modstate = get_signal_state(m); + // XXX For proper isolation, these values must be guaranteed + // to be effectively const (e.g. immortal). modstate->default_handler = state->default_handler; // borrowed ref modstate->ignore_handler = state->ignore_handler; // borrowed ref @@ -1783,7 +1742,7 @@ _PySignal_Fini(void) #ifdef MS_WINDOWS if (state->sigint_event != NULL) { - CloseHandle(state->sigint_event); + CloseHandle((HANDLE)state->sigint_event); state->sigint_event = NULL; } #endif @@ -2009,7 +1968,7 @@ _PySignal_Init(int install_signal_handlers) #ifdef MS_WINDOWS /* Create manual-reset event, initially unset */ - state->sigint_event = CreateEvent(NULL, TRUE, FALSE, FALSE); + state->sigint_event = (void *)CreateEvent(NULL, TRUE, FALSE, FALSE); if (state->sigint_event == NULL) { PyErr_SetFromWindowsErr(0); return -1; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e13660f51138..1cb0e4d747e1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -54,7 +54,6 @@ extern void _PyIO_Fini(void); #ifdef MS_WINDOWS # undef BYTE -# include "windows.h" extern PyTypeObject PyWindowsConsoleIO_Type; # define PyWindowsConsoleIO_Check(op) \ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index eb57f95abd17..fcec95bbb2f7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -365,13 +365,7 @@ Modules/itertoolsmodule.c - ziplongest_type - ################################## ## global non-objects to fix in builtin modules -##----------------------- -## state - -Modules/signalmodule.c - is_tripped - -Modules/signalmodule.c - signal_global_state - -Modules/signalmodule.c - wakeup - -Modules/signalmodule.c - Handlers - +# <none> ################################## From webhook-mailer at python.org Mon Dec 12 18:51:33 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 12 Dec 2022 23:51:33 -0000 Subject: [Python-checkins] gh-81057: Move _Py_RefTotal to the "Ignored Globals" List (gh-100203) Message-ID: <mailman.2916.1670889095.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0e081a089ec969c9a34f5ff25886205616ef4dd3 commit: 0e081a089ec969c9a34f5ff25886205616ef4dd3 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-12T16:51:27-07:00 summary: gh-81057: Move _Py_RefTotal to the "Ignored Globals" List (gh-100203) We can't move it to _PyRuntimeState because the symbol is exposed in the stable ABI. We'll have to sort that out before a per-interpreter GIL, but it shouldn't be too hard. https://github.com/python/cpython/issues/81057 files: M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index fcec95bbb2f7..479221cbd4b6 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -302,10 +302,7 @@ Objects/sliceobject.c - _Py_EllipsisObject - ################################## ## global non-objects to fix in core code -##----------------------- -## state - -Objects/object.c - _Py_RefTotal - +# <none> ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index b03198627f42..c71fc0d95821 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -142,6 +142,12 @@ Python/pylifecycle.c - runtime_initialized - Modules/syslogmodule.c - S_ident_o - Modules/syslogmodule.c - S_log_open - +##----------------------- +## kept for stable ABI compatibility + +# XXX should be per-interpreter, without impacting stable ABI extensions +Objects/object.c - _Py_RefTotal - + ##----------------------- ## one-off temporary state From webhook-mailer at python.org Tue Dec 13 05:54:27 2022 From: webhook-mailer at python.org (hugovk) Date: Tue, 13 Dec 2022 10:54:27 -0000 Subject: [Python-checkins] GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (#100207) Message-ID: <mailman.2917.1670928868.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d3ea82aaf940167482df1e08d6482de8f2dd8526 commit: d3ea82aaf940167482df1e08d6482de8f2dd8526 branch: main author: Filipe La?ns <lains at riseup.net> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-13T12:54:07+02:00 summary: GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (#100207) files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 024988777030..839c2c015b49 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -121,7 +121,7 @@ identifier. Python currently uses eight paths: Return the default scheme name for the current platform. - .. versionchanged:: 3.10 + .. versionadded:: 3.10 This function was previously named ``_get_default_scheme()`` and considered an implementation detail. From webhook-mailer at python.org Tue Dec 13 06:02:52 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 13 Dec 2022 11:02:52 -0000 Subject: [Python-checkins] GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) Message-ID: <mailman.2918.1670929373.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cd4166fec2e72438a1057820a98d078644c3a173 commit: cd4166fec2e72438a1057820a98d078644c3a173 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-13T03:02:41-08:00 summary: GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) (cherry picked from commit d3ea82aaf940167482df1e08d6482de8f2dd8526) Co-authored-by: Filipe La?ns <lains at riseup.net> files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index fa18d62d22af..0049b663973f 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -121,7 +121,7 @@ identifier. Python currently uses eight paths: Return the default scheme name for the current platform. - .. versionchanged:: 3.10 + .. versionadded:: 3.10 This function was previously named ``_get_default_scheme()`` and considered an implementation detail. From webhook-mailer at python.org Tue Dec 13 06:03:28 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 13 Dec 2022 11:03:28 -0000 Subject: [Python-checkins] GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) Message-ID: <mailman.2919.1670929409.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f5f68ff1924b5588a7f4b2dcc6b17ccf419859f6 commit: f5f68ff1924b5588a7f4b2dcc6b17ccf419859f6 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-13T03:03:22-08:00 summary: GH-100206: use versionadded for the addition of sysconfig.get_default_scheme (GH-100207) (cherry picked from commit d3ea82aaf940167482df1e08d6482de8f2dd8526) Co-authored-by: Filipe La?ns <lains at riseup.net> files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 7ef3b2489673..f42a5345357c 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -113,7 +113,7 @@ identifier. Python currently uses eight paths: Return the default scheme name for the current platform. - .. versionchanged:: 3.10 + .. versionadded:: 3.10 This function was previously named ``_get_default_scheme()`` and considered an implementation detail. From webhook-mailer at python.org Tue Dec 13 08:55:16 2022 From: webhook-mailer at python.org (iritkatriel) Date: Tue, 13 Dec 2022 13:55:16 -0000 Subject: [Python-checkins] gh-99955: undef ERROR and SUCCESS before redefining (fixes sanitizer warning) (#100215) Message-ID: <mailman.2920.1670939717.3313.python-checkins@python.org> https://github.com/python/cpython/commit/985a71032bf055240e61259285a0b4925c925383 commit: 985a71032bf055240e61259285a0b4925c925383 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2022-12-13T13:55:10Z summary: gh-99955: undef ERROR and SUCCESS before redefining (fixes sanitizer warning) (#100215) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 17b164a4d06e..813e0d5503b4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -55,6 +55,8 @@ */ #define STACK_USE_GUIDELINE 30 +#undef SUCCESS +#undef ERROR #define SUCCESS 0 #define ERROR -1 From webhook-mailer at python.org Wed Dec 14 06:12:59 2022 From: webhook-mailer at python.org (markshannon) Date: Wed, 14 Dec 2022 11:12:59 -0000 Subject: [Python-checkins] GH-100222: Redefine _Py_CODEUNIT as a union to clarify structure of code unit. (GH-100223) Message-ID: <mailman.2921.1671016380.3313.python-checkins@python.org> https://github.com/python/cpython/commit/6997e77bdf2297375962aaf82876da4e7ecdd61a commit: 6997e77bdf2297375962aaf82876da4e7ecdd61a branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-14T11:12:53Z summary: GH-100222: Redefine _Py_CODEUNIT as a union to clarify structure of code unit. (GH-100223) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst M Include/cpython/code.h M Include/internal/pycore_code.h M Objects/codeobject.c M Python/ceval.c M Python/compile.c M Python/generated_cases.c.h M Python/specialize.c M Tools/cases_generator/generate_cases.py diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fc7c5ed70243..6a13ff2dfd9f 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -16,21 +16,24 @@ extern "C" { * 2**32 - 1, rather than INT_MAX. */ -typedef uint16_t _Py_CODEUNIT; - -#ifdef WORDS_BIGENDIAN -# define _Py_OPCODE(word) ((word) >> 8) -# define _Py_OPARG(word) ((word) & 255) -# define _Py_MAKECODEUNIT(opcode, oparg) (((opcode)<<8)|(oparg)) -#else -# define _Py_OPCODE(word) ((word) & 255) -# define _Py_OPARG(word) ((word) >> 8) -# define _Py_MAKECODEUNIT(opcode, oparg) ((opcode)|((oparg)<<8)) -#endif +typedef union { + uint16_t cache; + struct { + uint8_t opcode; + uint8_t oparg; + }; +} _Py_CODEUNIT; + +#define _Py_OPCODE(word) ((word).opcode) +#define _Py_OPARG(word) ((word).oparg) + +static inline void +_py_set_opocde(_Py_CODEUNIT *word, uint8_t opcode) +{ + word->opcode = opcode; +} -// Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing: -#define _Py_SET_OPCODE(word, opcode) \ - do { ((unsigned char *)&(word))[0] = (opcode); } while (0) +#define _Py_SET_OPCODE(word, opcode) _py_set_opocde(&(word), opcode) typedef struct { PyObject *_co_code; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index f22fd45f8319..9e59fc98bf3d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -18,53 +18,53 @@ extern "C" { #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT index; - _Py_CODEUNIT module_keys_version[2]; - _Py_CODEUNIT builtin_keys_version; + uint16_t counter; + uint16_t index; + uint16_t module_keys_version[2]; + uint16_t builtin_keys_version; } _PyLoadGlobalCache; #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyUnpackSequenceCache; #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ CACHE_ENTRIES(_PyUnpackSequenceCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT mask; + uint16_t counter; + uint16_t mask; } _PyCompareOpCache; #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT type_version[2]; - _Py_CODEUNIT func_version; + uint16_t counter; + uint16_t type_version[2]; + uint16_t func_version; } _PyBinarySubscrCache; #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT version[2]; - _Py_CODEUNIT index; + uint16_t counter; + uint16_t version[2]; + uint16_t index; } _PyAttrCache; typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT type_version[2]; - _Py_CODEUNIT keys_version[2]; - _Py_CODEUNIT descr[4]; + uint16_t counter; + uint16_t type_version[2]; + uint16_t keys_version[2]; + uint16_t descr[4]; } _PyLoadMethodCache; @@ -74,21 +74,21 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) typedef struct { - _Py_CODEUNIT counter; - _Py_CODEUNIT func_version[2]; - _Py_CODEUNIT min_args; + uint16_t counter; + uint16_t func_version[2]; + uint16_t min_args; } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) typedef struct { - _Py_CODEUNIT counter; + uint16_t counter; } _PyForIterCache; #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) @@ -409,7 +409,7 @@ write_location_entry_start(uint8_t *ptr, int code, int length) static inline uint16_t adaptive_counter_bits(int value, int backoff) { return (value << ADAPTIVE_BACKOFF_BITS) | - (backoff & ((1<<ADAPTIVE_BACKOFF_BITS)-1)); + (backoff & ((1<<ADAPTIVE_BACKOFF_BITS)-1)); } static inline uint16_t diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst new file mode 100644 index 000000000000..032b49420d8b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-13-16-05-18.gh-issue-100222.OVVvYe.rst @@ -0,0 +1,2 @@ +Redefine the ``_Py_CODEUNIT`` typedef as a union to describe its layout to +the C compiler, avoiding type punning and improving clarity. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index c92c7deaf808..f455cc603aae 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1520,9 +1520,10 @@ deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len) _Py_CODEUNIT instruction = instructions[i]; int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; int caches = _PyOpcode_Caches[opcode]; - instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); + instructions[i].opcode = opcode; while (caches--) { - instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); + instructions[++i].opcode = CACHE; + instructions[i].oparg = 0; } } } @@ -1775,9 +1776,9 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - _Py_SET_OPCODE(co_instr, _PyOpcode_Deopt[_Py_OPCODE(co_instr)]); - _Py_SET_OPCODE(cp_instr, _PyOpcode_Deopt[_Py_OPCODE(cp_instr)]); - eq = co_instr == cp_instr; + co_instr.opcode = _PyOpcode_Deopt[_Py_OPCODE(co_instr)]; + cp_instr.opcode =_PyOpcode_Deopt[_Py_OPCODE(cp_instr)]; + eq = co_instr.cache == cp_instr.cache; if (!eq) { goto unequal; } diff --git a/Python/ceval.c b/Python/ceval.c index 9e4179e56071..45f42800d7ce 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -864,7 +864,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { STAT_INC(opcode, miss); \ STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ - if (ADAPTIVE_COUNTER_IS_ZERO(*next_instr)) { \ + if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ else { \ @@ -1289,7 +1289,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } opcode = _PyOpcode_Deopt[opcode]; if (_PyOpcode_Caches[opcode]) { - _Py_CODEUNIT *counter = &next_instr[1]; + uint16_t *counter = &next_instr[1].cache; // The instruction is going to decrement the counter, so we need to // increment it here to make sure it doesn't try to specialize: if (!ADAPTIVE_COUNTER_IS_MAX(*counter)) { diff --git a/Python/compile.c b/Python/compile.c index 813e0d5503b4..09eb4016940d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -263,22 +263,32 @@ write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: - *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF); + codestr->opcode = EXTENDED_ARG; + codestr->oparg = (oparg >> 24) & 0xFF; + codestr++; /* fall through */ case 3: - *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 16) & 0xFF); + codestr->opcode = EXTENDED_ARG; + codestr->oparg = (oparg >> 16) & 0xFF; + codestr++; /* fall through */ case 2: - *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 8) & 0xFF); + codestr->opcode = EXTENDED_ARG; + codestr->oparg = (oparg >> 8) & 0xFF; + codestr++; /* fall through */ case 1: - *codestr++ = _Py_MAKECODEUNIT(opcode, oparg & 0xFF); + codestr->opcode = opcode; + codestr->oparg = oparg & 0xFF; + codestr++; break; default: Py_UNREACHABLE(); } while (caches--) { - *codestr++ = _Py_MAKECODEUNIT(CACHE, 0); + codestr->opcode = CACHE; + codestr->oparg = 0; + codestr++; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 45382a466b1c..63635fbfc2f4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -562,8 +562,8 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = PEEK(1); PyObject *container = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t func_version = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t func_version = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -612,7 +612,7 @@ PyObject *sub = PEEK(1); PyObject *container = PEEK(2); PyObject *v = PEEK(3); - uint16_t counter = read_u16(next_instr + 0); + uint16_t counter = read_u16(&next_instr[0].cache); if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); next_instr--; @@ -1249,7 +1249,7 @@ PREDICTED(STORE_ATTR); PyObject *owner = PEEK(1); PyObject *v = PEEK(2); - uint16_t counter = read_u16(next_instr + 0); + uint16_t counter = read_u16(&next_instr[0].cache); if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg); @@ -2083,8 +2083,8 @@ TARGET(STORE_ATTR_INSTANCE_VALUE) { PyObject *owner = PEEK(1); PyObject *value = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t index = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2111,8 +2111,8 @@ TARGET(STORE_ATTR_WITH_HINT) { PyObject *owner = PEEK(1); PyObject *value = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t hint = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t hint = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2160,8 +2160,8 @@ TARGET(STORE_ATTR_SLOT) { PyObject *owner = PEEK(1); PyObject *value = PEEK(2); - uint32_t type_version = read_u32(next_instr + 1); - uint16_t index = read_u16(next_instr + 3); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2209,7 +2209,7 @@ PyObject *right = _tmp_1; PyObject *left = _tmp_2; size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); + uint16_t when_to_jump_mask = read_u16(&next_instr[1].cache); assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); @@ -2247,7 +2247,7 @@ PyObject *right = _tmp_1; PyObject *left = _tmp_2; size_t jump; - uint16_t when_to_jump_mask = read_u16(next_instr + 1); + uint16_t when_to_jump_mask = read_u16(&next_instr[1].cache); assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); @@ -2286,7 +2286,7 @@ PyObject *right = _tmp_1; PyObject *left = _tmp_2; size_t jump; - uint16_t invert = read_u16(next_instr + 1); + uint16_t invert = read_u16(&next_instr[1].cache); assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); diff --git a/Python/specialize.c b/Python/specialize.c index 785088eac8c5..678c5d66b660 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -267,26 +267,26 @@ _PyCode_Quicken(PyCodeObject *code) int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; int caches = _PyOpcode_Caches[opcode]; if (caches) { - instructions[i + 1] = adaptive_counter_warmup(); + instructions[i + 1].cache = adaptive_counter_warmup(); previous_opcode = 0; i += caches; continue; } switch (previous_opcode << 8 | opcode) { case LOAD_CONST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_CONST__LOAD_FAST); + instructions[i - 1].opcode = LOAD_CONST__LOAD_FAST; break; case LOAD_FAST << 8 | LOAD_CONST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_CONST); + instructions[i - 1].opcode = LOAD_FAST__LOAD_CONST; break; case LOAD_FAST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_FAST); + instructions[i - 1].opcode = LOAD_FAST__LOAD_FAST; break; case STORE_FAST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__LOAD_FAST); + instructions[i - 1].opcode = STORE_FAST__LOAD_FAST; break; case STORE_FAST << 8 | STORE_FAST: - _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__STORE_FAST); + instructions[i - 1].opcode = STORE_FAST__STORE_FAST; break; } previous_opcode = opcode; @@ -482,7 +482,7 @@ specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - _Py_SET_OPCODE(*instr, opcode_module); + _py_set_opocde(instr, opcode_module); return 0; } @@ -634,7 +634,7 @@ specialize_dict_access( } write_u32(cache->version, type->tp_version_tag); cache->index = (uint16_t)index; - _Py_SET_OPCODE(*instr, values_op); + _py_set_opocde(instr, values_op); } else { PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); @@ -651,7 +651,7 @@ specialize_dict_access( } cache->index = (uint16_t)index; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, hint_op); + _py_set_opocde(instr, hint_op); } return 1; } @@ -730,7 +730,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ write_obj(lm_cache->descr, fget); - _Py_SET_OPCODE(*instr, LOAD_ATTR_PROPERTY); + _py_set_opocde(instr, LOAD_ATTR_PROPERTY); goto success; } case OBJECT_SLOT: @@ -754,7 +754,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT); + _py_set_opocde(instr, LOAD_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -763,7 +763,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT); + _py_set_opocde(instr, LOAD_ATTR_SLOT); goto success; } case OTHER_SLOT: @@ -791,7 +791,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) /* borrowed */ write_obj(lm_cache->descr, descr); write_u32(lm_cache->type_version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + _py_set_opocde(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); goto success; } case BUILTIN_CLASSMETHOD: @@ -809,7 +809,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, LOAD_ATTR); + _py_set_opocde(instr, LOAD_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -868,7 +868,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _Py_SET_OPCODE(*instr, STORE_ATTR_SLOT); + _py_set_opocde(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -897,7 +897,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, STORE_ATTR); + _py_set_opocde(instr, STORE_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -961,7 +961,7 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, case NON_DESCRIPTOR: write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); write_obj(cache->descr, descr); - _Py_SET_OPCODE(*instr, LOAD_ATTR_CLASS); + _py_set_opocde(instr, LOAD_ATTR_CLASS); return 0; #ifdef Py_STATS case ABSENT: @@ -1043,21 +1043,21 @@ PyObject *descr, DescriptorClassification kind) } switch(dictkind) { case NO_DICT: - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_NO_DICT); + _py_set_opocde(instr, LOAD_ATTR_METHOD_NO_DICT); break; case MANAGED_VALUES: - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_VALUES); + _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); goto fail; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_WITH_DICT); + _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_DICT); break; case LAZY_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_LAZY_DICT); + _py_set_opocde(instr, LOAD_ATTR_METHOD_LAZY_DICT); break; } /* `descr` is borrowed. This is safe for methods (even inherited ones from @@ -1114,7 +1114,7 @@ _Py_Specialize_LoadGlobal( } cache->index = (uint16_t)index; write_u32(cache->module_keys_version, keys_version); - _Py_SET_OPCODE(*instr, LOAD_GLOBAL_MODULE); + _py_set_opocde(instr, LOAD_GLOBAL_MODULE); goto success; } if (!PyDict_CheckExact(builtins)) { @@ -1150,12 +1150,12 @@ _Py_Specialize_LoadGlobal( cache->index = (uint16_t)index; write_u32(cache->module_keys_version, globals_version); cache->builtin_keys_version = (uint16_t)builtins_version; - _Py_SET_OPCODE(*instr, LOAD_GLOBAL_BUILTIN); + _py_set_opocde(instr, LOAD_GLOBAL_BUILTIN); goto success; fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, LOAD_GLOBAL); + _py_set_opocde(instr, LOAD_GLOBAL); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1259,7 +1259,7 @@ _Py_Specialize_BinarySubscr( PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_LIST_INT); + _py_set_opocde(instr, BINARY_SUBSCR_LIST_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1268,7 +1268,7 @@ _Py_Specialize_BinarySubscr( } if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_TUPLE_INT); + _py_set_opocde(instr, BINARY_SUBSCR_TUPLE_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1276,7 +1276,7 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_DICT); + _py_set_opocde(instr, BINARY_SUBSCR_DICT); goto success; } PyTypeObject *cls = Py_TYPE(container); @@ -1307,7 +1307,7 @@ _Py_Specialize_BinarySubscr( } cache->func_version = version; ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor; - _Py_SET_OPCODE(*instr, BINARY_SUBSCR_GETITEM); + _py_set_opocde(instr, BINARY_SUBSCR_GETITEM); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1315,7 +1315,7 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, BINARY_SUBSCR); + _py_set_opocde(instr, BINARY_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1334,7 +1334,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) && ((PyLongObject *)sub)->ob_digit[0] < (size_t)PyList_GET_SIZE(container)) { - _Py_SET_OPCODE(*instr, STORE_SUBSCR_LIST_INT); + _py_set_opocde(instr, STORE_SUBSCR_LIST_INT); goto success; } else { @@ -1352,7 +1352,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins } } if (container_type == &PyDict_Type) { - _Py_SET_OPCODE(*instr, STORE_SUBSCR_DICT); + _py_set_opocde(instr, STORE_SUBSCR_DICT); goto success; } #ifdef Py_STATS @@ -1419,7 +1419,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, STORE_SUBSCR); + _py_set_opocde(instr, STORE_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1441,20 +1441,20 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, int oparg = _Py_OPARG(*instr); if (nargs == 1 && kwnames == NULL && oparg == 1) { if (tp == &PyUnicode_Type) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_STR_1); + _py_set_opocde(instr, CALL_NO_KW_STR_1); return 0; } else if (tp == &PyType_Type) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_TYPE_1); + _py_set_opocde(instr, CALL_NO_KW_TYPE_1); return 0; } else if (tp == &PyTuple_Type) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_TUPLE_1); + _py_set_opocde(instr, CALL_NO_KW_TUPLE_1); return 0; } } if (tp->tp_vectorcall != NULL) { - _Py_SET_OPCODE(*instr, CALL_BUILTIN_CLASS); + _py_set_opocde(instr, CALL_BUILTIN_CLASS); return 0; } SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? @@ -1506,7 +1506,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); + _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); return 0; } case METH_O: { @@ -1520,18 +1520,18 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (_Py_OPCODE(next) == POP_TOP); int oparg = _Py_OPARG(*instr); if ((PyObject *)descr == list_append && oparg == 1 && pop) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_LIST_APPEND); + _py_set_opocde(instr, CALL_NO_KW_LIST_APPEND); return 0; } - _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); + _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); return 0; } case METH_FASTCALL: { - _Py_SET_OPCODE(*instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); + _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); return 0; } case METH_FASTCALL|METH_KEYWORDS: { - _Py_SET_OPCODE(*instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + _py_set_opocde(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } @@ -1582,14 +1582,14 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, write_u32(cache->func_version, version); cache->min_args = min_args; if (argcount == nargs) { - _Py_SET_OPCODE(*instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); + _py_set_opocde(instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); } else if (bound_method) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); return -1; } else { - _Py_SET_OPCODE(*instr, CALL_PY_WITH_DEFAULTS); + _py_set_opocde(instr, CALL_PY_WITH_DEFAULTS); } return 0; } @@ -1616,10 +1616,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_LEN); + _py_set_opocde(instr, CALL_NO_KW_LEN); return 0; } - _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_O); + _py_set_opocde(instr, CALL_NO_KW_BUILTIN_O); return 0; } case METH_FASTCALL: { @@ -1631,15 +1631,15 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - _Py_SET_OPCODE(*instr, CALL_NO_KW_ISINSTANCE); + _py_set_opocde(instr, CALL_NO_KW_ISINSTANCE); return 0; } } - _Py_SET_OPCODE(*instr, CALL_NO_KW_BUILTIN_FAST); + _py_set_opocde(instr, CALL_NO_KW_BUILTIN_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - _Py_SET_OPCODE(*instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); + _py_set_opocde(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); return 0; } default: @@ -1732,7 +1732,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - _Py_SET_OPCODE(*instr, CALL); + _py_set_opocde(instr, CALL); cache->counter = adaptive_counter_backoff(cache->counter); } else { @@ -1829,18 +1829,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, bool to_store = (_Py_OPCODE(next) == STORE_FAST || _Py_OPCODE(next) == STORE_FAST__LOAD_FAST); if (to_store && locals[_Py_OPARG(next)] == lhs) { - _Py_SET_OPCODE(*instr, BINARY_OP_INPLACE_ADD_UNICODE); + _py_set_opocde(instr, BINARY_OP_INPLACE_ADD_UNICODE); goto success; } - _Py_SET_OPCODE(*instr, BINARY_OP_ADD_UNICODE); + _py_set_opocde(instr, BINARY_OP_ADD_UNICODE); goto success; } if (PyLong_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_ADD_INT); + _py_set_opocde(instr, BINARY_OP_ADD_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_ADD_FLOAT); + _py_set_opocde(instr, BINARY_OP_ADD_FLOAT); goto success; } break; @@ -1850,11 +1850,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_INT); + _py_set_opocde(instr, BINARY_OP_MULTIPLY_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_FLOAT); + _py_set_opocde(instr, BINARY_OP_MULTIPLY_FLOAT); goto success; } break; @@ -1864,18 +1864,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_INT); + _py_set_opocde(instr, BINARY_OP_SUBTRACT_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_FLOAT); + _py_set_opocde(instr, BINARY_OP_SUBTRACT_FLOAT); goto success; } break; } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - _Py_SET_OPCODE(*instr, BINARY_OP); + _py_set_opocde(instr, BINARY_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1957,13 +1957,13 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } if (PyFloat_CheckExact(lhs)) { - _Py_SET_OPCODE(*instr, COMPARE_OP_FLOAT_JUMP); + _py_set_opocde(instr, COMPARE_OP_FLOAT_JUMP); cache->mask = when_to_jump_mask; goto success; } if (PyLong_CheckExact(lhs)) { if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) { - _Py_SET_OPCODE(*instr, COMPARE_OP_INT_JUMP); + _py_set_opocde(instr, COMPARE_OP_INT_JUMP); cache->mask = when_to_jump_mask; goto success; } @@ -1978,7 +1978,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } else { - _Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP); + _py_set_opocde(instr, COMPARE_OP_STR_JUMP); cache->mask = (when_to_jump_mask & 2) == 0; goto success; } @@ -1986,7 +1986,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); - _Py_SET_OPCODE(*instr, COMPARE_OP); + _py_set_opocde(instr, COMPARE_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2020,10 +2020,10 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) goto failure; } if (PyTuple_GET_SIZE(seq) == 2) { - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TWO_TUPLE); + _py_set_opocde(instr, UNPACK_SEQUENCE_TWO_TUPLE); goto success; } - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TUPLE); + _py_set_opocde(instr, UNPACK_SEQUENCE_TUPLE); goto success; } if (PyList_CheckExact(seq)) { @@ -2031,13 +2031,13 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); goto failure; } - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_LIST); + _py_set_opocde(instr, UNPACK_SEQUENCE_LIST); goto success; } SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE); + _py_set_opocde(instr, UNPACK_SEQUENCE); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2126,26 +2126,26 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; int next_op = _PyOpcode_Deopt[_Py_OPCODE(next)]; if (tp == &PyListIter_Type) { - _Py_SET_OPCODE(*instr, FOR_ITER_LIST); + _py_set_opocde(instr, FOR_ITER_LIST); goto success; } else if (tp == &PyTupleIter_Type) { - _Py_SET_OPCODE(*instr, FOR_ITER_TUPLE); + _py_set_opocde(instr, FOR_ITER_TUPLE); goto success; } else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { - _Py_SET_OPCODE(*instr, FOR_ITER_RANGE); + _py_set_opocde(instr, FOR_ITER_RANGE); goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { assert(_Py_OPCODE(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1]) == END_FOR); - _Py_SET_OPCODE(*instr, FOR_ITER_GEN); + _py_set_opocde(instr, FOR_ITER_GEN); goto success; } SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); - _Py_SET_OPCODE(*instr, FOR_ITER); + _py_set_opocde(instr, FOR_ITER); cache->counter = adaptive_counter_backoff(cache->counter); return; success: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 5930c797b8a4..2dfc76f2560e 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -205,7 +205,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None else: typ = f"uint{bits}_t " func = f"read_u{bits}" - out.emit(f"{typ}{ceffect.name} = {func}(next_instr + {cache_offset});") + out.emit(f"{typ}{ceffect.name} = {func}(&next_instr[{cache_offset}].cache);") cache_offset += ceffect.size assert cache_offset == self.cache_offset + cache_adjust From webhook-mailer at python.org Wed Dec 14 06:37:26 2022 From: webhook-mailer at python.org (vsajip) Date: Wed, 14 Dec 2022 11:37:26 -0000 Subject: [Python-checkins] gh-100176: venv: Remove redundant compat code for Python <= 3.2 (#100177) Message-ID: <mailman.2922.1671017848.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3192c00a3cf136e06592d9a14d4d7b82412da4de commit: 3192c00a3cf136e06592d9a14d4d7b82412da4de branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: vsajip <vinay_sajip at yahoo.co.uk> date: 2022-12-14T11:37:11Z summary: gh-100176: venv: Remove redundant compat code for Python <= 3.2 (#100177) gh-100176: Remove redundant compat code for Python 3.2 and older Python 3.2 has been EOL since 2016-02-20 and 2.7 since 2020-01-01, so we can remove this old compatibility check and unindent the old else-block. Also, in the unindented block, replace a .format() call with an f-string. Plus similar changes in the documentation. files: M Doc/library/venv.rst M Lib/venv/__init__.py diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index adc6cd339ac1..2a41096de006 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -497,76 +497,68 @@ subclass which installs setuptools and pip into a created virtual environment:: url = 'https://bootstrap.pypa.io/get-pip.py' self.install_script(context, 'pip', url) + def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory in which to create the ' + 'virtual environment.') + parser.add_argument('--no-setuptools', default=False, + action='store_true', dest='nodist', + help="Don't install setuptools or pip in the " + "virtual environment.") + parser.add_argument('--no-pip', default=False, + action='store_true', dest='nopip', + help="Don't install pip in the virtual " + "environment.") + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the ' - 'virtual environment.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'virtual environment ' - 'directory if it already ' - 'exists, before virtual ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the virtual ' - 'environment directory to ' - 'use this version of ' - 'Python, assuming Python ' - 'has been upgraded ' - 'in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) + use_symlinks = True + parser.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'virtual environment ' + 'directory if it already ' + 'exists, before virtual ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the virtual ' + 'environment directory to ' + 'use this version of ' + 'Python, assuming Python ' + 'has been upgraded ' + 'in-place.') + parser.add_argument('--verbose', default=False, action='store_true', + dest='verbose', help='Display the output ' + 'from the scripts which ' + 'install setuptools and pip.') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = ExtendedEnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + nodist=options.nodist, + nopip=options.nopip, + verbose=options.verbose) + for d in options.dirs: + builder.create(d) if __name__ == '__main__': rc = 1 diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 7bfc2d1b6fe0..978c98336f2b 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -223,7 +223,7 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): force_copy = not self.symlinks if not force_copy: try: - if not os.path.islink(dst): # can't link to itself! + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) @@ -418,11 +418,11 @@ def install_scripts(self, context, path): binpath = context.bin_path plen = len(path) for root, dirs, files in os.walk(path): - if root == path: # at top-level, remove irrelevant dirs + if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: if d not in ('common', os.name): dirs.remove(d) - continue # ignore files in top level + continue # ignore files in top level for f in files: if (os.name == 'nt' and f.startswith('python') and f.endswith(('.exe', '.pdb'))): @@ -468,83 +468,76 @@ def create(env_dir, system_site_packages=False, clear=False, prompt=prompt, upgrade_deps=upgrade_deps) builder.create(env_dir) + def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with Python >= 3.3') + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.', + epilog='Once an environment has been ' + 'created, you may wish to ' + 'activate it, e.g. by ' + 'sourcing an activate script ' + 'in its bin directory.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory to create the environment in.') + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.', - epilog='Once an environment has been ' - 'created, you may wish to ' - 'activate it, e.g. by ' - 'sourcing an activate script ' - 'in its bin directory.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory to create the environment in.') - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - group = parser.add_mutually_exclusive_group() - group.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - group.add_argument('--copies', default=not use_symlinks, - action='store_false', dest='symlinks', - help='Try to use copies rather than symlinks, ' - 'even when symlinks are the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'environment directory if it ' - 'already exists, before ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the environment ' - 'directory to use this version ' - 'of Python, assuming Python ' - 'has been upgraded in-place.') - parser.add_argument('--without-pip', dest='with_pip', - default=True, action='store_false', - help='Skips installing or upgrading pip in the ' - 'virtual environment (pip is bootstrapped ' - 'by default)') - parser.add_argument('--prompt', - help='Provides an alternative prompt prefix for ' - 'this environment.') - parser.add_argument('--upgrade-deps', default=False, action='store_true', - dest='upgrade_deps', - help='Upgrade core dependencies: {} to the latest ' - 'version in PyPI'.format( - ' '.join(CORE_VENV_DEPS))) - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = EnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - with_pip=options.with_pip, - prompt=options.prompt, - upgrade_deps=options.upgrade_deps) - for d in options.dirs: - builder.create(d) + use_symlinks = True + group = parser.add_mutually_exclusive_group() + group.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + group.add_argument('--copies', default=not use_symlinks, + action='store_false', dest='symlinks', + help='Try to use copies rather than symlinks, ' + 'even when symlinks are the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'environment directory if it ' + 'already exists, before ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the environment ' + 'directory to use this version ' + 'of Python, assuming Python ' + 'has been upgraded in-place.') + parser.add_argument('--without-pip', dest='with_pip', + default=True, action='store_false', + help='Skips installing or upgrading pip in the ' + 'virtual environment (pip is bootstrapped ' + 'by default)') + parser.add_argument('--prompt', + help='Provides an alternative prompt prefix for ' + 'this environment.') + parser.add_argument('--upgrade-deps', default=False, action='store_true', + dest='upgrade_deps', + help=f'Upgrade core dependencies: {" ".join(CORE_VENV_DEPS)} ' + 'to the latest version in PyPI') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = EnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + with_pip=options.with_pip, + prompt=options.prompt, + upgrade_deps=options.upgrade_deps) + for d in options.dirs: + builder.create(d) + if __name__ == '__main__': rc = 1 From webhook-mailer at python.org Wed Dec 14 10:50:42 2022 From: webhook-mailer at python.org (markshannon) Date: Wed, 14 Dec 2022 15:50:42 -0000 Subject: [Python-checkins] Assorted minor fixes for specialization stats. (GH-100219) Message-ID: <mailman.2923.1671033042.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5693f45b19de47f8cb2f08d3baf43e626d3fbdf3 commit: 5693f45b19de47f8cb2f08d3baf43e626d3fbdf3 branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-14T15:50:02Z summary: Assorted minor fixes for specialization stats. (GH-100219) files: M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Python/specialize.c b/Python/specialize.c index 678c5d66b660..4fa2c2ffc04b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -307,6 +307,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_NOT_PY_FUNCTION 7 +#define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17 #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18 /* Attributes */ @@ -332,6 +333,8 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 +#define SPEC_FAIL_ATTR_NOT_IN_KEYS 29 +#define SPEC_FAIL_ATTR_NOT_IN_DICT 30 /* Binary subscr and store subscr */ @@ -448,41 +451,44 @@ static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); static int -specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr, - PyObject *name, int opcode, int opcode_module) -{ +specialize_module_load_attr( + PyObject *owner, _Py_CODEUNIT *instr, PyObject *name +) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyModuleObject *m = (PyModuleObject *)owner; assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); PyDictObject *dict = (PyDictObject *)m->md_dict; if (dict == NULL) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT); return -1; } if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); return -1; } Py_ssize_t index = _PyDict_LookupIndex(dict, &_Py_ID(__getattr__)); assert(index != DKIX_ERROR); if (index != DKIX_EMPTY) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND); return -1; } index = _PyDict_LookupIndex(dict, name); assert (index != DKIX_ERROR); if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(LOAD_ATTR, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND : + SPEC_FAIL_OUT_OF_RANGE); return -1; } uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys); if (keys_version == 0) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - _py_set_opocde(instr, opcode_module); + _py_set_opocde(instr, LOAD_ATTR_MODULE); return 0; } @@ -629,7 +635,10 @@ specialize_dict_access( Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); assert (index != DKIX_ERROR); if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_KEYS : + SPEC_FAIL_OUT_OF_RANGE); return 0; } write_u32(cache->version, type->tp_version_tag); @@ -646,7 +655,10 @@ specialize_dict_access( Py_ssize_t index = _PyDict_LookupIndex(dict, name); if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_DICT : + SPEC_FAIL_OUT_OF_RANGE); return 0; } cache->index = (uint16_t)index; @@ -674,8 +686,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) goto fail; } if (PyModule_CheckExact(owner)) { - if (specialize_module_load_attr(owner, instr, name, LOAD_ATTR, - LOAD_ATTR_MODULE)) + if (specialize_module_load_attr(owner, instr, name)) { goto fail; } @@ -702,7 +713,9 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) goto success; } } - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + else { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + } goto fail; } case PROPERTY: @@ -1076,7 +1089,6 @@ PyObject *descr, DescriptorClassification kind) */ write_u32(cache->type_version, owner_cls->tp_version_tag); write_obj(cache->descr, descr); - // Fall through. return 1; fail: return 0; @@ -1092,6 +1104,7 @@ _Py_Specialize_LoadGlobal( _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1); assert(PyUnicode_CheckExact(name)); if (!PyDict_CheckExact(globals)) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT); goto fail; } PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys; @@ -1101,15 +1114,17 @@ _Py_Specialize_LoadGlobal( } Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name); if (index == DKIX_ERROR) { - SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } if (index != DKIX_EMPTY) { if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); if (keys_version == 0) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } cache->index = (uint16_t)index; @@ -1118,6 +1133,7 @@ _Py_Specialize_LoadGlobal( goto success; } if (!PyDict_CheckExact(builtins)) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT); goto fail; } PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys; @@ -1127,10 +1143,11 @@ _Py_Specialize_LoadGlobal( } index = _PyDictKeys_StringLookup(builtin_keys, name); if (index == DKIX_ERROR) { - SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index c30a60e9514b..81b06f9f7469 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -32,7 +32,7 @@ opmap = {name: i for i, name in enumerate(opname)} opmap = dict(sorted(opmap.items())) -TOTAL = "specialization.deferred", "specialization.hit", "specialization.miss", "execution_count" +TOTAL = "specialization.hit", "specialization.miss", "execution_count" def format_ratio(num, den): """ @@ -90,7 +90,7 @@ def calculate_specialization_stats(family_stats, total): if key in ("specialization.hit", "specialization.miss"): label = key[len("specialization."):] elif key == "execution_count": - label = "unquickened" + continue elif key in ("specialization.success", "specialization.failure", "specializable"): continue elif key.startswith("pair"): @@ -115,7 +115,7 @@ def calculate_specialization_success_failure(family_stats): def calculate_specialization_failure_kinds(name, family_stats, defines): total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [ 0 ] * 30 + failure_kinds = [ 0 ] * 40 for key in family_stats: if not key.startswith("specialization.failure_kind"): continue @@ -224,7 +224,7 @@ def pretty(defname): return defname.replace("_", " ").lower() def kind_to_text(kind, defines, opname): - if kind < 7: + if kind <= 7: return pretty(defines[kind][0]) if opname.endswith("ATTR"): opname = "ATTR" @@ -241,10 +241,7 @@ def categorized_counts(opcode_stats): not_specialized = 0 specialized_instructions = { op for op in opcode._specialized_instructions - if "__" not in op and "ADAPTIVE" not in op} - adaptive_instructions = { - op for op in opcode._specialized_instructions - if "ADAPTIVE" in op} + if "__" not in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: continue @@ -252,8 +249,6 @@ def categorized_counts(opcode_stats): name = opname[i] if "specializable" in opcode_stat: not_specialized += count - elif name in adaptive_instructions: - not_specialized += count elif name in specialized_instructions: miss = opcode_stat.get("specialization.miss", 0) not_specialized += miss From webhook-mailer at python.org Wed Dec 14 11:57:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 14 Dec 2022 16:57:32 -0000 Subject: [Python-checkins] gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (#100249) Message-ID: <mailman.2924.1671037053.3313.python-checkins@python.org> https://github.com/python/cpython/commit/96638538002fc6e209755c06f613b0a59eb91166 commit: 96638538002fc6e209755c06f613b0a59eb91166 branch: main author: busywhitespace <busywhitespace at tuta.io> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-14T22:27:02+05:30 summary: gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (#100249) files: M Doc/library/asyncio-stream.rst diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index d87e3c042c99..c1ae8abb9abc 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -52,6 +52,7 @@ and work with streams: limit=None, ssl=None, family=0, proto=0, \ flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ happy_eyeballs_delay=None, interleave=None) Establish a network connection and return a pair of @@ -82,6 +83,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_server(client_connected_cb, host=None, \ port=None, *, limit=None, \ @@ -89,7 +93,7 @@ and work with streams: flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ reuse_port=None, ssl_handshake_timeout=None, \ - start_serving=True) + ssl_shutdown_timeout=None, start_serving=True) Start a socket server. @@ -121,12 +125,15 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. rubric:: Unix Sockets .. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \ ssl=None, sock=None, server_hostname=None, \ - ssl_handshake_timeout=None) + ssl_handshake_timeout=None, ssl_shutdown_timeout=None) Establish a Unix socket connection and return a pair of ``(reader, writer)``. @@ -150,10 +157,14 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ *, limit=None, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, start_serving=True) + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, start_serving=True) Start a Unix socket server. @@ -176,6 +187,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + StreamReader ============ From webhook-mailer at python.org Wed Dec 14 12:07:42 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 14 Dec 2022 17:07:42 -0000 Subject: [Python-checkins] gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (GH-100249) Message-ID: <mailman.2925.1671037663.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cb1001ce23205eeed69395cdbfdb99e99a37aca1 commit: cb1001ce23205eeed69395cdbfdb99e99a37aca1 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-14T09:07:36-08:00 summary: gh-100248: Add missing `ssl_shutdown_timeout` parameter in `asyncio` docs (GH-100249) (cherry picked from commit 96638538002fc6e209755c06f613b0a59eb91166) Co-authored-by: busywhitespace <busywhitespace at tuta.io> files: M Doc/library/asyncio-stream.rst diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index d87e3c042c99..c1ae8abb9abc 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -52,6 +52,7 @@ and work with streams: limit=None, ssl=None, family=0, proto=0, \ flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ happy_eyeballs_delay=None, interleave=None) Establish a network connection and return a pair of @@ -82,6 +83,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_server(client_connected_cb, host=None, \ port=None, *, limit=None, \ @@ -89,7 +93,7 @@ and work with streams: flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ reuse_port=None, ssl_handshake_timeout=None, \ - start_serving=True) + ssl_shutdown_timeout=None, start_serving=True) Start a socket server. @@ -121,12 +125,15 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. rubric:: Unix Sockets .. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \ ssl=None, sock=None, server_hostname=None, \ - ssl_handshake_timeout=None) + ssl_handshake_timeout=None, ssl_shutdown_timeout=None) Establish a Unix socket connection and return a pair of ``(reader, writer)``. @@ -150,10 +157,14 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ *, limit=None, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, start_serving=True) + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, start_serving=True) Start a Unix socket server. @@ -176,6 +187,9 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. + StreamReader ============ From webhook-mailer at python.org Wed Dec 14 13:54:23 2022 From: webhook-mailer at python.org (ericsnowcurrently) Date: Wed, 14 Dec 2022 18:54:23 -0000 Subject: [Python-checkins] gh-90111: Minor Cleanup for Runtime-Global Objects (gh-100254) Message-ID: <mailman.2926.1671044064.3313.python-checkins@python.org> https://github.com/python/cpython/commit/aa8591e9ca9a3e55e96a03c17fa9d3902f0674dc commit: aa8591e9ca9a3e55e96a03c17fa9d3902f0674dc branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: ericsnowcurrently <ericsnowcurrently at gmail.com> date: 2022-12-14T11:53:57-07:00 summary: gh-90111: Minor Cleanup for Runtime-Global Objects (gh-100254) * move _PyRuntime.global_objects.interned to _PyRuntime.cached_objects.interned_strings (and use _Py_CACHED_OBJECT()) * rename _PyRuntime.global_objects to _PyRuntime.static_objects (This also relates to gh-96075.) https://github.com/python/cpython/issues/90111 files: M Include/internal/pycore_global_objects.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Objects/unicodeobject.c diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 80f6bb2c8a7c..d0461fa7e82e 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -28,14 +28,16 @@ extern "C" { struct _Py_cached_objects { PyObject *str_replace_inf; + + PyObject *interned_strings; }; #define _Py_GLOBAL_OBJECT(NAME) \ - _PyRuntime.global_objects.NAME + _PyRuntime.static_objects.NAME #define _Py_SINGLETON(NAME) \ _Py_GLOBAL_OBJECT(singletons.NAME) -struct _Py_global_objects { +struct _Py_static_objects { struct { /* Small integers are preallocated in this array so that they * can be shared. @@ -59,8 +61,6 @@ struct _Py_global_objects { PyHamtNode_Bitmap hamt_bitmap_node_empty; _PyContextTokenMissing context_token_missing; } singletons; - - PyObject *interned; }; #define _Py_INTERP_CACHED_OBJECT(interp, NAME) \ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 92ed45956c99..d100e836c7d1 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -163,7 +163,7 @@ typedef struct pyruntimestate { /* All the objects that are shared by the runtime's interpreters. */ struct _Py_cached_objects cached_objects; - struct _Py_global_objects global_objects; + struct _Py_static_objects static_objects; /* The following fields are here to avoid allocation during init. The data is exposed through _PyRuntimeState pointer fields. diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 1431096e2d24..6342a28f4df9 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -70,7 +70,7 @@ extern "C" { .types = { \ .next_version_tag = 1, \ }, \ - .global_objects = { \ + .static_objects = { \ .singletons = { \ .small_ints = _Py_small_ints_INIT, \ .bytes_empty = _PyBytes_SIMPLE_INIT(0, 0), \ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index b721ccd805ed..f0c7aa7707fd 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -233,12 +233,12 @@ static inline PyObject* unicode_new_empty(void) */ static inline PyObject *get_interned_dict(void) { - return _PyRuntime.global_objects.interned; + return _Py_CACHED_OBJECT(interned_strings); } static inline void set_interned_dict(PyObject *dict) { - _PyRuntime.global_objects.interned = dict; + _Py_CACHED_OBJECT(interned_strings) = dict; } #define _Py_RETURN_UNICODE_EMPTY() \ From webhook-mailer at python.org Wed Dec 14 14:14:26 2022 From: webhook-mailer at python.org (markshannon) Date: Wed, 14 Dec 2022 19:14:26 -0000 Subject: [Python-checkins] GH-100000: Cleanup and polish various watchers code (GH-99998) Message-ID: <mailman.2927.1671045267.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ae83c782155ffe86830c3255e338f366e331ad30 commit: ae83c782155ffe86830c3255e338f366e331ad30 branch: main author: Itamar Ostricher <itamarost at gmail.com> committer: markshannon <mark at hotpy.org> date: 2022-12-14T19:14:16Z summary: GH-100000: Cleanup and polish various watchers code (GH-99998) * Initialize `type_watchers` array to `NULL`s * Optimize code watchers notification * Optimize func watchers notification files: M Include/internal/pycore_interp.h M Modules/_testcapi/watchers.c M Objects/codeobject.c M Objects/funcobject.c M Objects/typeobject.c M Python/pystate.c diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ffda1351952d..0e3d46852f2e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -142,7 +142,6 @@ struct _is { // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; - PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS]; PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS]; // One bit is set for each non-NULL entry in func_watchers uint8_t active_func_watchers; diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 1d91c206f630..2e8fe1dbf786 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -630,14 +630,16 @@ static PyMethodDef test_methods[] = { {"clear_dict_watcher", clear_dict_watcher, METH_O, NULL}, {"watch_dict", watch_dict, METH_VARARGS, NULL}, {"unwatch_dict", unwatch_dict, METH_VARARGS, NULL}, - {"get_dict_watcher_events", get_dict_watcher_events, METH_NOARGS, NULL}, + {"get_dict_watcher_events", + (PyCFunction) get_dict_watcher_events, METH_NOARGS, NULL}, // Type watchers. {"add_type_watcher", add_type_watcher, METH_O, NULL}, {"clear_type_watcher", clear_type_watcher, METH_O, NULL}, {"watch_type", watch_type, METH_VARARGS, NULL}, {"unwatch_type", unwatch_type, METH_VARARGS, NULL}, - {"get_type_modified_events", get_type_modified_events, METH_NOARGS, NULL}, + {"get_type_modified_events", + (PyCFunction) get_type_modified_events, METH_NOARGS, NULL}, // Code object watchers. {"add_code_watcher", add_code_watcher, METH_O, NULL}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index f455cc603aae..1e5a92270be8 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -15,14 +15,21 @@ static void notify_code_watchers(PyCodeEvent event, PyCodeObject *co) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->active_code_watchers) { - assert(interp->_initialized); - for (int i = 0; i < CODE_MAX_WATCHERS; i++) { + assert(interp->_initialized); + uint8_t bits = interp->active_code_watchers; + int i = 0; + while (bits) { + assert(i < CODE_MAX_WATCHERS); + if (bits & 1) { PyCode_WatchCallback cb = interp->code_watchers[i]; - if ((cb != NULL) && (cb(event, co) < 0)) { + // callback must be non-null if the watcher bit is set + assert(cb != NULL); + if (cb(event, co) < 0) { PyErr_WriteUnraisable((PyObject *) co); } } + i++; + bits >>= 1; } } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 9df06520586a..d5cf5b9277b3 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -12,11 +12,20 @@ static void notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) { - for (int i = 0; i < FUNC_MAX_WATCHERS; i++) { - PyFunction_WatchCallback cb = interp->func_watchers[i]; - if ((cb != NULL) && (cb(event, func, new_value) < 0)) { - PyErr_WriteUnraisable((PyObject *) func); + uint8_t bits = interp->active_func_watchers; + int i = 0; + while (bits) { + assert(i < FUNC_MAX_WATCHERS); + if (bits & 1) { + PyFunction_WatchCallback cb = interp->func_watchers[i]; + // callback must be non-null if the watcher bit is set + assert(cb != NULL); + if (cb(event, func, new_value) < 0) { + PyErr_WriteUnraisable((PyObject *) func); + } } + i++; + bits >>= 1; } } @@ -25,6 +34,7 @@ handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) { PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); if (interp->active_func_watchers) { notify_func_watchers(interp, event, func, new_value); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ae80f5a8fd88..2c3b39521a6d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -485,23 +485,24 @@ PyType_Modified(PyTypeObject *type) } } + // Notify registered type watchers, if any if (type->tp_watched) { PyInterpreterState *interp = _PyInterpreterState_GET(); int bits = type->tp_watched; int i = 0; - while(bits && i < TYPE_MAX_WATCHERS) { + while (bits) { + assert(i < TYPE_MAX_WATCHERS); if (bits & 1) { PyType_WatchCallback cb = interp->type_watchers[i]; if (cb && (cb(type) < 0)) { PyErr_WriteUnraisable((PyObject *)type); } } - i += 1; + i++; bits >>= 1; } } - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; /* 0 is not a valid version tag */ } diff --git a/Python/pystate.c b/Python/pystate.c index ea3c22c5d71a..f52fc38b3586 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -461,6 +461,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->dict_state.watchers[i] = NULL; } + for (int i=0; i < TYPE_MAX_WATCHERS; i++) { + interp->type_watchers[i] = NULL; + } + for (int i=0; i < FUNC_MAX_WATCHERS; i++) { interp->func_watchers[i] = NULL; } From webhook-mailer at python.org Wed Dec 14 19:39:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 15 Dec 2022 00:39:11 -0000 Subject: [Python-checkins] GH-100222: fix typo _py_set_opocde -> _py_set_opcode (GH-100259) Message-ID: <mailman.2928.1671064752.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bdd86741bebd3efb51e540d5148e658cb34fd3ce commit: bdd86741bebd3efb51e540d5148e658cb34fd3ce branch: main author: Carl Meyer <carl at oddbird.net> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-14T16:39:00-08:00 summary: GH-100222: fix typo _py_set_opocde -> _py_set_opcode (GH-100259) Typo introduced in #100223. Automerge-Triggered-By: GH:brandtbucher files: M Include/cpython/code.h M Python/specialize.c diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 6a13ff2dfd9f..1c619322926e 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -28,7 +28,7 @@ typedef union { #define _Py_OPARG(word) ((word).oparg) static inline void -_py_set_opocde(_Py_CODEUNIT *word, uint8_t opcode) +_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) { word->opcode = opcode; } diff --git a/Python/specialize.c b/Python/specialize.c index 4fa2c2ffc04b..e88fdde79a67 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -488,7 +488,7 @@ specialize_module_load_attr( } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - _py_set_opocde(instr, LOAD_ATTR_MODULE); + _py_set_opcode(instr, LOAD_ATTR_MODULE); return 0; } @@ -643,7 +643,7 @@ specialize_dict_access( } write_u32(cache->version, type->tp_version_tag); cache->index = (uint16_t)index; - _py_set_opocde(instr, values_op); + _py_set_opcode(instr, values_op); } else { PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); @@ -663,7 +663,7 @@ specialize_dict_access( } cache->index = (uint16_t)index; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, hint_op); + _py_set_opcode(instr, hint_op); } return 1; } @@ -743,7 +743,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ write_obj(lm_cache->descr, fget); - _py_set_opocde(instr, LOAD_ATTR_PROPERTY); + _py_set_opcode(instr, LOAD_ATTR_PROPERTY); goto success; } case OBJECT_SLOT: @@ -767,7 +767,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, LOAD_ATTR_SLOT); + _py_set_opcode(instr, LOAD_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -776,7 +776,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, LOAD_ATTR_SLOT); + _py_set_opcode(instr, LOAD_ATTR_SLOT); goto success; } case OTHER_SLOT: @@ -804,7 +804,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) /* borrowed */ write_obj(lm_cache->descr, descr); write_u32(lm_cache->type_version, type->tp_version_tag); - _py_set_opocde(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + _py_set_opcode(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); goto success; } case BUILTIN_CLASSMETHOD: @@ -822,7 +822,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, LOAD_ATTR); + _py_set_opcode(instr, LOAD_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -881,7 +881,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opocde(instr, STORE_ATTR_SLOT); + _py_set_opcode(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -910,7 +910,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, STORE_ATTR); + _py_set_opcode(instr, STORE_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -974,7 +974,7 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, case NON_DESCRIPTOR: write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); write_obj(cache->descr, descr); - _py_set_opocde(instr, LOAD_ATTR_CLASS); + _py_set_opcode(instr, LOAD_ATTR_CLASS); return 0; #ifdef Py_STATS case ABSENT: @@ -1056,21 +1056,21 @@ PyObject *descr, DescriptorClassification kind) } switch(dictkind) { case NO_DICT: - _py_set_opocde(instr, LOAD_ATTR_METHOD_NO_DICT); + _py_set_opcode(instr, LOAD_ATTR_METHOD_NO_DICT); break; case MANAGED_VALUES: - _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_VALUES); + _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); goto fail; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _py_set_opocde(instr, LOAD_ATTR_METHOD_WITH_DICT); + _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_DICT); break; case LAZY_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _py_set_opocde(instr, LOAD_ATTR_METHOD_LAZY_DICT); + _py_set_opcode(instr, LOAD_ATTR_METHOD_LAZY_DICT); break; } /* `descr` is borrowed. This is safe for methods (even inherited ones from @@ -1129,7 +1129,7 @@ _Py_Specialize_LoadGlobal( } cache->index = (uint16_t)index; write_u32(cache->module_keys_version, keys_version); - _py_set_opocde(instr, LOAD_GLOBAL_MODULE); + _py_set_opcode(instr, LOAD_GLOBAL_MODULE); goto success; } if (!PyDict_CheckExact(builtins)) { @@ -1167,12 +1167,12 @@ _Py_Specialize_LoadGlobal( cache->index = (uint16_t)index; write_u32(cache->module_keys_version, globals_version); cache->builtin_keys_version = (uint16_t)builtins_version; - _py_set_opocde(instr, LOAD_GLOBAL_BUILTIN); + _py_set_opcode(instr, LOAD_GLOBAL_BUILTIN); goto success; fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, LOAD_GLOBAL); + _py_set_opcode(instr, LOAD_GLOBAL); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1276,7 +1276,7 @@ _Py_Specialize_BinarySubscr( PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opocde(instr, BINARY_SUBSCR_LIST_INT); + _py_set_opcode(instr, BINARY_SUBSCR_LIST_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1285,7 +1285,7 @@ _Py_Specialize_BinarySubscr( } if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opocde(instr, BINARY_SUBSCR_TUPLE_INT); + _py_set_opcode(instr, BINARY_SUBSCR_TUPLE_INT); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1293,7 +1293,7 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - _py_set_opocde(instr, BINARY_SUBSCR_DICT); + _py_set_opcode(instr, BINARY_SUBSCR_DICT); goto success; } PyTypeObject *cls = Py_TYPE(container); @@ -1324,7 +1324,7 @@ _Py_Specialize_BinarySubscr( } cache->func_version = version; ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor; - _py_set_opocde(instr, BINARY_SUBSCR_GETITEM); + _py_set_opcode(instr, BINARY_SUBSCR_GETITEM); goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1332,7 +1332,7 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, BINARY_SUBSCR); + _py_set_opcode(instr, BINARY_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1351,7 +1351,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) && ((PyLongObject *)sub)->ob_digit[0] < (size_t)PyList_GET_SIZE(container)) { - _py_set_opocde(instr, STORE_SUBSCR_LIST_INT); + _py_set_opcode(instr, STORE_SUBSCR_LIST_INT); goto success; } else { @@ -1369,7 +1369,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins } } if (container_type == &PyDict_Type) { - _py_set_opocde(instr, STORE_SUBSCR_DICT); + _py_set_opcode(instr, STORE_SUBSCR_DICT); goto success; } #ifdef Py_STATS @@ -1436,7 +1436,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, STORE_SUBSCR); + _py_set_opcode(instr, STORE_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1458,20 +1458,20 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, int oparg = _Py_OPARG(*instr); if (nargs == 1 && kwnames == NULL && oparg == 1) { if (tp == &PyUnicode_Type) { - _py_set_opocde(instr, CALL_NO_KW_STR_1); + _py_set_opcode(instr, CALL_NO_KW_STR_1); return 0; } else if (tp == &PyType_Type) { - _py_set_opocde(instr, CALL_NO_KW_TYPE_1); + _py_set_opcode(instr, CALL_NO_KW_TYPE_1); return 0; } else if (tp == &PyTuple_Type) { - _py_set_opocde(instr, CALL_NO_KW_TUPLE_1); + _py_set_opcode(instr, CALL_NO_KW_TUPLE_1); return 0; } } if (tp->tp_vectorcall != NULL) { - _py_set_opocde(instr, CALL_BUILTIN_CLASS); + _py_set_opcode(instr, CALL_BUILTIN_CLASS); return 0; } SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? @@ -1523,7 +1523,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); + _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); return 0; } case METH_O: { @@ -1537,18 +1537,18 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (_Py_OPCODE(next) == POP_TOP); int oparg = _Py_OPARG(*instr); if ((PyObject *)descr == list_append && oparg == 1 && pop) { - _py_set_opocde(instr, CALL_NO_KW_LIST_APPEND); + _py_set_opcode(instr, CALL_NO_KW_LIST_APPEND); return 0; } - _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); + _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); return 0; } case METH_FASTCALL: { - _py_set_opocde(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); + _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); return 0; } case METH_FASTCALL|METH_KEYWORDS: { - _py_set_opocde(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + _py_set_opcode(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } @@ -1599,14 +1599,14 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, write_u32(cache->func_version, version); cache->min_args = min_args; if (argcount == nargs) { - _py_set_opocde(instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); + _py_set_opcode(instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); } else if (bound_method) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); return -1; } else { - _py_set_opocde(instr, CALL_PY_WITH_DEFAULTS); + _py_set_opcode(instr, CALL_PY_WITH_DEFAULTS); } return 0; } @@ -1633,10 +1633,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - _py_set_opocde(instr, CALL_NO_KW_LEN); + _py_set_opcode(instr, CALL_NO_KW_LEN); return 0; } - _py_set_opocde(instr, CALL_NO_KW_BUILTIN_O); + _py_set_opcode(instr, CALL_NO_KW_BUILTIN_O); return 0; } case METH_FASTCALL: { @@ -1648,15 +1648,15 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - _py_set_opocde(instr, CALL_NO_KW_ISINSTANCE); + _py_set_opcode(instr, CALL_NO_KW_ISINSTANCE); return 0; } } - _py_set_opocde(instr, CALL_NO_KW_BUILTIN_FAST); + _py_set_opcode(instr, CALL_NO_KW_BUILTIN_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - _py_set_opocde(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); + _py_set_opcode(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); return 0; } default: @@ -1749,7 +1749,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - _py_set_opocde(instr, CALL); + _py_set_opcode(instr, CALL); cache->counter = adaptive_counter_backoff(cache->counter); } else { @@ -1846,18 +1846,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, bool to_store = (_Py_OPCODE(next) == STORE_FAST || _Py_OPCODE(next) == STORE_FAST__LOAD_FAST); if (to_store && locals[_Py_OPARG(next)] == lhs) { - _py_set_opocde(instr, BINARY_OP_INPLACE_ADD_UNICODE); + _py_set_opcode(instr, BINARY_OP_INPLACE_ADD_UNICODE); goto success; } - _py_set_opocde(instr, BINARY_OP_ADD_UNICODE); + _py_set_opcode(instr, BINARY_OP_ADD_UNICODE); goto success; } if (PyLong_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_ADD_INT); + _py_set_opcode(instr, BINARY_OP_ADD_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_ADD_FLOAT); + _py_set_opcode(instr, BINARY_OP_ADD_FLOAT); goto success; } break; @@ -1867,11 +1867,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_MULTIPLY_INT); + _py_set_opcode(instr, BINARY_OP_MULTIPLY_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_MULTIPLY_FLOAT); + _py_set_opcode(instr, BINARY_OP_MULTIPLY_FLOAT); goto success; } break; @@ -1881,18 +1881,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_SUBTRACT_INT); + _py_set_opcode(instr, BINARY_OP_SUBTRACT_INT); goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, BINARY_OP_SUBTRACT_FLOAT); + _py_set_opcode(instr, BINARY_OP_SUBTRACT_FLOAT); goto success; } break; } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - _py_set_opocde(instr, BINARY_OP); + _py_set_opcode(instr, BINARY_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1974,13 +1974,13 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } if (PyFloat_CheckExact(lhs)) { - _py_set_opocde(instr, COMPARE_OP_FLOAT_JUMP); + _py_set_opcode(instr, COMPARE_OP_FLOAT_JUMP); cache->mask = when_to_jump_mask; goto success; } if (PyLong_CheckExact(lhs)) { if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) { - _py_set_opocde(instr, COMPARE_OP_INT_JUMP); + _py_set_opcode(instr, COMPARE_OP_INT_JUMP); cache->mask = when_to_jump_mask; goto success; } @@ -1995,7 +1995,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } else { - _py_set_opocde(instr, COMPARE_OP_STR_JUMP); + _py_set_opcode(instr, COMPARE_OP_STR_JUMP); cache->mask = (when_to_jump_mask & 2) == 0; goto success; } @@ -2003,7 +2003,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); - _py_set_opocde(instr, COMPARE_OP); + _py_set_opcode(instr, COMPARE_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2037,10 +2037,10 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) goto failure; } if (PyTuple_GET_SIZE(seq) == 2) { - _py_set_opocde(instr, UNPACK_SEQUENCE_TWO_TUPLE); + _py_set_opcode(instr, UNPACK_SEQUENCE_TWO_TUPLE); goto success; } - _py_set_opocde(instr, UNPACK_SEQUENCE_TUPLE); + _py_set_opcode(instr, UNPACK_SEQUENCE_TUPLE); goto success; } if (PyList_CheckExact(seq)) { @@ -2048,13 +2048,13 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); goto failure; } - _py_set_opocde(instr, UNPACK_SEQUENCE_LIST); + _py_set_opcode(instr, UNPACK_SEQUENCE_LIST); goto success; } SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - _py_set_opocde(instr, UNPACK_SEQUENCE); + _py_set_opcode(instr, UNPACK_SEQUENCE); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2143,26 +2143,26 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; int next_op = _PyOpcode_Deopt[_Py_OPCODE(next)]; if (tp == &PyListIter_Type) { - _py_set_opocde(instr, FOR_ITER_LIST); + _py_set_opcode(instr, FOR_ITER_LIST); goto success; } else if (tp == &PyTupleIter_Type) { - _py_set_opocde(instr, FOR_ITER_TUPLE); + _py_set_opcode(instr, FOR_ITER_TUPLE); goto success; } else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { - _py_set_opocde(instr, FOR_ITER_RANGE); + _py_set_opcode(instr, FOR_ITER_RANGE); goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { assert(_Py_OPCODE(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1]) == END_FOR); - _py_set_opocde(instr, FOR_ITER_GEN); + _py_set_opcode(instr, FOR_ITER_GEN); goto success; } SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); - _py_set_opocde(instr, FOR_ITER); + _py_set_opcode(instr, FOR_ITER); cache->counter = adaptive_counter_backoff(cache->counter); return; success: From webhook-mailer at python.org Thu Dec 15 04:45:09 2022 From: webhook-mailer at python.org (markshannon) Date: Thu, 15 Dec 2022 09:45:09 -0000 Subject: [Python-checkins] Move stats for the method cache into the `Py_STAT` machinery (GH-100255) Message-ID: <mailman.2929.1671097510.3313.python-checkins@python.org> https://github.com/python/cpython/commit/48e352a2410b6e962d40359939a0d43aaba5ece9 commit: 48e352a2410b6e962d40359939a0d43aaba5ece9 branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-15T09:45:03Z summary: Move stats for the method cache into the `Py_STAT` machinery (GH-100255) files: M Include/internal/pycore_typeobject.h M Include/pystats.h M Objects/typeobject.c M Python/specialize.c diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 71f3068900da..c207ce615c6f 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -36,15 +36,9 @@ struct type_cache_entry { }; #define MCACHE_SIZE_EXP 12 -#define MCACHE_STATS 0 struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; -#if MCACHE_STATS - size_t hits; - size_t misses; - size_t collisions; -#endif }; /* For now we hard-code this to a value for which we are confident diff --git a/Include/pystats.h b/Include/pystats.h index 87e92aa4f05f..04630c9e0f92 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -65,8 +65,15 @@ typedef struct _object_stats { uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; + uint64_t type_cache_hits; + uint64_t type_cache_misses; + uint64_t type_cache_dunder_hits; + uint64_t type_cache_dunder_misses; + uint64_t type_cache_collisions; } ObjectStats; +# + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2c3b39521a6d..a96f993e99dd 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -327,18 +327,6 @@ static unsigned int _PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->types.type_cache; -#if MCACHE_STATS - size_t total = cache->hits + cache->collisions + cache->misses; - fprintf(stderr, "-- Method cache hits = %zd (%d%%)\n", - cache->hits, (int) (100.0 * cache->hits / total)); - fprintf(stderr, "-- Method cache true misses = %zd (%d%%)\n", - cache->misses, (int) (100.0 * cache->misses / total)); - fprintf(stderr, "-- Method cache collisions = %zd (%d%%)\n", - cache->collisions, (int) (100.0 * cache->collisions / total)); - fprintf(stderr, "-- Method cache size = %zd KiB\n", - sizeof(cache->hashtable) / 1024); -#endif - // Set to None, rather than NULL, so _PyType_Lookup() can // use Py_SETREF() rather than using slower Py_XSETREF(). type_cache_clear(cache, Py_None); @@ -4148,6 +4136,24 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) return res; } +/* Check if the "readied" PyUnicode name + is a double-underscore special name. */ +static int +is_dunder_name(PyObject *name) +{ + Py_ssize_t length = PyUnicode_GET_LENGTH(name); + int kind = PyUnicode_KIND(name); + /* Special names contain at least "__x__" and are always ASCII. */ + if (length > 4 && kind == PyUnicode_1BYTE_KIND) { + const Py_UCS1 *characters = PyUnicode_1BYTE_DATA(name); + return ( + ((characters[length-2] == '_') && (characters[length-1] == '_')) && + ((characters[0] == '_') && (characters[1] == '_')) + ); + } + return 0; +} + /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ PyObject * @@ -4161,12 +4167,13 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) struct type_cache_entry *entry = &cache->hashtable[h]; if (entry->version == type->tp_version_tag && entry->name == name) { -#if MCACHE_STATS - cache->hits++; -#endif assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); + OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); return entry->value; } + OBJECT_STAT_INC_COND(type_cache_misses, !is_dunder_name(name)); + OBJECT_STAT_INC_COND(type_cache_dunder_misses, is_dunder_name(name)); /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); @@ -4194,14 +4201,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) entry->version = type->tp_version_tag; entry->value = res; /* borrowed */ assert(_PyASCIIObject_CAST(name)->hash != -1); -#if MCACHE_STATS - if (entry->name != Py_None && entry->name != name) { - cache->collisions++; - } - else { - cache->misses++; - } -#endif + OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); Py_SETREF(entry->name, Py_NewRef(name)); } @@ -4218,24 +4218,6 @@ _PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) return _PyType_Lookup(type, oname); } -/* Check if the "readied" PyUnicode name - is a double-underscore special name. */ -static int -is_dunder_name(PyObject *name) -{ - Py_ssize_t length = PyUnicode_GET_LENGTH(name); - int kind = PyUnicode_KIND(name); - /* Special names contain at least "__x__" and are always ASCII. */ - if (length > 4 && kind == PyUnicode_1BYTE_KIND) { - const Py_UCS1 *characters = PyUnicode_1BYTE_DATA(name); - return ( - ((characters[length-2] == '_') && (characters[length-1] == '_')) && - ((characters[0] == '_') && (characters[1] == '_')) - ); - } - return 0; -} - /* This is similar to PyObject_GenericGetAttr(), but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ static PyObject * diff --git a/Python/specialize.c b/Python/specialize.c index e88fdde79a67..d1a38450fff0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -187,6 +187,11 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); + fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); + fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); + fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); + fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); + fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); } static void From webhook-mailer at python.org Thu Dec 15 06:26:14 2022 From: webhook-mailer at python.org (markshannon) Date: Thu, 15 Dec 2022 11:26:14 -0000 Subject: [Python-checkins] GH-99767: update PyTypeObject docs for type watchers (GH-99928) Message-ID: <mailman.2930.1671103575.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b7e4f1d97c6e784d2dee182d2b81541ddcff5751 commit: b7e4f1d97c6e784d2dee182d2b81541ddcff5751 branch: main author: Carl Meyer <carl at oddbird.net> committer: markshannon <mark at hotpy.org> date: 2022-12-15T11:26:08Z summary: GH-99767: update PyTypeObject docs for type watchers (GH-99928) files: M Doc/c-api/typeobj.rst M Doc/includes/typestruct.h diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 8f8869ec668a..c7b318b21853 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,6 +147,8 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_watched` | char | | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] @@ -2090,6 +2092,13 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) +.. c:member:: char PyTypeObject.tp_watched + + Internal. Do not use. + + .. versionadded:: 3.12 + + .. _static-types: Static Types diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 02f8ccfe4438..f0ad1e47cb0d 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -80,4 +80,7 @@ typedef struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + + /* bitset of which type-watchers care about this type */ + char tp_watched; } PyTypeObject; From webhook-mailer at python.org Thu Dec 15 13:39:37 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 15 Dec 2022 18:39:37 -0000 Subject: [Python-checkins] Remove uninformative itertools recipe (GH-100253) Message-ID: <mailman.2931.1671129578.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8356c14b4f81f4d0010afb61610edacf4068b804 commit: 8356c14b4f81f4d0010afb61610edacf4068b804 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-15T12:39:01-06:00 summary: Remove uninformative itertools recipe (GH-100253) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 624d2430ac20..9146ed1bfb62 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -829,10 +829,6 @@ which incur interpreter overhead. "Count how many times the predicate is true" return sum(map(pred, iterable)) - def pad_none(iterable): - "Returns the sequence elements and then returns None indefinitely." - return chain(iterable, repeat(None)) - def ncycles(iterable, n): "Returns the sequence elements n times" return chain.from_iterable(repeat(tuple(iterable), n)) @@ -1193,9 +1189,6 @@ which incur interpreter overhead. >>> take(5, map(int, repeatfunc(random.random))) [0, 0, 0, 0, 0] - >>> list(islice(pad_none('abc'), 0, 6)) - ['a', 'b', 'c', None, None, None] - >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] From webhook-mailer at python.org Thu Dec 15 13:40:51 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 15 Dec 2022 18:40:51 -0000 Subject: [Python-checkins] GH-100234: Set a default value for random.expovariate() (GH-100235) Message-ID: <mailman.2932.1671129653.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b430399d41fa88e9040cd055e55cf9211bf63c61 commit: b430399d41fa88e9040cd055e55cf9211bf63c61 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-15T12:40:45-06:00 summary: GH-100234: Set a default value for random.expovariate() (GH-100235) files: A Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst M Doc/library/random.rst M Lib/random.py M Lib/test/test_random.py diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 669204ba65b7..d944518a7909 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -320,7 +320,7 @@ be found in any statistics text. ``beta > 0``. Returned values range between 0 and 1. -.. function:: expovariate(lambd) +.. function:: expovariate(lambd = 1.0) Exponential distribution. *lambd* is 1.0 divided by the desired mean. It should be nonzero. (The parameter would be called @@ -328,6 +328,9 @@ be found in any statistics text. range from 0 to positive infinity if *lambd* is positive, and from negative infinity to 0 if *lambd* is negative. + .. versionchanged:: 3.12 + Added the default value for ``lambd``. + .. function:: gammavariate(alpha, beta) diff --git a/Lib/random.py b/Lib/random.py index c70294ee0cbf..e60b7294b6dd 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -577,7 +577,7 @@ def lognormvariate(self, mu, sigma): """ return _exp(self.normalvariate(mu, sigma)) - def expovariate(self, lambd): + def expovariate(self, lambd=1.0): """Exponential distribution. lambd is 1.0 divided by the desired mean. It should be diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 1e825c3572d2..67de54c7db8b 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -988,6 +988,7 @@ def test_zeroinputs(self): g.random = x[:].pop; g.uniform(1,10) g.random = x[:].pop; g.paretovariate(1.0) g.random = x[:].pop; g.expovariate(1.0) + g.random = x[:].pop; g.expovariate() g.random = x[:].pop; g.weibullvariate(1.0, 1.0) g.random = x[:].pop; g.vonmisesvariate(1.0, 1.0) g.random = x[:].pop; g.normalvariate(0.0, 1.0) diff --git a/Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst b/Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst new file mode 100644 index 000000000000..6d9b909d874b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-14-11-45-38.gh-issue-100234.kn6yWV.rst @@ -0,0 +1,2 @@ +Set a default value of 1.0 for the ``lambd`` parameter in +random.expovariate(). From webhook-mailer at python.org Thu Dec 15 13:48:04 2022 From: webhook-mailer at python.org (rhettinger) Date: Thu, 15 Dec 2022 18:48:04 -0000 Subject: [Python-checkins] [3.11] Remove uninformative itertools recipe (GH-100253) (GH-100275) Message-ID: <mailman.2933.1671130085.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9d2dcbbccdf4207249665b58c8bd28430c4f7afd commit: 9d2dcbbccdf4207249665b58c8bd28430c4f7afd branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-15T12:47:58-06:00 summary: [3.11] Remove uninformative itertools recipe (GH-100253) (GH-100275) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 3d93c2c0573e..ee70b16f42fb 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -790,10 +790,6 @@ which incur interpreter overhead. "Count how many times the predicate is true" return sum(map(pred, iterable)) - def pad_none(iterable): - "Returns the sequence elements and then returns None indefinitely." - return chain(iterable, repeat(None)) - def ncycles(iterable, n): "Returns the sequence elements n times" return chain.from_iterable(repeat(tuple(iterable), n)) @@ -1163,9 +1159,6 @@ which incur interpreter overhead. >>> take(5, map(int, repeatfunc(random.random))) [0, 0, 0, 0, 0] - >>> list(islice(pad_none('abc'), 0, 6)) - ['a', 'b', 'c', None, None, None] - >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] From webhook-mailer at python.org Thu Dec 15 17:59:43 2022 From: webhook-mailer at python.org (FFY00) Date: Thu, 15 Dec 2022 22:59:43 -0000 Subject: [Python-checkins] gh-78997: AttributeError if loading fails in LibraryLoader.__getattr__ Message-ID: <mailman.2934.1671145184.3313.python-checkins@python.org> https://github.com/python/cpython/commit/101cfe679fe10b9b662e815999dadf81fcc32c9c commit: 101cfe679fe10b9b662e815999dadf81fcc32c9c branch: main author: Ateeq Sharfuddin <ateeq.sharfuddin at gmail.com> committer: FFY00 <filipe.lains at gmail.com> date: 2022-12-15T22:59:37Z summary: gh-78997: AttributeError if loading fails in LibraryLoader.__getattr__ Co-authored-by: Zachary Ware <zachary.ware at gmail.com> Co-authored-by: Filipe La?ns <filipe.lains at gmail.com> Co-authored-by: Eric Wieser <wieser.eric at gmail.com> files: A Lib/ctypes/test/test_loading.py A Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst M Lib/ctypes/__init__.py diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index b94b337a3157..2e9d4c5e7238 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -444,7 +444,10 @@ def __init__(self, dlltype): def __getattr__(self, name): if name[0] == '_': raise AttributeError(name) - dll = self._dlltype(name) + try: + dll = self._dlltype(name) + except OSError: + raise AttributeError(name) setattr(self, name, dll) return dll diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py new file mode 100644 index 000000000000..b61d6fa2912a --- /dev/null +++ b/Lib/ctypes/test/test_loading.py @@ -0,0 +1,188 @@ +from ctypes import * +import os +import shutil +import subprocess +import sys +import unittest +import test.support +from test.support import import_helper +from test.support import os_helper +from ctypes.util import find_library + +libc_name = None + +def setUpModule(): + global libc_name + if os.name == "nt": + libc_name = find_library("c") + elif sys.platform == "cygwin": + libc_name = "cygwin1.dll" + else: + libc_name = find_library("c") + + if test.support.verbose: + print("libc_name is", libc_name) + +class LoaderTest(unittest.TestCase): + + unknowndll = "xxrandomnamexx" + + def test_load(self): + if libc_name is None: + self.skipTest('could not find libc') + CDLL(libc_name) + CDLL(os.path.basename(libc_name)) + self.assertRaises(OSError, CDLL, self.unknowndll) + + def test_load_version(self): + if libc_name is None: + self.skipTest('could not find libc') + if os.path.basename(libc_name) != 'libc.so.6': + self.skipTest('wrong libc path for test') + cdll.LoadLibrary("libc.so.6") + # linux uses version, libc 9 should not exist + self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") + self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) + + def test_find(self): + for name in ("c", "m"): + lib = find_library(name) + if lib: + cdll.LoadLibrary(lib) + CDLL(lib) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_library(self): + # CRT is no longer directly loadable. See issue23606 for the + # discussion about alternative approaches. + #self.assertIsNotNone(libc_name) + if test.support.verbose: + print(find_library("kernel32")) + print(find_library("user32")) + + if os.name == "nt": + windll.kernel32.GetModuleHandleW + windll["kernel32"].GetModuleHandleW + windll.LoadLibrary("kernel32").GetModuleHandleW + WinDLL("kernel32").GetModuleHandleW + # embedded null character + self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_ordinal_functions(self): + import _ctypes_test + dll = WinDLL(_ctypes_test.__file__) + # We load the same function both via ordinal and name + func_ord = dll[2] + func_name = dll.GetString + # addressof gets the address where the function pointer is stored + a_ord = addressof(func_ord) + a_name = addressof(func_name) + f_ord_addr = c_void_p.from_address(a_ord).value + f_name_addr = c_void_p.from_address(a_name).value + self.assertEqual(hex(f_ord_addr), hex(f_name_addr)) + + self.assertRaises(AttributeError, dll.__getitem__, 1234) + + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_1703286_A(self): + from _ctypes import LoadLibrary, FreeLibrary + # On winXP 64-bit, advapi32 loads at an address that does + # NOT fit into a 32-bit integer. FreeLibrary must be able + # to accept this address. + + # These are tests for https://www.python.org/sf/1703286 + handle = LoadLibrary("advapi32") + FreeLibrary(handle) + + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_1703286_B(self): + # Since on winXP 64-bit advapi32 loads like described + # above, the (arbitrarily selected) CloseEventLog function + # also has a high address. 'call_function' should accept + # addresses so large. + from _ctypes import call_function + advapi32 = windll.advapi32 + # Calling CloseEventLog with a NULL argument should fail, + # but the call should not segfault or so. + self.assertEqual(0, advapi32.CloseEventLog(None)) + windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p + windll.kernel32.GetProcAddress.restype = c_void_p + proc = windll.kernel32.GetProcAddress(advapi32._handle, + b"CloseEventLog") + self.assertTrue(proc) + # This is the real test: call the function via 'call_function' + self.assertEqual(0, call_function(proc, (None,))) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_hasattr(self): + # bpo-34816: shouldn't raise OSError + self.assertFalse(hasattr(windll, 'test')) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_dll_with_flags(self): + _sqlite3 = import_helper.import_module("_sqlite3") + src = _sqlite3.__file__ + if src.lower().endswith("_d.pyd"): + ext = "_d.dll" + else: + ext = ".dll" + + with os_helper.temp_dir() as tmp: + # We copy two files and load _sqlite3.dll (formerly .pyd), + # which has a dependency on sqlite3.dll. Then we test + # loading it in subprocesses to avoid it starting in memory + # for each test. + target = os.path.join(tmp, "_sqlite3.dll") + shutil.copy(src, target) + shutil.copy(os.path.join(os.path.dirname(src), "sqlite3" + ext), + os.path.join(tmp, "sqlite3" + ext)) + + def should_pass(command): + with self.subTest(command): + subprocess.check_output( + [sys.executable, "-c", + "from ctypes import *; import nt;" + command], + cwd=tmp + ) + + def should_fail(command): + with self.subTest(command): + with self.assertRaises(subprocess.CalledProcessError): + subprocess.check_output( + [sys.executable, "-c", + "from ctypes import *; import nt;" + command], + cwd=tmp, stderr=subprocess.STDOUT, + ) + + # Default load should not find this in CWD + should_fail("WinDLL('_sqlite3.dll')") + + # Relative path (but not just filename) should succeed + should_pass("WinDLL('./_sqlite3.dll')") + + # Insecure load flags should succeed + # Clear the DLL directory to avoid safe search settings propagating + should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)") + + # Full path load without DLL_LOAD_DIR shouldn't find dependency + should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + + "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32)") + + # Full path load with DLL_LOAD_DIR should succeed + should_pass("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + + "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32|" + + "nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)") + + # User-specified directory should succeed + should_pass("import os; p = os.add_dll_directory(os.getcwd());" + + "WinDLL('_sqlite3.dll'); p.close()") + + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst b/Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst new file mode 100644 index 000000000000..6fe5f12cc9d9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-04-08-00-36-37.bpo-34816.4Xe0id.rst @@ -0,0 +1,3 @@ +``hasattr(ctypes.windll, 'nonexistant')`` now returns ``False`` instead of raising :exc:`OSError`. + + From webhook-mailer at python.org Fri Dec 16 05:26:40 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 16 Dec 2022 10:26:40 -0000 Subject: [Python-checkins] Fix typo in introduction.rst (#100266) Message-ID: <mailman.2935.1671186401.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2cd5146f273f8e6e7cfeccb19676907294ba1ace commit: 2cd5146f273f8e6e7cfeccb19676907294ba1ace branch: main author: Mikhail Berkov <uhasker at protonmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-16T15:56:30+05:30 summary: Fix typo in introduction.rst (#100266) files: M Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 558b1c3eec60..6532a4e2cc83 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -269,7 +269,7 @@ Indices may also be negative numbers, to start counting from the right:: Note that since -0 is the same as 0, negative indices start from -1. In addition to indexing, *slicing* is also supported. While indexing is used -to obtain individual characters, *slicing* allows you to obtain substring:: +to obtain individual characters, *slicing* allows you to obtain a substring:: >>> word[0:2] # characters from position 0 (included) to 2 (excluded) 'Py' From webhook-mailer at python.org Fri Dec 16 05:32:15 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 16 Dec 2022 10:32:15 -0000 Subject: [Python-checkins] gh-99767: mark `PyTypeObject.tp_watched` as internal use only in table (#100271) Message-ID: <mailman.2936.1671186737.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e00266f39800af9cfb15cb3ad6b795c77044d798 commit: e00266f39800af9cfb15cb3ad6b795c77044d798 branch: main author: Carl Meyer <carl at oddbird.net> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-16T16:02:09+05:30 summary: gh-99767: mark `PyTypeObject.tp_watched` as internal use only in table (#100271) files: M Doc/c-api/typeobj.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c7b318b21853..016a92f11dbd 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,7 +147,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_watched` | char | | | | | | + | [:c:member:`~PyTypeObject.tp_watched`] | char | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] From webhook-mailer at python.org Fri Dec 16 05:40:16 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 16 Dec 2022 10:40:16 -0000 Subject: [Python-checkins] Document that zipfile's pwd parameter is a `bytes` object (#100209) Message-ID: <mailman.2937.1671187217.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5ee7eb9debb12914f36c5ccee92460a681516fd6 commit: 5ee7eb9debb12914f36c5ccee92460a681516fd6 branch: main author: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-16T16:10:10+05:30 summary: Document that zipfile's pwd parameter is a `bytes` object (#100209) files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 82709dbc9249..ef6934483bcd 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -273,7 +273,8 @@ ZipFile Objects Access a member of the archive as a binary file-like object. *name* can be either the name of a file within the archive or a :class:`ZipInfo` object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files. + or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files as a + :class:`bytes` object. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -325,7 +326,7 @@ ZipFile Objects must be its full name or a :class:`ZipInfo` object. Its file information is extracted as accurately as possible. *path* specifies a different directory to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files. + *pwd* is the password used for encrypted files as a :class:`bytes` object. Returns the normalized path created (a directory or new file). @@ -352,7 +353,7 @@ ZipFile Objects Extract all members from the archive to the current working directory. *path* specifies a different directory to extract to. *members* is optional and must be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files. + used for encrypted files as a :class:`bytes` object. .. warning:: @@ -377,16 +378,16 @@ ZipFile Objects .. method:: ZipFile.setpassword(pwd) - Set *pwd* as default password to extract encrypted files. + Set *pwd* (a :class:`bytes` object) as default password to extract encrypted files. .. method:: ZipFile.read(name, pwd=None) Return the bytes of the file *name* in the archive. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files and, if specified, - it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a ZipFile that uses a compression method other than + read or append. *pwd* is the password used for encrypted files as a :class:`bytes` + object and, if specified, overrides the default password set with :meth:`setpassword`. + Calling :meth:`read` on a ZipFile that uses a compression method other than :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also be raised if the corresponding compression module is not available. From webhook-mailer at python.org Fri Dec 16 08:48:09 2022 From: webhook-mailer at python.org (pablogsal) Date: Fri, 16 Dec 2022 13:48:09 -0000 Subject: [Python-checkins] gh-81057: Move the Cached Parser Dummy Name to _PyRuntimeState (#100277) Message-ID: <mailman.2938.1671198491.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0415cf895f96ae3f896f1f25f0c030a820845e13 commit: 0415cf895f96ae3f896f1f25f0c030a820845e13 branch: main author: Eric Snow <ericsnowcurrently at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-16T13:48:03Z summary: gh-81057: Move the Cached Parser Dummy Name to _PyRuntimeState (#100277) files: M Include/internal/pycore_parser.h M Include/internal/pycore_runtime_init.h M Parser/action_helpers.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2d2b56bd824c..dd51b92801ae 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -9,6 +9,8 @@ extern "C" { #endif +#include "pycore_ast.h" // struct _expr +#include "pycore_global_strings.h" // _Py_DECLARE_STR() #include "pycore_pyarena.h" // PyArena @@ -22,9 +24,22 @@ struct _parser_runtime_state { #else int _not_used; #endif + struct _expr dummy_name; }; - +_Py_DECLARE_STR(empty, "") +#define _parser_runtime_state_INIT \ + { \ + .dummy_name = { \ + .kind = Name_kind, \ + .v.Name.id = &_Py_STR(empty), \ + .v.Name.ctx = Load, \ + .lineno = 1, \ + .col_offset = 0, \ + .end_lineno = 1, \ + .end_col_offset = 0, \ + }, \ + } extern struct _mod* _PyParser_ASTFromString( const char *str, diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 6342a28f4df9..cb3fce3732c7 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_object.h" +#include "pycore_parser.h" #include "pycore_pymem_init.h" #include "pycore_obmalloc_init.h" @@ -32,6 +33,7 @@ extern "C" { until _PyInterpreterState_Enable() is called. */ \ .next_id = -1, \ }, \ + .parser = _parser_runtime_state_INIT, \ .imports = { \ .lock = { \ .mutex = NULL, \ diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index f12dad095aca..46390966892d 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -2,30 +2,12 @@ #include "pegen.h" #include "string_parser.h" - -static PyObject * -_create_dummy_identifier(Parser *p) -{ - return _PyPegen_new_identifier(p, ""); -} +#include "pycore_runtime.h" // _PyRuntime void * _PyPegen_dummy_name(Parser *p, ...) { - // XXX This leaks memory from the initial arena. - // Use a statically allocated variable instead of a pointer? - static void *cache = NULL; - - if (cache != NULL) { - return cache; - } - - PyObject *id = _create_dummy_identifier(p); - if (!id) { - return NULL; - } - cache = _PyAST_Name(id, Load, 1, 0, 1, 0, p->arena); - return cache; + return &_PyRuntime.parser.dummy_name; } /* Creates a single-element asdl_seq* that contains a */ diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c71fc0d95821..02531692a884 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -50,10 +50,6 @@ Python/getversion.c - version - Python/bootstrap_hash.c - _Py_HashSecret_Initialized - Python/pyhash.c - _Py_HashSecret - -## internal state - set lazily (*after* first init) -# XXX Move to _PyRuntimeState (i.e. tie to init/fini cycle)? -Parser/action_helpers.c _PyPegen_dummy_name cache - - ################################## ## state tied to Py_Main() From webhook-mailer at python.org Fri Dec 16 10:41:29 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 16 Dec 2022 15:41:29 -0000 Subject: [Python-checkins] Better stats for `LOAD_ATTR` and `STORE_ATTR` (GH-100295) Message-ID: <mailman.2939.1671205290.3313.python-checkins@python.org> https://github.com/python/cpython/commit/289c1126dd59e5daae59a7ee8a0aca547c1e351e commit: 289c1126dd59e5daae59a7ee8a0aca547c1e351e branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-16T15:41:23Z summary: Better stats for `LOAD_ATTR` and `STORE_ATTR` (GH-100295) * Don't attempt to specialize for LOAD_ATTR on instance if class has attribute * Improvement to LOAD_ATTR and STORE_ATTR specialization stats. files: M Include/pystats.h M Python/specialize.c diff --git a/Include/pystats.h b/Include/pystats.h index 04630c9e0f92..25ed4bddc724 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -8,7 +8,7 @@ extern "C" { #ifdef Py_STATS -#define SPECIALIZATION_FAILURE_KINDS 32 +#define SPECIALIZATION_FAILURE_KINDS 36 /* Stats for determining who is calling PyEval_EvalFrame */ #define EVAL_CALL_TOTAL 0 diff --git a/Python/specialize.c b/Python/specialize.c index d1a38450fff0..2e135be010ff 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -340,6 +340,9 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 #define SPEC_FAIL_ATTR_NOT_IN_KEYS 29 #define SPEC_FAIL_ATTR_NOT_IN_DICT 30 +#define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31 +#define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32 +#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33 /* Binary subscr and store subscr */ @@ -813,16 +816,29 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) goto success; } case BUILTIN_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); + goto fail; case PYTHON_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); + goto fail; case NON_OVERRIDING: + SPECIALIZATION_FAIL(LOAD_ATTR, + (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ? + SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR : + SPEC_FAIL_ATTR_NOT_MANAGED_DICT); + goto fail; case NON_DESCRIPTOR: + SPECIALIZATION_FAIL(LOAD_ATTR, + (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ? + SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE : + SPEC_FAIL_ATTR_NOT_MANAGED_DICT); + goto fail; case ABSENT: - break; - } - if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, - LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) - { - goto success; + if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, + LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) + { + goto success; + } } fail: STAT_INC(LOAD_ATTR, failure); @@ -901,16 +917,23 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; case BUILTIN_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); + goto fail; case PYTHON_CLASSMETHOD: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); + goto fail; case NON_OVERRIDING: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR); + goto fail; case NON_DESCRIPTOR: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); + goto fail; case ABSENT: - break; - } - if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, - STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) - { - goto success; + if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, + STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) + { + goto success; + } } fail: STAT_INC(STORE_ATTR, failure); From webhook-mailer at python.org Fri Dec 16 10:43:10 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 16 Dec 2022 15:43:10 -0000 Subject: [Python-checkins] Improve stats presentation for calls. (GH-100274) Message-ID: <mailman.2940.1671205391.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d4052d835bd6f1b648d533dab8c228b6e3944651 commit: d4052d835bd6f1b648d533dab8c228b6e3944651 branch: main author: Mark Shannon <mark at hotpy.org> committer: markshannon <mark at hotpy.org> date: 2022-12-16T15:43:04Z summary: Improve stats presentation for calls. (GH-100274) files: M Python/specialize.c diff --git a/Python/specialize.c b/Python/specialize.c index 2e135be010ff..020127e9bf3c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -394,14 +394,14 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_INSTANCE_METHOD 11 #define SPEC_FAIL_CALL_CMETHOD 12 -#define SPEC_FAIL_CALL_PYCFUNCTION 13 -#define SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS 14 -#define SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS 15 -#define SPEC_FAIL_CALL_PYCFUNCTION_NOARGS 16 +#define SPEC_FAIL_CALL_CFUNC_VARARGS 13 +#define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14 +#define SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS 15 +#define SPEC_FAIL_CALL_CFUNC_NOARGS 16 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17 -#define SPEC_FAIL_CALL_CLASS 18 +#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 18 #define SPEC_FAIL_CALL_PYTHON_CLASS 19 -#define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20 +#define SPEC_FAIL_CALL_PEP_523 20 #define SPEC_FAIL_CALL_BOUND_METHOD 21 #define SPEC_FAIL_CALL_STR 22 #define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23 @@ -409,8 +409,6 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_KWNAMES 25 #define SPEC_FAIL_CALL_METHOD_WRAPPER 26 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27 -#define SPEC_FAIL_CALL_PYFUNCTION 28 -#define SPEC_FAIL_CALL_PEP_523 29 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -1517,17 +1515,19 @@ builtin_call_fail_kind(int ml_flags) switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { case METH_VARARGS: - return SPEC_FAIL_CALL_PYCFUNCTION; + return SPEC_FAIL_CALL_CFUNC_VARARGS; case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS; + return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS; + return SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS; case METH_NOARGS: - return SPEC_FAIL_CALL_PYCFUNCTION_NOARGS; - /* This case should never happen with PyCFunctionObject -- only - PyMethodObject. See zlib.compressobj()'s methods for an example. - */ + return SPEC_FAIL_CALL_CFUNC_NOARGS; case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + return SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS; + /* These cases should be optimized, but return "other" just in case */ + case METH_O: + case METH_FASTCALL: + return SPEC_FAIL_OTHER; default: return SPEC_FAIL_CALL_BAD_CALL_FLAGS; } @@ -1698,33 +1698,18 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, static int call_fail_kind(PyObject *callable) { - if (PyCFunction_CheckExact(callable)) { - return SPEC_FAIL_CALL_PYCFUNCTION; - } - else if (PyFunction_Check(callable)) { - return SPEC_FAIL_CALL_PYFUNCTION; - } - else if (PyInstanceMethod_Check(callable)) { + assert(!PyCFunction_CheckExact(callable)); + assert(!PyFunction_Check(callable)); + assert(!PyType_Check(callable)); + assert(!Py_IS_TYPE(callable, &PyMethodDescr_Type)); + assert(!PyMethod_Check(callable)); + if (PyInstanceMethod_Check(callable)) { return SPEC_FAIL_CALL_INSTANCE_METHOD; } - else if (PyMethod_Check(callable)) { - return SPEC_FAIL_CALL_BOUND_METHOD; - } // builtin method else if (PyCMethod_Check(callable)) { return SPEC_FAIL_CALL_CMETHOD; } - else if (PyType_Check(callable)) { - if (((PyTypeObject *)callable)->tp_new == PyBaseObject_Type.tp_new) { - return SPEC_FAIL_CALL_PYTHON_CLASS; - } - else { - return SPEC_FAIL_CALL_CLASS; - } - } - else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { - return SPEC_FAIL_CALL_METHOD_DESCRIPTOR; - } else if (Py_TYPE(callable) == &PyWrapperDescr_Type) { return SPEC_FAIL_CALL_OPERATOR_WRAPPER; } @@ -1760,7 +1745,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, fail = specialize_method_descriptor((PyMethodDescrObject *)callable, instr, nargs, kwnames); } - else if (Py_TYPE(callable) == &PyMethod_Type) { + else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { fail = specialize_py_call((PyFunctionObject *)func, From webhook-mailer at python.org Fri Dec 16 13:18:41 2022 From: webhook-mailer at python.org (brandtbucher) Date: Fri, 16 Dec 2022 18:18:41 -0000 Subject: [Python-checkins] GH-90043: Handle NaNs in COMPARE_OP_FLOAT_JUMP (GH-100278) Message-ID: <mailman.2941.1671214723.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9076455d1b8132928470c11df874ca7c8580b012 commit: 9076455d1b8132928470c11df874ca7c8580b012 branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: brandtbucher <brandtbucher at gmail.com> date: 2022-12-16T10:18:31-08:00 summary: GH-90043: Handle NaNs in COMPARE_OP_FLOAT_JUMP (GH-100278) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst M Python/bytecodes.c M Python/generated_cases.c.h M Python/specialize.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst new file mode 100644 index 000000000000..9a4f896e2cf5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-15-00-50-25.gh-issue-90043.gyoKdx.rst @@ -0,0 +1,2 @@ +Handle NaNs when specializing :opcode:`COMPARE_OP` for :class:`float` +values. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c56f1d3ef9f4..8e95b7370302 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2024,13 +2024,11 @@ dummy_func( // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (dleft >= dright) + (dleft <= dright)); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); jump = sign_ish & when_to_jump_mask; @@ -2057,8 +2055,8 @@ dummy_func( assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + // 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (ileft >= iright) + (ileft <= iright)); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); jump = sign_ish & when_to_jump_mask; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 63635fbfc2f4..6d84a643b457 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2214,13 +2214,11 @@ // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright); - DEOPT_IF(isnan(dleft), COMPARE_OP); - DEOPT_IF(isnan(dright), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (dleft >= dright) + (dleft <= dright)); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); jump = sign_ish & when_to_jump_mask; @@ -2258,8 +2256,8 @@ assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - // 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask - int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright); + // 2 if <, 4 if >, 8 if ==; this matches when_to_jump_mask + int sign_ish = 1 << (2 * (ileft >= iright) + (ileft <= iright)); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); jump = sign_ish & when_to_jump_mask; diff --git a/Python/specialize.c b/Python/specialize.c index 020127e9bf3c..a1666ccc9159 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1951,15 +1951,16 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) static int compare_masks[] = { - // 1-bit: jump if less than - // 2-bit: jump if equal + // 1-bit: jump if unordered + // 2-bit: jump if less // 4-bit: jump if greater - [Py_LT] = 1 | 0 | 0, - [Py_LE] = 1 | 2 | 0, - [Py_EQ] = 0 | 2 | 0, - [Py_NE] = 1 | 0 | 4, - [Py_GT] = 0 | 0 | 4, - [Py_GE] = 0 | 2 | 4, + // 8-bit: jump if equal + [Py_LT] = 0 | 2 | 0 | 0, + [Py_LE] = 0 | 2 | 0 | 8, + [Py_EQ] = 0 | 0 | 0 | 8, + [Py_NE] = 1 | 2 | 4 | 0, + [Py_GT] = 0 | 0 | 4 | 0, + [Py_GE] = 0 | 0 | 4 | 8, }; void @@ -1980,7 +1981,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, assert(oparg <= Py_GE); int when_to_jump_mask = compare_masks[oparg]; if (next_opcode == POP_JUMP_IF_FALSE) { - when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask; + when_to_jump_mask = (1 | 2 | 4 | 8) & ~when_to_jump_mask; } if (Py_TYPE(lhs) != Py_TYPE(rhs)) { SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); @@ -2009,7 +2010,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } else { _py_set_opcode(instr, COMPARE_OP_STR_JUMP); - cache->mask = (when_to_jump_mask & 2) == 0; + cache->mask = (when_to_jump_mask & 8) == 0; goto success; } } From webhook-mailer at python.org Fri Dec 16 13:43:48 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Fri, 16 Dec 2022 18:43:48 -0000 Subject: [Python-checkins] "Compound statement" docs: Fix with-statement step indexing (#100286) Message-ID: <mailman.2942.1671216229.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f23236a92d8796ae91772adaf27c3485fda963e8 commit: f23236a92d8796ae91772adaf27c3485fda963e8 branch: main author: Frank Dana <ferdnyc at gmail.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-16T18:43:43Z summary: "Compound statement" docs: Fix with-statement step indexing (#100286) Back in commit 226e6e7d4326cf91ef37e13528eb1f62de1bb832 an item was added to the list, renumbering all the rest of the items, but the forward-reference wasn't updated to match. files: M Doc/reference/compound_stmts.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 9e09515f50d1..fe9dda933c80 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -506,7 +506,7 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo method returns without an error, then :meth:`__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would - be. See step 6 below. + be. See step 7 below. #. The suite is executed. From webhook-mailer at python.org Fri Dec 16 14:01:30 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:01:30 -0000 Subject: [Python-checkins] [3.11] "Compound statement" docs: Fix with-statement step indexing (GH-100286) (GH-100300) Message-ID: <mailman.2943.1671217291.3313.python-checkins@python.org> https://github.com/python/cpython/commit/138c2e6f3fa13444db527a272dc8eda8565a54e5 commit: 138c2e6f3fa13444db527a272dc8eda8565a54e5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:01:24-08:00 summary: [3.11] "Compound statement" docs: Fix with-statement step indexing (GH-100286) (GH-100300) Back in commit 226e6e7d4326cf91ef37e13528eb1f62de1bb832 an item was added to the list, renumbering all the rest of the items, but the forward-reference wasn't updated to match. (cherry picked from commit f23236a92d8796ae91772adaf27c3485fda963e8) Co-authored-by: Frank Dana <ferdnyc at gmail.com> files: M Doc/reference/compound_stmts.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 9e09515f50d1..fe9dda933c80 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -506,7 +506,7 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo method returns without an error, then :meth:`__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would - be. See step 6 below. + be. See step 7 below. #. The suite is executed. From webhook-mailer at python.org Fri Dec 16 14:02:46 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:02:46 -0000 Subject: [Python-checkins] "Compound statement" docs: Fix with-statement step indexing (GH-100286) Message-ID: <mailman.2944.1671217367.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0f610202553744deef4a2d639ed53a9793a87e72 commit: 0f610202553744deef4a2d639ed53a9793a87e72 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:02:41-08:00 summary: "Compound statement" docs: Fix with-statement step indexing (GH-100286) Back in commit 226e6e7d4326cf91ef37e13528eb1f62de1bb832 an item was added to the list, renumbering all the rest of the items, but the forward-reference wasn't updated to match. (cherry picked from commit f23236a92d8796ae91772adaf27c3485fda963e8) Co-authored-by: Frank Dana <ferdnyc at gmail.com> files: M Doc/reference/compound_stmts.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 93f5682f3f43..58b88c4d37f1 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -416,7 +416,7 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo method returns without an error, then :meth:`__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would - be. See step 6 below. + be. See step 7 below. #. The suite is executed. From webhook-mailer at python.org Fri Dec 16 14:05:00 2022 From: webhook-mailer at python.org (gvanrossum) Date: Fri, 16 Dec 2022 19:05:00 -0000 Subject: [Python-checkins] gh-99830: asyncio: Document returns of remove_{reader, writer} (#100302) Message-ID: <mailman.2945.1671217501.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5234e1cbea686e38392f113707db322ad8405048 commit: 5234e1cbea686e38392f113707db322ad8405048 branch: main author: Ben Darnell <ben at bendarnell.com> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-16T11:04:55-08:00 summary: gh-99830: asyncio: Document returns of remove_{reader,writer} (#100302) files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index fd47b0c24d8a..470d1aa130e4 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -932,7 +932,8 @@ Watching file descriptors .. method:: loop.remove_reader(fd) - Stop monitoring the *fd* file descriptor for read availability. + Stop monitoring the *fd* file descriptor for read availability. Returns + ``True`` if *fd* was previously being monitored for reads. .. method:: loop.add_writer(fd, callback, *args) @@ -945,7 +946,8 @@ Watching file descriptors .. method:: loop.remove_writer(fd) - Stop monitoring the *fd* file descriptor for write availability. + Stop monitoring the *fd* file descriptor for write availability. Returns + ``True`` if *fd* was previously being monitored for writes. See also :ref:`Platform Support <asyncio-platform-support>` section for some limitations of these methods. From webhook-mailer at python.org Fri Dec 16 14:14:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:14:34 -0000 Subject: [Python-checkins] gh-99830: asyncio: Document returns of remove_{reader, writer} (GH-100302) Message-ID: <mailman.2946.1671218075.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4c810f92baed7562e7f2da0c70e2edac2e89bf75 commit: 4c810f92baed7562e7f2da0c70e2edac2e89bf75 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:14:28-08:00 summary: gh-99830: asyncio: Document returns of remove_{reader,writer} (GH-100302) (cherry picked from commit 5234e1cbea686e38392f113707db322ad8405048) Co-authored-by: Ben Darnell <ben at bendarnell.com> files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 28b7a900583b..d32e848a8413 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -915,7 +915,8 @@ Watching file descriptors .. method:: loop.remove_reader(fd) - Stop monitoring the *fd* file descriptor for read availability. + Stop monitoring the *fd* file descriptor for read availability. Returns + ``True`` if *fd* was previously being monitored for reads. .. method:: loop.add_writer(fd, callback, *args) @@ -928,7 +929,8 @@ Watching file descriptors .. method:: loop.remove_writer(fd) - Stop monitoring the *fd* file descriptor for write availability. + Stop monitoring the *fd* file descriptor for write availability. Returns + ``True`` if *fd* was previously being monitored for writes. See also :ref:`Platform Support <asyncio-platform-support>` section for some limitations of these methods. From webhook-mailer at python.org Fri Dec 16 14:15:18 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 16 Dec 2022 19:15:18 -0000 Subject: [Python-checkins] gh-99830: asyncio: Document returns of remove_{reader, writer} (GH-100302) Message-ID: <mailman.2947.1671218119.3313.python-checkins@python.org> https://github.com/python/cpython/commit/958be792a7fd50971a048662a377f4ae101652bd commit: 958be792a7fd50971a048662a377f4ae101652bd branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T11:15:12-08:00 summary: gh-99830: asyncio: Document returns of remove_{reader,writer} (GH-100302) (cherry picked from commit 5234e1cbea686e38392f113707db322ad8405048) Co-authored-by: Ben Darnell <ben at bendarnell.com> files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index a23be64ec39a..7b4c3c7682fe 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -858,7 +858,8 @@ Watching file descriptors .. method:: loop.remove_reader(fd) - Stop monitoring the *fd* file descriptor for read availability. + Stop monitoring the *fd* file descriptor for read availability. Returns + ``True`` if *fd* was previously being monitored for reads. .. method:: loop.add_writer(fd, callback, *args) @@ -871,7 +872,8 @@ Watching file descriptors .. method:: loop.remove_writer(fd) - Stop monitoring the *fd* file descriptor for write availability. + Stop monitoring the *fd* file descriptor for write availability. Returns + ``True`` if *fd* was previously being monitored for writes. See also :ref:`Platform Support <asyncio-platform-support>` section for some limitations of these methods. From webhook-mailer at python.org Fri Dec 16 15:30:53 2022 From: webhook-mailer at python.org (ethanfurman) Date: Fri, 16 Dec 2022 20:30:53 -0000 Subject: [Python-checkins] gh-100039: enhance __signature__ to work with str and callables (GH-100168) Message-ID: <mailman.2948.1671222654.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a5a7cea202d34ca699d9594d428ba527ef7ff2ce commit: a5a7cea202d34ca699d9594d428ba527ef7ff2ce branch: main author: Ethan Furman <ethan at stoneleaf.us> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-16T12:30:47-08:00 summary: gh-100039: enhance __signature__ to work with str and callables (GH-100168) Callables should be either class- or static-methods. Enum now uses the classmethod version to greatly improve the help given for enums and flags. files: A Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst M Lib/enum.py M Lib/inspect.py M Lib/test/test_enum.py M Lib/test/test_inspect.py diff --git a/Lib/enum.py b/Lib/enum.py index a0cad066dc23..21f63881d78d 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -685,7 +685,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k 'member order does not match _order_:\n %r\n %r' % (enum_class._member_names_, _order_) ) - + # return enum_class def __bool__(cls): @@ -1083,6 +1083,13 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ + @classmethod + def __signature__(cls): + if cls._member_names_: + return '(*values)' + else: + return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' + def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' diff --git a/Lib/inspect.py b/Lib/inspect.py index e165937e448a..e92c355220fd 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2443,10 +2443,18 @@ def _signature_from_callable(obj, *, pass else: if sig is not None: + # since __text_signature__ is not writable on classes, __signature__ + # may contain text (or be a callable that returns text); + # if so, convert it + o_sig = sig + if not isinstance(sig, (Signature, str)) and callable(sig): + sig = sig() + if isinstance(sig, str): + sig = _signature_fromstr(sigcls, obj, sig) if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' - 'attribute'.format(sig)) + 'attribute'.format(o_sig)) return sig try: diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index d876c8e5fb77..1e653e94f6b5 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4259,7 +4259,7 @@ class TestEnumTypeSubclassing(unittest.TestCase): Help on class Color in module %s: class Color(enum.Enum) - | Color(value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None) + | Color(*values) | | Method resolution order: | Color @@ -4315,7 +4315,7 @@ class Color(enum.Enum) Help on class Color in module %s: class Color(enum.Enum) - | Color(value, names=None, *values, module=None, qualname=None, type=None, start=1) + | Color(*values) | | Method resolution order: | Color @@ -4462,6 +4462,27 @@ def test_inspect_classify_class_attrs(self): if failed: self.fail("result does not equal expected, see print above") + def test_inspect_signatures(self): + from inspect import signature, Signature, Parameter + self.assertEqual( + signature(Enum), + Signature([ + Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None), + ]), + ) + self.assertEqual( + signature(enum.FlagBoundary), + Signature([ + Parameter('values', Parameter.VAR_POSITIONAL), + ]), + ) + def test_test_simple_enum(self): @_simple_enum(Enum) class SimpleColor: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 2b7977b1648f..0f8217ba3b75 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3614,6 +3614,38 @@ def foo(): pass self.assertEqual(signature_func(foo), inspect.Signature()) self.assertEqual(inspect.get_annotations(foo), {}) + def test_signature_as_str(self): + self.maxDiff = None + class S: + __signature__ = '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + + def test_signature_as_callable(self): + # __signature__ should be either a staticmethod or a bound classmethod + class S: + @classmethod + def __signature__(cls): + return '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + + class S: + @staticmethod + def __signature__(): + return '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): diff --git a/Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst b/Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst new file mode 100644 index 000000000000..d56643f7e871 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-10-20-52-28.gh-issue-100039.zDqjT4.rst @@ -0,0 +1 @@ +Improve signatures for enums and flags. From webhook-mailer at python.org Fri Dec 16 15:36:19 2022 From: webhook-mailer at python.org (ethanfurman) Date: Fri, 16 Dec 2022 20:36:19 -0000 Subject: [Python-checkins] gh-99540: Constant hash for _PyNone_Type to aid reproducibility (GH-99541) Message-ID: <mailman.2949.1671222980.3313.python-checkins@python.org> https://github.com/python/cpython/commit/432117cd1f59c76d97da2eaff55a7d758301dbc7 commit: 432117cd1f59c76d97da2eaff55a7d758301dbc7 branch: main author: yonillasky <yonillasky at users.noreply.github.com> committer: ethanfurman <ethan at stoneleaf.us> date: 2022-12-16T12:36:13-08:00 summary: gh-99540: Constant hash for _PyNone_Type to aid reproducibility (GH-99541) Needed for ASLR builds of Python. files: A Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst M Objects/object.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst new file mode 100644 index 000000000000..ae043f3aba55 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-10-20-00-13.gh-issue-99540.ZZZHeP.rst @@ -0,0 +1 @@ +``None`` now hashes to a constant value. This is not a requirements change. diff --git a/Objects/object.c b/Objects/object.c index 687bd36d2b4a..028b0edc9111 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1641,6 +1641,11 @@ none_bool(PyObject *v) return 0; } +static Py_hash_t none_hash(PyObject *v) +{ + return 0xFCA86420; +} + static PyNumberMethods none_as_number = { 0, /* nb_add */ 0, /* nb_subtract */ @@ -1692,7 +1697,7 @@ PyTypeObject _PyNone_Type = { &none_as_number, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ - 0, /*tp_hash */ + (hashfunc)none_hash,/*tp_hash */ 0, /*tp_call */ 0, /*tp_str */ 0, /*tp_getattro */ From webhook-mailer at python.org Fri Dec 16 19:14:33 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 00:14:33 -0000 Subject: [Python-checkins] gh-78707: deprecate passing >1 argument to `PurePath.[is_]relative_to()` (GH-94469) Message-ID: <mailman.2950.1671236074.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5a991da32961ef5780996d58b8816d5f2085f540 commit: 5a991da32961ef5780996d58b8816d5f2085f540 branch: main author: Barney Gale <barney.gale at gmail.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T16:14:27-08:00 summary: gh-78707: deprecate passing >1 argument to `PurePath.[is_]relative_to()` (GH-94469) This brings `relative_to()` and `is_relative_to()` more in line with other pathlib methods like `rename()` and `symlink_to()`. Resolves #78707. files: A Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst M Doc/library/pathlib.rst M Lib/pathlib.py M Lib/test/test_pathlib.py diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 6537637f33c7..47687400c14e 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -490,7 +490,7 @@ Pure paths provide the following methods and properties: True -.. method:: PurePath.is_relative_to(*other) +.. method:: PurePath.is_relative_to(other) Return whether or not this path is relative to the *other* path. @@ -502,6 +502,10 @@ Pure paths provide the following methods and properties: .. versionadded:: 3.9 + .. deprecated-removed:: 3.12 3.14 + + Passing additional arguments is deprecated; if supplied, they are joined + with *other*. .. method:: PurePath.is_reserved() @@ -564,7 +568,7 @@ Pure paths provide the following methods and properties: True -.. method:: PurePath.relative_to(*other, walk_up=False) +.. method:: PurePath.relative_to(other, walk_up=False) Compute a version of this path relative to the path represented by *other*. If it's impossible, :exc:`ValueError` is raised:: @@ -581,7 +585,7 @@ Pure paths provide the following methods and properties: raise ValueError(error_message.format(str(self), str(formatted))) ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other is absolute. -When *walk_up* is False (the default), the path must start with *other*. + When *walk_up* is False (the default), the path must start with *other*. When the argument is True, ``..`` entries may be added to form the relative path. In all other cases, such as the paths referencing different drives, :exc:`ValueError` is raised.:: @@ -605,6 +609,10 @@ When *walk_up* is False (the default), the path must start with *other*. .. versionadded:: 3.12 The *walk_up* argument (old behavior is the same as ``walk_up=False``). + .. deprecated-removed:: 3.12 3.14 + + Passing additional positional arguments is deprecated; if supplied, + they are joined with *other*. .. method:: PurePath.with_name(name) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f31eb3010368..7890fdade420 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -624,7 +624,7 @@ def with_suffix(self, suffix): return self._from_parsed_parts(self._drv, self._root, self._parts[:-1] + [name]) - def relative_to(self, *other, walk_up=False): + def relative_to(self, other, /, *_deprecated, walk_up=False): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not related to the other path), raise ValueError. @@ -632,10 +632,14 @@ def relative_to(self, *other, walk_up=False): The *walk_up* parameter controls whether `..` may be used to resolve the path. """ - if not other: - raise TypeError("need at least one argument") + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, + remove=(3, 14)) path_cls = type(self) - other = path_cls(*other) + other = path_cls(other, *_deprecated) for step, path in enumerate([other] + list(other.parents)): if self.is_relative_to(path): break @@ -646,12 +650,16 @@ def relative_to(self, *other, walk_up=False): parts = ('..',) * step + self.parts[len(path.parts):] return path_cls(*parts) - def is_relative_to(self, *other): + def is_relative_to(self, other, /, *_deprecated): """Return True if the path is relative to another path or False. """ - if not other: - raise TypeError("need at least one argument") - other = type(self)(*other) + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", + msg, remove=(3, 14)) + other = type(self)(other, *_deprecated) return other == self or other in self.parents @property diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 1d01d3cbd91d..fa6ea0ac63d8 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -654,8 +654,9 @@ def test_relative_to_common(self): self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) # With several args. - self.assertEqual(p.relative_to('a', 'b'), P()) - self.assertEqual(p.relative_to('a', 'b', walk_up=True), P()) + with self.assertWarns(DeprecationWarning): + p.relative_to('a', 'b') + p.relative_to('a', 'b', walk_up=True) # Unrelated paths. self.assertRaises(ValueError, p.relative_to, P('c')) self.assertRaises(ValueError, p.relative_to, P('a/b/c')) @@ -706,7 +707,8 @@ def test_is_relative_to_common(self): self.assertTrue(p.is_relative_to(P('a/b'))) self.assertTrue(p.is_relative_to('a/b')) # With several args. - self.assertTrue(p.is_relative_to('a', 'b')) + with self.assertWarns(DeprecationWarning): + p.is_relative_to('a', 'b') # Unrelated paths. self.assertFalse(p.is_relative_to(P('c'))) self.assertFalse(p.is_relative_to(P('a/b/c'))) diff --git a/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst b/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst new file mode 100644 index 000000000000..c490a3c1bd7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst @@ -0,0 +1,3 @@ +Deprecate passing more than one positional argument to +:meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. From webhook-mailer at python.org Fri Dec 16 23:46:04 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 17 Dec 2022 04:46:04 -0000 Subject: [Python-checkins] GH-98831: Add DECREF_INPUTS(), expanding to DECREF() each stack input (#100205) Message-ID: <mailman.2951.1671252365.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9cdd2fa63b7549d00830bccf19a34e9d69d0b15e commit: 9cdd2fa63b7549d00830bccf19a34e9d69d0b15e branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-16T20:45:55-08:00 summary: GH-98831: Add DECREF_INPUTS(), expanding to DECREF() each stack input (#100205) The presence of this macro indicates that a particular instruction may be considered for conversion to a register-based format (see https://github.com/faster-cpython/ideas/issues/485). An invariant (currently unchecked) is that `DEOPT_IF()` may only occur *before* `DECREF_INPUTS()`, and `ERROR_IF()` may only occur *after* it. One reason not to check this is that there are a few places where we insert *two* `DECREF_INPUTS()` calls, in different branches of the code. The invariant checking would have to be able to do some flow control analysis to understand this. Note that many instructions, especially specialized ones, can't be converted to use this macro straightforwardly. This is because the generator currently only generates plain `Py_DECREF(variable)` statements, and cannot generate things like `_Py_DECREF_SPECIALIZED()` let alone deal with `_PyList_AppendTakeRef()`. files: M Python/bytecodes.c M Tools/cases_generator/generate_cases.py diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8e95b7370302..b29e16e080e9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -161,7 +161,7 @@ dummy_func( super(LOAD_CONST__LOAD_FAST) = LOAD_CONST + LOAD_FAST; inst(POP_TOP, (value --)) { - Py_DECREF(value); + DECREF_INPUTS(); } inst(PUSH_NULL, (-- res)) { @@ -172,19 +172,19 @@ dummy_func( inst(UNARY_POSITIVE, (value -- res)) { res = PyNumber_Positive(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(UNARY_NOT, (value -- res)) { int err = PyObject_IsTrue(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(err < 0, error); if (err == 0) { res = Py_True; @@ -197,7 +197,7 @@ dummy_func( inst(UNARY_INVERT, (value -- res)) { res = PyNumber_Invert(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -351,8 +351,7 @@ dummy_func( STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -438,8 +437,7 @@ dummy_func( ERROR_IF(true, error); } Py_INCREF(res); // Do this before DECREF'ing dict, sub - Py_DECREF(dict); - Py_DECREF(sub); + DECREF_INPUTS(); } inst(BINARY_SUBSCR_GETITEM, (unused/1, type_version/2, func_version/1, container, sub -- unused)) { @@ -500,9 +498,7 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -538,8 +534,7 @@ dummy_func( inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -550,11 +545,11 @@ dummy_func( if (hook == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(true, error); } res = PyObject_CallOneArg(hook, value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); Py_DECREF(res); } @@ -625,12 +620,12 @@ dummy_func( "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + DECREF_INPUTS(); ERROR_IF(true, error); } iter = (*getter)(obj); - Py_DECREF(obj); + DECREF_INPUTS(); ERROR_IF(iter == NULL, error); if (Py_TYPE(iter)->tp_as_async == NULL || diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 2dfc76f2560e..85a7c6098e0b 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -209,7 +209,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None cache_offset += ceffect.size assert cache_offset == self.cache_offset + cache_adjust - # Write the body, substituting a goto for ERROR_IF() + # Write the body, substituting a goto for ERROR_IF() and other stuff assert dedent <= 0 extra = " " * -dedent for line in self.block_text: @@ -232,6 +232,10 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None ) else: out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") + elif m := re.match(r"(\s*)DECREF_INPUTS;\s*$", line): + space = m.group(1) + for ieff in self.input_effects: + out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") else: out.write_raw(extra + line) From webhook-mailer at python.org Sat Dec 17 01:35:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 06:35:24 -0000 Subject: [Python-checkins] [3.11] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100230) Message-ID: <mailman.2952.1671258926.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dd323afea81523482e14c57eaebd6d3c7d61c212 commit: dd323afea81523482e14c57eaebd6d3c7d61c212 branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:04:54+05:30 summary: [3.11] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100230) (cherry picked from commit c450c8c9ed6e420025f39d0e4850a79f8160cdcd) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst A Modules/_testclinic.c A Modules/clinic/_testclinic.c.h M Lib/test/test_clinic.py M Modules/Setup.stdlib.in M Tools/scripts/generate_stdlib_module_names.py M configure M configure.ac M setup.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4aa9691a4829..86c4f94ac028 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3,7 +3,7 @@ # Licensed to the PSF under a contributor agreement. from test import support, test_tools -from test.support import os_helper +from test.support import import_helper, os_helper from unittest import TestCase import collections import inspect @@ -820,5 +820,397 @@ def test_external(self): self.assertEqual(new_mtime_ns, old_mtime_ns) +ac_tester = import_helper.import_module('_testclinic') + + +class ClinicFunctionalTest(unittest.TestCase): + locals().update((name, getattr(ac_tester, name)) + for name in dir(ac_tester) if name.startswith('test_')) + + def test_objects_converter(self): + with self.assertRaises(TypeError): + ac_tester.objects_converter() + self.assertEqual(ac_tester.objects_converter(1, 2), (1, 2)) + self.assertEqual(ac_tester.objects_converter([], 'whatever class'), ([], 'whatever class')) + self.assertEqual(ac_tester.objects_converter(1), (1, None)) + + def test_bytes_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.bytes_object_converter(1) + self.assertEqual(ac_tester.bytes_object_converter(b'BytesObject'), (b'BytesObject',)) + + def test_byte_array_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.byte_array_object_converter(1) + byte_arr = bytearray(b'ByteArrayObject') + self.assertEqual(ac_tester.byte_array_object_converter(byte_arr), (byte_arr,)) + + def test_unicode_converter(self): + with self.assertRaises(TypeError): + ac_tester.unicode_converter(1) + self.assertEqual(ac_tester.unicode_converter('unicode'), ('unicode',)) + + def test_bool_converter(self): + with self.assertRaises(TypeError): + ac_tester.bool_converter(False, False, 'not a int') + self.assertEqual(ac_tester.bool_converter(), (True, True, True)) + self.assertEqual(ac_tester.bool_converter('', [], 5), (False, False, True)) + self.assertEqual(ac_tester.bool_converter(('not empty',), {1: 2}, 0), (True, True, False)) + + def test_char_converter(self): + with self.assertRaises(TypeError): + ac_tester.char_converter(1) + with self.assertRaises(TypeError): + ac_tester.char_converter(b'ab') + chars = [b'A', b'\a', b'\b', b'\t', b'\n', b'\v', b'\f', b'\r', b'"', b"'", b'?', b'\\', b'\000', b'\377'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(), expected) + chars = [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'a', b'b', b'c', b'd'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(*chars), expected) + + def test_unsigned_char_converter(self): + from _testcapi import UCHAR_MAX + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(UCHAR_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(0, UCHAR_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_char_converter([]) + self.assertEqual(ac_tester.unsigned_char_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_short_converter(self): + from _testcapi import SHRT_MIN, SHRT_MAX + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.short_converter([]) + self.assertEqual(ac_tester.short_converter(-1234), (-1234,)) + self.assertEqual(ac_tester.short_converter(4321), (4321,)) + + def test_unsigned_short_converter(self): + from _testcapi import USHRT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_short_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(USHRT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(0, USHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_short_converter([]) + self.assertEqual(ac_tester.unsigned_short_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_int_converter(self): + from _testcapi import INT_MIN, INT_MAX + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.int_converter(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.int_converter([]) + self.assertEqual(ac_tester.int_converter(), (12, 34, 45)) + self.assertEqual(ac_tester.int_converter(1, 2, '3'), (1, 2, ord('3'))) + + def test_unsigned_int_converter(self): + from _testcapi import UINT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_int_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(UINT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(0, UINT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_int_converter([]) + self.assertEqual(ac_tester.unsigned_int_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_converter(self): + from _testcapi import LONG_MIN, LONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_converter([]) + self.assertEqual(ac_tester.long_converter(), (12,)) + self.assertEqual(ac_tester.long_converter(-1234), (-1234,)) + + def test_unsigned_long_converter(self): + from _testcapi import ULONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(ULONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(0, ULONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_long_converter(self): + from _testcapi import LLONG_MIN, LLONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_long_converter([]) + self.assertEqual(ac_tester.long_long_converter(), (12,)) + self.assertEqual(ac_tester.long_long_converter(-1234), (-1234,)) + + def test_unsigned_long_long_converter(self): + from _testcapi import ULLONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(ULLONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(0, ULLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_py_ssize_t_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.py_ssize_t_converter([]) + self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None), (1, 2, 56)) + + def test_slice_index_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(TypeError): + ac_tester.slice_index_converter([]) + self.assertEqual(ac_tester.slice_index_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.slice_index_converter(1, 2, None), (1, 2, 56)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MAX, PY_SSIZE_T_MAX + 1, PY_SSIZE_T_MAX + 1234), + (PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MIN, PY_SSIZE_T_MIN - 1, PY_SSIZE_T_MIN - 1234), + (PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, PY_SSIZE_T_MIN)) + + def test_size_t_converter(self): + with self.assertRaises(ValueError): + ac_tester.size_t_converter(-1) + with self.assertRaises(TypeError): + ac_tester.size_t_converter([]) + self.assertEqual(ac_tester.size_t_converter(), (12,)) + + def test_float_converter(self): + with self.assertRaises(TypeError): + ac_tester.float_converter([]) + self.assertEqual(ac_tester.float_converter(), (12.5,)) + self.assertEqual(ac_tester.float_converter(-0.5), (-0.5,)) + + def test_double_converter(self): + with self.assertRaises(TypeError): + ac_tester.double_converter([]) + self.assertEqual(ac_tester.double_converter(), (12.5,)) + self.assertEqual(ac_tester.double_converter(-0.5), (-0.5,)) + + def test_py_complex_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_complex_converter([]) + self.assertEqual(ac_tester.py_complex_converter(complex(1, 2)), (complex(1, 2),)) + self.assertEqual(ac_tester.py_complex_converter(complex('-1-2j')), (complex('-1-2j'),)) + self.assertEqual(ac_tester.py_complex_converter(-0.5), (-0.5,)) + self.assertEqual(ac_tester.py_complex_converter(10), (10,)) + + def test_str_converter(self): + with self.assertRaises(TypeError): + ac_tester.str_converter(1) + with self.assertRaises(TypeError): + ac_tester.str_converter('a', 'b', 'c') + with self.assertRaises(ValueError): + ac_tester.str_converter('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter('a', b'b', 'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + + def test_py_buffer_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_buffer_converter('a', 'b') + self.assertEqual(ac_tester.py_buffer_converter('abc', bytearray([1, 2, 3])), (b'abc', b'\x01\x02\x03')) + + def test_keywords(self): + self.assertEqual(ac_tester.keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.keywords(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords(a=1, b=2), (1, 2)) + + def test_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly(1, 2) + self.assertEqual(ac_tester.keywords_kwonly(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords_kwonly(a=1, b=2), (1, 2)) + + def test_keywords_opt(self): + self.assertEqual(ac_tester.keywords_opt(1), (1, None, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, 3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_keywords_opt_kwonly(self): + self.assertEqual(ac_tester.keywords_opt_kwonly(1), (1, None, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_opt_kwonly(1, 2, 3) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, c=3), (1, None, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_keywords_kwonly_opt(self): + self.assertEqual(ac_tester.keywords_kwonly_opt(1), (1, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly_opt(1, 2) + self.assertEqual(ac_tester.keywords_kwonly_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.posonly_keywords(1, b=2), (1, 2)) + + def test_posonly_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(a=1, b=2) + self.assertEqual(ac_tester.posonly_kwonly(1, b=2), (1, 2)) + + def test_posonly_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(a=1, b=2, c=3, d=4) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_opt_keywords_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(a=1, b=2, c=3, d=4) + + def test_posonly_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1, 2) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(a=1, b=2, c=3, d=4) + + def test_posonly_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_keywords_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, b=2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_keywords_opt_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2), (1, 2, None, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, 4) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4, e=5), (1, 2, 3, 4, 5)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_opt_keywords_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, 4) + + def test_keyword_only_parameter(self): + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter() + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter(1) + self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst new file mode 100644 index 000000000000..dc86e1d70f12 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst @@ -0,0 +1 @@ +Add functional test for Argument Clinic. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 2730030a1565..2009a4d2ea66 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,6 +169,7 @@ @MODULE__TESTBUFFER_TRUE at _testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE at _testinternalcapi _testinternalcapi.c @MODULE__TESTCAPI_TRUE at _testcapi _testcapimodule.c + at MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. *shared* diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c new file mode 100644 index 000000000000..c9858e964457 --- /dev/null +++ b/Modules/_testclinic.c @@ -0,0 +1,952 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +/* Always enable assertions */ +#undef NDEBUG + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" + +#include "clinic/_testclinic.c.h" + + +/* Pack arguments to a tuple, implicitly increase all the arguments' refcount. + * NULL arguments will be replaced to Py_None. */ +static PyObject * +pack_arguments_newref(int argc, ...) +{ + assert(!PyErr_Occurred()); + PyObject *tuple = PyTuple_New(argc); + if (!tuple) { + return NULL; + } + + va_list vargs; + va_start(vargs, argc); + for (int i = 0; i < argc; i++) { + PyObject *arg = va_arg(vargs, PyObject *); + if (arg) { + if (_PyObject_IsFreed(arg)) { + PyErr_Format(PyExc_AssertionError, + "argument %d at %p is freed or corrupted!", + i, arg); + va_end(vargs); + Py_DECREF(tuple); + return NULL; + } + } + else { + arg = Py_None; + } + PyTuple_SET_ITEM(tuple, i, Py_NewRef(arg)); + } + va_end(vargs); + return tuple; +} + +/* Pack arguments to a tuple. + * `wrapper` is function which converts primitive type to PyObject. + * `arg_type` is type that arguments should be converted to before wrapped. */ +#define RETURN_PACKED_ARGS(argc, wrapper, arg_type, ...) do { \ + assert(!PyErr_Occurred()); \ + arg_type in[argc] = {__VA_ARGS__}; \ + PyObject *out[argc] = {NULL,}; \ + for (int _i = 0; _i < argc; _i++) { \ + out[_i] = wrapper(in[_i]); \ + assert(out[_i] || PyErr_Occurred()); \ + if (!out[_i]) { \ + for (int _j = 0; _j < _i; _j++) { \ + Py_DECREF(out[_j]); \ + } \ + return NULL; \ + } \ + } \ + PyObject *tuple = PyTuple_New(argc); \ + if (!tuple) { \ + for (int _i = 0; _i < argc; _i++) { \ + Py_DECREF(out[_i]); \ + } \ + return NULL; \ + } \ + for (int _i = 0; _i < argc; _i++) { \ + PyTuple_SET_ITEM(tuple, _i, out[_i]); \ + } \ + return tuple; \ + } while (0) + + +/*[clinic input] +module _testclinic +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d4981b80d6efdb12]*/ + + +/*[clinic input] +test_empty_function + +[clinic start generated code]*/ + +static PyObject * +test_empty_function_impl(PyObject *module) +/*[clinic end generated code: output=0f8aeb3ddced55cb input=0dd7048651ad4ae4]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +objects_converter + + a: object + b: object = NULL + / + +[clinic start generated code]*/ + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=3f9c9415ec86c695 input=1533b1bd94187de4]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +bytes_object_converter + + a: PyBytesObject + / + +[clinic start generated code]*/ + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a) +/*[clinic end generated code: output=7732da869d74b784 input=94211751e7996236]*/ +{ + if (!PyBytes_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyBytesObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +byte_array_object_converter + + a: PyByteArrayObject + / + +[clinic start generated code]*/ + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a) +/*[clinic end generated code: output=51f15c76f302b1f7 input=b04d253db51c6f56]*/ +{ + if (!PyByteArray_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyByteArrayObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +unicode_converter + + a: unicode + / + +[clinic start generated code]*/ + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=1b4a4adbb6ac6e34 input=de7b5adbf07435ba]*/ +{ + if (!PyUnicode_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a unicode object"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +bool_converter + + a: bool = True + b: bool(accept={object}) = True + c: bool(accept={int}) = True + / + +[clinic start generated code]*/ + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=17005b0c29afd590 input=7f6537705b2f32f4]*/ +{ + PyObject *obj_a = a ? Py_True : Py_False; + PyObject *obj_b = b ? Py_True : Py_False; + PyObject *obj_c = c ? Py_True : Py_False; + return pack_arguments_newref(3, obj_a, obj_b, obj_c); +} + + +/*[clinic input] +char_converter + + a: char = b'A' + b: char = b'\a' + c: char = b'\b' + d: char = b'\t' + e: char = b'\n' + f: char = b'\v' + g: char = b'\f' + h: char = b'\r' + i: char = b'"' + j: char = b"'" + k: char = b'?' + l: char = b'\\' + m: char = b'\000' + n: char = b'\377' + / + +[clinic start generated code]*/ + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n) +/*[clinic end generated code: output=f929dbd2e55a9871 input=b601bc5bc7fe85e3]*/ +{ + RETURN_PACKED_ARGS(14, PyLong_FromUnsignedLong, unsigned char, + a, b, c, d, e, f, g, h, i, j, k, l, m, n); +} + + +/*[clinic input] +unsigned_char_converter + + a: unsigned_char = 12 + b: unsigned_char(bitwise=False) = 34 + c: unsigned_char(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c) +/*[clinic end generated code: output=490af3b39ce0b199 input=e859502fbe0b3185]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned char, a, b, c); +} + + +/*[clinic input] +short_converter + + a: short = 12 + / + +[clinic start generated code]*/ + +static PyObject * +short_converter_impl(PyObject *module, short a) +/*[clinic end generated code: output=1ebb7ddb64248988 input=b4e2309a66f650ae]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_short_converter + + a: unsigned_short = 12 + b: unsigned_short(bitwise=False) = 34 + c: unsigned_short(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c) +/*[clinic end generated code: output=5f92cc72fc8707a7 input=9d15cd11e741d0c6]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +int_converter + + a: int = 12 + b: int(accept={int}) = 34 + c: int(accept={str}) = 45 + / + +[clinic start generated code]*/ + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=8e56b59be7d0c306 input=a1dbc6344853db7a]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromLong, long, a, b, c); +} + + +/*[clinic input] +unsigned_int_converter + + a: unsigned_int = 12 + b: unsigned_int(bitwise=False) = 34 + c: unsigned_int(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c) +/*[clinic end generated code: output=399a57a05c494cc7 input=8427ed9a3f96272d]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_converter + + a: long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_converter_impl(PyObject *module, long a) +/*[clinic end generated code: output=9663d936a652707a input=84ad0ef28f24bd85]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_long_converter + + a: unsigned_long = 12 + b: unsigned_long(bitwise=False) = 34 + c: unsigned_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c) +/*[clinic end generated code: output=120b82ea9ebd93a8 input=440dd6f1817f5d91]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_long_converter + + a: long_long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_long_converter_impl(PyObject *module, long long a) +/*[clinic end generated code: output=5fb5f2220770c3e1 input=730fcb3eecf4d993]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLongLong, long long, a); +} + + +/*[clinic input] +unsigned_long_long_converter + + a: unsigned_long_long = 12 + b: unsigned_long_long(bitwise=False) = 34 + c: unsigned_long_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c) +/*[clinic end generated code: output=65b7273e63501762 input=300737b0bdb230e9]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLongLong, unsigned long long, + a, b, c); +} + + +/*[clinic input] +py_ssize_t_converter + + a: Py_ssize_t = 12 + b: Py_ssize_t(accept={int}) = 34 + c: Py_ssize_t(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=ce252143e0ed0372 input=76d0f342e9317a1f]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +slice_index_converter + + a: slice_index = 12 + b: slice_index(accept={int}) = 34 + c: slice_index(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=923c6cac77666a6b input=64f99f3f83265e47]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +size_t_converter + + a: size_t = 12 + / + +[clinic start generated code]*/ + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a) +/*[clinic end generated code: output=412b5b7334ab444d input=83ae7d9171fbf208]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromSize_t, size_t, a); +} + + +/*[clinic input] +float_converter + + a: float = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +float_converter_impl(PyObject *module, float a) +/*[clinic end generated code: output=1c98f64f2cf1d55c input=a625b59ad68047d8]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +double_converter + + a: double = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +double_converter_impl(PyObject *module, double a) +/*[clinic end generated code: output=a4e8532d284d035d input=098df188f24e7c62]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +py_complex_converter + + a: Py_complex + / + +[clinic start generated code]*/ + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a) +/*[clinic end generated code: output=9e6ca2eb53b14846 input=e9148a8ca1dbf195]*/ +{ + RETURN_PACKED_ARGS(1, PyComplex_FromCComplex, Py_complex, a); +} + + +/*[clinic input] +str_converter + + a: str = "a" + b: str(accept={robuffer}) = "b" + c: str(accept={robuffer, str}, zeroes=True) = "c" + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_t c_length) +/*[clinic end generated code: output=475bea40548c8cd6 input=bff2656c92ee25de]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + +static PyObject * +bytes_from_buffer(Py_buffer *buf) +{ + PyObject *bytes_obj = PyBytes_FromStringAndSize(NULL, buf->len); + if (!bytes_obj) { + return NULL; + } + void *bytes_obj_buf = ((PyBytesObject *)bytes_obj)->ob_sval; + if (PyBuffer_ToContiguous(bytes_obj_buf, buf, buf->len, 'C') < 0) { + Py_DECREF(bytes_obj); + return NULL; + } + return bytes_obj; +} + +/*[clinic input] +py_buffer_converter + + a: Py_buffer(accept={str, buffer, NoneType}) + b: Py_buffer(accept={rwbuffer}) + / + +[clinic start generated code]*/ + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b) +/*[clinic end generated code: output=52fb13311e3d6d03 input=775de727de5c7421]*/ +{ + RETURN_PACKED_ARGS(2, bytes_from_buffer, Py_buffer *, a, b); +} + + +/*[clinic input] +keywords + + a: object + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=850aaed53e26729e input=f44b89e718c1a93b]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_kwonly + + a: object + * + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=a45c48241da584dc input=1f08e39c3312b015]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_opt + + a: object + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) +/*[clinic end generated code: output=25e4b67d91c76a66 input=b0ba0e4f04904556]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +keywords_opt_kwonly + + a: object + b: object = None + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=6aa5b655a6e9aeb0 input=f79da689d6c51076]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keywords_kwonly_opt + + a: object + * + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=707f78eb0f55c2b1 input=e0fa1a0e46dca791]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords + + a: object + / + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=6ac88f4a5f0bfc8d input=fde0a2f79fe82b06]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_kwonly + + a: object + / + * + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=483e6790d3482185 input=78b3712768da9a19]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_keywords_kwonly + + a: object + / + b: object + * + c: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=2fae573e8cc3fad8 input=a1ad5d2295eb803c]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords_opt + + a: object + / + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=f5eb66241bcf68fb input=51c10de2a120e279]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_keywords_opt + + a: object + b: object = None + / + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=d54a30e549296ffd input=f408a1de7dfaf31f]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_kwonly_opt + + a: object + / + * + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=a20503fe36b4fd62 input=3494253975272f52]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_kwonly_opt + + a: object + b: object = None + / + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=64f3204a3a0413b6 input=d17516581e478412]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_keywords_kwonly_opt + + a: object + / + b: object + * + c: object + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=dbd7e7ddd6257fa0 input=33529f29e97e5adb]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_keywords_opt_kwonly_opt + + a: object + / + b: object + c: object = None + * + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e) +/*[clinic end generated code: output=775d12ae44653045 input=4d4cc62f11441301]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_opt_keywords_opt_kwonly_opt + + a: object + b: object = None + / + c: object = None + * + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=40c6dc422591eade input=3964960a68622431]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keyword_only_parameter + + * + a: object + +[clinic start generated code]*/ + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=c454b6ce98232787 input=8d2868b8d0b27bdb]*/ +{ + return pack_arguments_newref(1, a); +} + + +static PyMethodDef tester_methods[] = { + TEST_EMPTY_FUNCTION_METHODDEF + OBJECTS_CONVERTER_METHODDEF + BYTES_OBJECT_CONVERTER_METHODDEF + BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF + UNICODE_CONVERTER_METHODDEF + BOOL_CONVERTER_METHODDEF + CHAR_CONVERTER_METHODDEF + UNSIGNED_CHAR_CONVERTER_METHODDEF + SHORT_CONVERTER_METHODDEF + UNSIGNED_SHORT_CONVERTER_METHODDEF + INT_CONVERTER_METHODDEF + UNSIGNED_INT_CONVERTER_METHODDEF + LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_CONVERTER_METHODDEF + LONG_LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_LONG_CONVERTER_METHODDEF + PY_SSIZE_T_CONVERTER_METHODDEF + SLICE_INDEX_CONVERTER_METHODDEF + SIZE_T_CONVERTER_METHODDEF + FLOAT_CONVERTER_METHODDEF + DOUBLE_CONVERTER_METHODDEF + PY_COMPLEX_CONVERTER_METHODDEF + STR_CONVERTER_METHODDEF + PY_BUFFER_CONVERTER_METHODDEF + KEYWORDS_METHODDEF + KEYWORDS_KWONLY_METHODDEF + KEYWORDS_OPT_METHODDEF + KEYWORDS_OPT_KWONLY_METHODDEF + KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_METHODDEF + POSONLY_KWONLY_METHODDEF + POSONLY_KEYWORDS_KWONLY_METHODDEF + POSONLY_KEYWORDS_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_METHODDEF + POSONLY_KWONLY_OPT_METHODDEF + POSONLY_OPT_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + KEYWORD_ONLY_PARAMETER_METHODDEF + {NULL, NULL} +}; + +static struct PyModuleDef _testclinic_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testclinic", + .m_size = 0, + .m_methods = tester_methods, +}; + +PyMODINIT_FUNC +PyInit__testclinic(void) +{ + return PyModule_Create(&_testclinic_module); +} + +#undef RETURN_PACKED_ARGS diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h new file mode 100644 index 000000000000..d363dd3f358e --- /dev/null +++ b/Modules/clinic/_testclinic.c.h @@ -0,0 +1,1917 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(test_empty_function__doc__, +"test_empty_function($module, /)\n" +"--\n" +"\n"); + +#define TEST_EMPTY_FUNCTION_METHODDEF \ + {"test_empty_function", (PyCFunction)test_empty_function, METH_NOARGS, test_empty_function__doc__}, + +static PyObject * +test_empty_function_impl(PyObject *module); + +static PyObject * +test_empty_function(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_empty_function_impl(module); +} + +PyDoc_STRVAR(objects_converter__doc__, +"objects_converter($module, a, b=<unrepresentable>, /)\n" +"--\n" +"\n"); + +#define OBJECTS_CONVERTER_METHODDEF \ + {"objects_converter", _PyCFunction_CAST(objects_converter), METH_FASTCALL, objects_converter__doc__}, + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +objects_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b = NULL; + + if (!_PyArg_CheckPositional("objects_converter", nargs, 1, 2)) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + return_value = objects_converter_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_object_converter__doc__, +"bytes_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTES_OBJECT_CONVERTER_METHODDEF \ + {"bytes_object_converter", (PyCFunction)bytes_object_converter, METH_O, bytes_object_converter__doc__}, + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a); + +static PyObject * +bytes_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyBytesObject *a; + + if (!PyBytes_Check(arg)) { + _PyArg_BadArgument("bytes_object_converter", "argument", "bytes", arg); + goto exit; + } + a = (PyBytesObject *)arg; + return_value = bytes_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(byte_array_object_converter__doc__, +"byte_array_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF \ + {"byte_array_object_converter", (PyCFunction)byte_array_object_converter, METH_O, byte_array_object_converter__doc__}, + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a); + +static PyObject * +byte_array_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyByteArrayObject *a; + + if (!PyByteArray_Check(arg)) { + _PyArg_BadArgument("byte_array_object_converter", "argument", "bytearray", arg); + goto exit; + } + a = (PyByteArrayObject *)arg; + return_value = byte_array_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_converter__doc__, +"unicode_converter($module, a, /)\n" +"--\n" +"\n"); + +#define UNICODE_CONVERTER_METHODDEF \ + {"unicode_converter", (PyCFunction)unicode_converter, METH_O, unicode_converter__doc__}, + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a); + +static PyObject * +unicode_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("unicode_converter", "argument", "str", arg); + goto exit; + } + if (PyUnicode_READY(arg) == -1) { + goto exit; + } + a = arg; + return_value = unicode_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(bool_converter__doc__, +"bool_converter($module, a=True, b=True, c=True, /)\n" +"--\n" +"\n"); + +#define BOOL_CONVERTER_METHODDEF \ + {"bool_converter", _PyCFunction_CAST(bool_converter), METH_FASTCALL, bool_converter__doc__}, + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +bool_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 1; + int b = 1; + int c = 1; + + if (!_PyArg_CheckPositional("bool_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyObject_IsTrue(args[0]); + if (a < 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = PyObject_IsTrue(args[1]); + if (b < 0) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = _PyLong_AsInt(args[2]); + if (c == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = bool_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(char_converter__doc__, +"char_converter($module, a=b\'A\', b=b\'\\x07\', c=b\'\\x08\', d=b\'\\t\', e=b\'\\n\',\n" +" f=b\'\\x0b\', g=b\'\\x0c\', h=b\'\\r\', i=b\'\"\', j=b\"\'\", k=b\'?\',\n" +" l=b\'\\\\\', m=b\'\\x00\', n=b\'\\xff\', /)\n" +"--\n" +"\n"); + +#define CHAR_CONVERTER_METHODDEF \ + {"char_converter", _PyCFunction_CAST(char_converter), METH_FASTCALL, char_converter__doc__}, + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n); + +static PyObject * +char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char a = 'A'; + char b = '\x07'; + char c = '\x08'; + char d = '\t'; + char e = '\n'; + char f = '\x0b'; + char g = '\x0c'; + char h = '\r'; + char i = '"'; + char j = '\''; + char k = '?'; + char l = '\\'; + char m = '\x00'; + char n = '\xff'; + + if (!_PyArg_CheckPositional("char_converter", nargs, 0, 14)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + a = PyBytes_AS_STRING(args[0])[0]; + } + else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + a = PyByteArray_AS_STRING(args[0])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 1", "a byte string of length 1", args[0]); + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + b = PyBytes_AS_STRING(args[1])[0]; + } + else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + b = PyByteArray_AS_STRING(args[1])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 2", "a byte string of length 1", args[1]); + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + c = PyBytes_AS_STRING(args[2])[0]; + } + else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + c = PyByteArray_AS_STRING(args[2])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 3", "a byte string of length 1", args[2]); + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + d = PyBytes_AS_STRING(args[3])[0]; + } + else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + d = PyByteArray_AS_STRING(args[3])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 4", "a byte string of length 1", args[3]); + goto exit; + } + if (nargs < 5) { + goto skip_optional; + } + if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + e = PyBytes_AS_STRING(args[4])[0]; + } + else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + e = PyByteArray_AS_STRING(args[4])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 5", "a byte string of length 1", args[4]); + goto exit; + } + if (nargs < 6) { + goto skip_optional; + } + if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + f = PyBytes_AS_STRING(args[5])[0]; + } + else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + f = PyByteArray_AS_STRING(args[5])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 6", "a byte string of length 1", args[5]); + goto exit; + } + if (nargs < 7) { + goto skip_optional; + } + if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + g = PyBytes_AS_STRING(args[6])[0]; + } + else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + g = PyByteArray_AS_STRING(args[6])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 7", "a byte string of length 1", args[6]); + goto exit; + } + if (nargs < 8) { + goto skip_optional; + } + if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + h = PyBytes_AS_STRING(args[7])[0]; + } + else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + h = PyByteArray_AS_STRING(args[7])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 8", "a byte string of length 1", args[7]); + goto exit; + } + if (nargs < 9) { + goto skip_optional; + } + if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + i = PyBytes_AS_STRING(args[8])[0]; + } + else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + i = PyByteArray_AS_STRING(args[8])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 9", "a byte string of length 1", args[8]); + goto exit; + } + if (nargs < 10) { + goto skip_optional; + } + if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + j = PyBytes_AS_STRING(args[9])[0]; + } + else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + j = PyByteArray_AS_STRING(args[9])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 10", "a byte string of length 1", args[9]); + goto exit; + } + if (nargs < 11) { + goto skip_optional; + } + if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + k = PyBytes_AS_STRING(args[10])[0]; + } + else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + k = PyByteArray_AS_STRING(args[10])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 11", "a byte string of length 1", args[10]); + goto exit; + } + if (nargs < 12) { + goto skip_optional; + } + if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + l = PyBytes_AS_STRING(args[11])[0]; + } + else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + l = PyByteArray_AS_STRING(args[11])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 12", "a byte string of length 1", args[11]); + goto exit; + } + if (nargs < 13) { + goto skip_optional; + } + if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + m = PyBytes_AS_STRING(args[12])[0]; + } + else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + m = PyByteArray_AS_STRING(args[12])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 13", "a byte string of length 1", args[12]); + goto exit; + } + if (nargs < 14) { + goto skip_optional; + } + if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + n = PyBytes_AS_STRING(args[13])[0]; + } + else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + n = PyByteArray_AS_STRING(args[13])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 14", "a byte string of length 1", args[13]); + goto exit; + } +skip_optional: + return_value = char_converter_impl(module, a, b, c, d, e, f, g, h, i, j, k, l, m, n); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_char_converter__doc__, +"unsigned_char_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_CHAR_CONVERTER_METHODDEF \ + {"unsigned_char_converter", _PyCFunction_CAST(unsigned_char_converter), METH_FASTCALL, unsigned_char_converter__doc__}, + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c); + +static PyObject * +unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned char a = 12; + unsigned char b = 34; + unsigned char c = 56; + + if (!_PyArg_CheckPositional("unsigned_char_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + a = (unsigned char) ival; + } + } + if (nargs < 2) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + b = (unsigned char) ival; + } + } + if (nargs < 3) { + goto skip_optional; + } + { + unsigned long ival = PyLong_AsUnsignedLongMask(args[2]); + if (ival == (unsigned long)-1 && PyErr_Occurred()) { + goto exit; + } + else { + c = (unsigned char) ival; + } + } +skip_optional: + return_value = unsigned_char_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(short_converter__doc__, +"short_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SHORT_CONVERTER_METHODDEF \ + {"short_converter", _PyCFunction_CAST(short_converter), METH_FASTCALL, short_converter__doc__}, + +static PyObject * +short_converter_impl(PyObject *module, short a); + +static PyObject * +short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + short a = 12; + + if (!_PyArg_CheckPositional("short_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + a = (short) ival; + } + } +skip_optional: + return_value = short_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_short_converter__doc__, +"unsigned_short_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_SHORT_CONVERTER_METHODDEF \ + {"unsigned_short_converter", _PyCFunction_CAST(unsigned_short_converter), METH_FASTCALL, unsigned_short_converter__doc__}, + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c); + +static PyObject * +unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned short a = 12; + unsigned short b = 34; + unsigned short c = 56; + + if (!_PyArg_CheckPositional("unsigned_short_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned short)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned short)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_short_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(int_converter__doc__, +"int_converter($module, a=12, b=34, c=45, /)\n" +"--\n" +"\n"); + +#define INT_CONVERTER_METHODDEF \ + {"int_converter", _PyCFunction_CAST(int_converter), METH_FASTCALL, int_converter__doc__}, + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 12; + int b = 34; + int c = 45; + + if (!_PyArg_CheckPositional("int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = _PyLong_AsInt(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = _PyLong_AsInt(args[1]); + if (b == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyUnicode_Check(args[2])) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + if (PyUnicode_READY(args[2])) { + goto exit; + } + if (PyUnicode_GET_LENGTH(args[2]) != 1) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + c = PyUnicode_READ_CHAR(args[2], 0); +skip_optional: + return_value = int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_int_converter__doc__, +"unsigned_int_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_INT_CONVERTER_METHODDEF \ + {"unsigned_int_converter", _PyCFunction_CAST(unsigned_int_converter), METH_FASTCALL, unsigned_int_converter__doc__}, + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c); + +static PyObject * +unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int a = 12; + unsigned int b = 34; + unsigned int c = 56; + + if (!_PyArg_CheckPositional("unsigned_int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned int)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned int)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_converter__doc__, +"long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_CONVERTER_METHODDEF \ + {"long_converter", _PyCFunction_CAST(long_converter), METH_FASTCALL, long_converter__doc__}, + +static PyObject * +long_converter_impl(PyObject *module, long a); + +static PyObject * +long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long a = 12; + + if (!_PyArg_CheckPositional("long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_converter__doc__, +"unsigned_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_converter", _PyCFunction_CAST(unsigned_long_converter), METH_FASTCALL, unsigned_long_converter__doc__}, + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c); + +static PyObject * +unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long a = 12; + unsigned long b = 34; + unsigned long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongMask(args[2]); +skip_optional: + return_value = unsigned_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_long_converter__doc__, +"long_long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_LONG_CONVERTER_METHODDEF \ + {"long_long_converter", _PyCFunction_CAST(long_long_converter), METH_FASTCALL, long_long_converter__doc__}, + +static PyObject * +long_long_converter_impl(PyObject *module, long long a); + +static PyObject * +long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long long a = 12; + + if (!_PyArg_CheckPositional("long_long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLongLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_long_converter__doc__, +"unsigned_long_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_long_converter", _PyCFunction_CAST(unsigned_long_long_converter), METH_FASTCALL, unsigned_long_long_converter__doc__}, + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c); + +static PyObject * +unsigned_long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long long a = 12; + unsigned long long b = 34; + unsigned long long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongLongMask(args[2]); +skip_optional: + return_value = unsigned_long_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_ssize_t_converter__doc__, +"py_ssize_t_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define PY_SSIZE_T_CONVERTER_METHODDEF \ + {"py_ssize_t_converter", _PyCFunction_CAST(py_ssize_t_converter), METH_FASTCALL, py_ssize_t_converter__doc__}, + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("py_ssize_t_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + a = ival; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + b = ival; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = py_ssize_t_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(slice_index_converter__doc__, +"slice_index_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define SLICE_INDEX_CONVERTER_METHODDEF \ + {"slice_index_converter", _PyCFunction_CAST(slice_index_converter), METH_FASTCALL, slice_index_converter__doc__}, + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +slice_index_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("slice_index_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = slice_index_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(size_t_converter__doc__, +"size_t_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SIZE_T_CONVERTER_METHODDEF \ + {"size_t_converter", _PyCFunction_CAST(size_t_converter), METH_FASTCALL, size_t_converter__doc__}, + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a); + +static PyObject * +size_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + size_t a = 12; + + if (!_PyArg_CheckPositional("size_t_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_Size_t_Converter(args[0], &a)) { + goto exit; + } +skip_optional: + return_value = size_t_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(float_converter__doc__, +"float_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define FLOAT_CONVERTER_METHODDEF \ + {"float_converter", _PyCFunction_CAST(float_converter), METH_FASTCALL, float_converter__doc__}, + +static PyObject * +float_converter_impl(PyObject *module, float a); + +static PyObject * +float_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float a = 12.5; + + if (!_PyArg_CheckPositional("float_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + a = (float) PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = float_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(double_converter__doc__, +"double_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define DOUBLE_CONVERTER_METHODDEF \ + {"double_converter", _PyCFunction_CAST(double_converter), METH_FASTCALL, double_converter__doc__}, + +static PyObject * +double_converter_impl(PyObject *module, double a); + +static PyObject * +double_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double a = 12.5; + + if (!_PyArg_CheckPositional("double_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = PyFloat_AS_DOUBLE(args[0]); + } + else + { + a = PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = double_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_complex_converter__doc__, +"py_complex_converter($module, a, /)\n" +"--\n" +"\n"); + +#define PY_COMPLEX_CONVERTER_METHODDEF \ + {"py_complex_converter", (PyCFunction)py_complex_converter, METH_O, py_complex_converter__doc__}, + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a); + +static PyObject * +py_complex_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_complex a; + + a = PyComplex_AsCComplex(arg); + if (PyErr_Occurred()) { + goto exit; + } + return_value = py_complex_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(str_converter__doc__, +"str_converter($module, a=\'a\', b=\'b\', c=\'c\', /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_METHODDEF \ + {"str_converter", _PyCFunction_CAST(str_converter), METH_FASTCALL, str_converter__doc__}, + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_t c_length); + +static PyObject * +str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + const char *a = "a"; + const char *b = "b"; + const char *c = "c"; + Py_ssize_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "|sys#:str_converter", + &a, &b, &c, &c_length)) { + goto exit; + } + return_value = str_converter_impl(module, a, b, c, c_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_buffer_converter__doc__, +"py_buffer_converter($module, a, b, /)\n" +"--\n" +"\n"); + +#define PY_BUFFER_CONVERTER_METHODDEF \ + {"py_buffer_converter", _PyCFunction_CAST(py_buffer_converter), METH_FASTCALL, py_buffer_converter__doc__}, + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b); + +static PyObject * +py_buffer_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer a = {NULL, NULL}; + Py_buffer b = {NULL, NULL}; + + if (!_PyArg_ParseStack(args, nargs, "z*w*:py_buffer_converter", + &a, &b)) { + goto exit; + } + return_value = py_buffer_converter_impl(module, &a, &b); + +exit: + /* Cleanup for a */ + if (a.obj) { + PyBuffer_Release(&a); + } + /* Cleanup for b */ + if (b.obj) { + PyBuffer_Release(&b); + } + + return return_value; +} + +PyDoc_STRVAR(keywords__doc__, +"keywords($module, /, a, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_METHODDEF \ + {"keywords", _PyCFunction_CAST(keywords), METH_FASTCALL|METH_KEYWORDS, keywords__doc__}, + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly__doc__, +"keywords_kwonly($module, /, a, *, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_METHODDEF \ + {"keywords_kwonly", _PyCFunction_CAST(keywords_kwonly), METH_FASTCALL|METH_KEYWORDS, keywords_kwonly__doc__}, + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt__doc__, +"keywords_opt($module, /, a, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_METHODDEF \ + {"keywords_opt", _PyCFunction_CAST(keywords_opt), METH_FASTCALL|METH_KEYWORDS, keywords_opt__doc__}, + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); + +static PyObject * +keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = keywords_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt_kwonly__doc__, +"keywords_opt_kwonly($module, /, a, b=None, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_KWONLY_METHODDEF \ + {"keywords_opt_kwonly", _PyCFunction_CAST(keywords_opt_kwonly), METH_FASTCALL|METH_KEYWORDS, keywords_opt_kwonly__doc__}, + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt_kwonly", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = keywords_opt_kwonly_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly_opt__doc__, +"keywords_kwonly_opt($module, /, a, *, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_OPT_METHODDEF \ + {"keywords_kwonly_opt", _PyCFunction_CAST(keywords_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, keywords_kwonly_opt__doc__}, + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[2]; +skip_optional_kwonly: + return_value = keywords_kwonly_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords__doc__, +"posonly_keywords($module, a, /, b)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_METHODDEF \ + {"posonly_keywords", _PyCFunction_CAST(posonly_keywords), METH_FASTCALL|METH_KEYWORDS, posonly_keywords__doc__}, + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly__doc__, +"posonly_kwonly($module, a, /, *, b)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_METHODDEF \ + {"posonly_kwonly", _PyCFunction_CAST(posonly_kwonly), METH_FASTCALL|METH_KEYWORDS, posonly_kwonly__doc__}, + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly__doc__, +"posonly_keywords_kwonly($module, a, /, b, *, c)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_METHODDEF \ + {"posonly_keywords_kwonly", _PyCFunction_CAST(posonly_keywords_kwonly), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly__doc__}, + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly", 0}; + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = posonly_keywords_kwonly_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt__doc__, +"posonly_keywords_opt($module, a, /, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_METHODDEF \ + {"posonly_keywords_opt", _PyCFunction_CAST(posonly_keywords_opt), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt__doc__}, + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt__doc__, +"posonly_opt_keywords_opt($module, a, b=None, /, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_METHODDEF \ + {"posonly_opt_keywords_opt", _PyCFunction_CAST(posonly_opt_keywords_opt), METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_opt_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly_opt__doc__, +"posonly_kwonly_opt($module, a, /, *, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_OPT_METHODDEF \ + {"posonly_kwonly_opt", _PyCFunction_CAST(posonly_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_kwonly_opt__doc__}, + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_kwonly_opt__doc__, +"posonly_opt_kwonly_opt($module, a, b=None, /, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_kwonly_opt", _PyCFunction_CAST(posonly_opt_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly_opt__doc__, +"posonly_keywords_kwonly_opt($module, a, /, b, *, c, d=None, e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_kwonly_opt", _PyCFunction_CAST(posonly_keywords_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt_kwonly_opt__doc__, +"posonly_keywords_opt_kwonly_opt($module, a, /, b, c=None, *, d=None,\n" +" e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_opt_kwonly_opt", _PyCFunction_CAST(posonly_keywords_opt_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e); + +static PyObject * +posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_opt_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt_kwonly_opt__doc__, +"posonly_opt_keywords_opt_kwonly_opt($module, a, b=None, /, c=None, *,\n" +" d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_keywords_opt_kwonly_opt", _PyCFunction_CAST(posonly_opt_keywords_opt_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d); + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_keywords_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keyword_only_parameter__doc__, +"keyword_only_parameter($module, /, *, a)\n" +"--\n" +"\n"); + +#define KEYWORD_ONLY_PARAMETER_METHODDEF \ + {"keyword_only_parameter", _PyCFunction_CAST(keyword_only_parameter), METH_FASTCALL|METH_KEYWORDS, keyword_only_parameter__doc__}, + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a); + +static PyObject * +keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keyword_only_parameter", 0}; + PyObject *argsbuf[1]; + PyObject *a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = keyword_only_parameter_impl(module, a); + +exit: + return return_value; +} +/*[clinic end generated code: output=83f439d06635a2e9 input=a9049054013a1b77]*/ diff --git a/Tools/scripts/generate_stdlib_module_names.py b/Tools/scripts/generate_stdlib_module_names.py index fe1e429ebce1..02d2710ca4ad 100644 --- a/Tools/scripts/generate_stdlib_module_names.py +++ b/Tools/scripts/generate_stdlib_module_names.py @@ -27,6 +27,7 @@ '_ctypes_test', '_testbuffer', '_testcapi', + '_testclinic', '_testconsole', '_testimportmultiple', '_testinternalcapi', diff --git a/configure b/configure index 9567ac3a62c8..9e76287725b9 100755 --- a/configure +++ b/configure @@ -640,6 +640,8 @@ MODULE__TESTBUFFER_FALSE MODULE__TESTBUFFER_TRUE MODULE__TESTINTERNALCAPI_FALSE MODULE__TESTINTERNALCAPI_TRUE +MODULE__TESTCLINIC_FALSE +MODULE__TESTCLINIC_TRUE MODULE__TESTCAPI_FALSE MODULE__TESTCAPI_TRUE MODULE__HASHLIB_FALSE @@ -25495,6 +25497,40 @@ fi $as_echo "$py_cv_module__testcapi" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testclinic" >&5 +$as_echo_n "checking for stdlib extension module _testclinic... " >&6; } + if test "$py_cv_module__testclinic" != "n/a"; then : + + if test "$TEST_MODULES" = yes; then : + if true; then : + py_cv_module__testclinic=yes +else + py_cv_module__testclinic=missing +fi +else + py_cv_module__testclinic=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTCLINIC_STATE=$py_cv_module__testclinic$as_nl" + if test "x$py_cv_module__testclinic" = xyes; then : + + + + +fi + if test "$py_cv_module__testclinic" = yes; then + MODULE__TESTCLINIC_TRUE= + MODULE__TESTCLINIC_FALSE='#' +else + MODULE__TESTCLINIC_TRUE='#' + MODULE__TESTCLINIC_FALSE= +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testclinic" >&5 +$as_echo "$py_cv_module__testclinic" >&6; } + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testinternalcapi" >&5 $as_echo_n "checking for stdlib extension module _testinternalcapi... " >&6; } if test "$py_cv_module__testinternalcapi" != "n/a"; then : @@ -26160,6 +26196,10 @@ if test -z "${MODULE__TESTCAPI_TRUE}" && test -z "${MODULE__TESTCAPI_FALSE}"; th as_fn_error $? "conditional \"MODULE__TESTCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTCLINIC_TRUE}" && test -z "${MODULE__TESTCLINIC_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTCLINIC\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__TESTINTERNALCAPI_TRUE}" && test -z "${MODULE__TESTINTERNALCAPI_FALSE}"; then as_fn_error $? "conditional \"MODULE__TESTINTERNALCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 90008bcae17f..c62a565eb68f 100644 --- a/configure.ac +++ b/configure.ac @@ -7090,6 +7090,7 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) diff --git a/setup.py b/setup.py index 15d0d4576a47..4f122b62e0e7 100644 --- a/setup.py +++ b/setup.py @@ -1012,6 +1012,9 @@ def detect_test_extensions(self): # Python C API test module self.addext(Extension('_testcapi', ['_testcapimodule.c'])) + # Python Argument Clinc functional test module + self.addext(Extension('_testclinic', ['_testclinic.c'])) + # Python Internal C API test module self.addext(Extension('_testinternalcapi', ['_testinternalcapi.c'])) From webhook-mailer at python.org Sat Dec 17 01:36:11 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 06:36:11 -0000 Subject: [Python-checkins] [3.10] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100232) Message-ID: <mailman.2953.1671258972.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3144aca3dae51d3d88cc58fd8e9a04d6d6601ced commit: 3144aca3dae51d3d88cc58fd8e9a04d6d6601ced branch: 3.10 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:06:06+05:30 summary: [3.10] gh-96002: Add functional test for Argument Clinic (GH-96178) (#100232) (cherry picked from commit c450c8c9ed6e420025f39d0e4850a79f8160cdcd) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst A Modules/_testclinic.c A Modules/clinic/_testclinic.c.h M Lib/test/test_clinic.py M Tools/scripts/generate_stdlib_module_names.py M setup.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4aa9691a4829..86c4f94ac028 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3,7 +3,7 @@ # Licensed to the PSF under a contributor agreement. from test import support, test_tools -from test.support import os_helper +from test.support import import_helper, os_helper from unittest import TestCase import collections import inspect @@ -820,5 +820,397 @@ def test_external(self): self.assertEqual(new_mtime_ns, old_mtime_ns) +ac_tester = import_helper.import_module('_testclinic') + + +class ClinicFunctionalTest(unittest.TestCase): + locals().update((name, getattr(ac_tester, name)) + for name in dir(ac_tester) if name.startswith('test_')) + + def test_objects_converter(self): + with self.assertRaises(TypeError): + ac_tester.objects_converter() + self.assertEqual(ac_tester.objects_converter(1, 2), (1, 2)) + self.assertEqual(ac_tester.objects_converter([], 'whatever class'), ([], 'whatever class')) + self.assertEqual(ac_tester.objects_converter(1), (1, None)) + + def test_bytes_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.bytes_object_converter(1) + self.assertEqual(ac_tester.bytes_object_converter(b'BytesObject'), (b'BytesObject',)) + + def test_byte_array_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.byte_array_object_converter(1) + byte_arr = bytearray(b'ByteArrayObject') + self.assertEqual(ac_tester.byte_array_object_converter(byte_arr), (byte_arr,)) + + def test_unicode_converter(self): + with self.assertRaises(TypeError): + ac_tester.unicode_converter(1) + self.assertEqual(ac_tester.unicode_converter('unicode'), ('unicode',)) + + def test_bool_converter(self): + with self.assertRaises(TypeError): + ac_tester.bool_converter(False, False, 'not a int') + self.assertEqual(ac_tester.bool_converter(), (True, True, True)) + self.assertEqual(ac_tester.bool_converter('', [], 5), (False, False, True)) + self.assertEqual(ac_tester.bool_converter(('not empty',), {1: 2}, 0), (True, True, False)) + + def test_char_converter(self): + with self.assertRaises(TypeError): + ac_tester.char_converter(1) + with self.assertRaises(TypeError): + ac_tester.char_converter(b'ab') + chars = [b'A', b'\a', b'\b', b'\t', b'\n', b'\v', b'\f', b'\r', b'"', b"'", b'?', b'\\', b'\000', b'\377'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(), expected) + chars = [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'a', b'b', b'c', b'd'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(*chars), expected) + + def test_unsigned_char_converter(self): + from _testcapi import UCHAR_MAX + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(UCHAR_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(0, UCHAR_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_char_converter([]) + self.assertEqual(ac_tester.unsigned_char_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_short_converter(self): + from _testcapi import SHRT_MIN, SHRT_MAX + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.short_converter([]) + self.assertEqual(ac_tester.short_converter(-1234), (-1234,)) + self.assertEqual(ac_tester.short_converter(4321), (4321,)) + + def test_unsigned_short_converter(self): + from _testcapi import USHRT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_short_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(USHRT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(0, USHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_short_converter([]) + self.assertEqual(ac_tester.unsigned_short_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_int_converter(self): + from _testcapi import INT_MIN, INT_MAX + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.int_converter(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.int_converter([]) + self.assertEqual(ac_tester.int_converter(), (12, 34, 45)) + self.assertEqual(ac_tester.int_converter(1, 2, '3'), (1, 2, ord('3'))) + + def test_unsigned_int_converter(self): + from _testcapi import UINT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_int_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(UINT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(0, UINT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_int_converter([]) + self.assertEqual(ac_tester.unsigned_int_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_converter(self): + from _testcapi import LONG_MIN, LONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_converter([]) + self.assertEqual(ac_tester.long_converter(), (12,)) + self.assertEqual(ac_tester.long_converter(-1234), (-1234,)) + + def test_unsigned_long_converter(self): + from _testcapi import ULONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(ULONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(0, ULONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_long_converter(self): + from _testcapi import LLONG_MIN, LLONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_long_converter([]) + self.assertEqual(ac_tester.long_long_converter(), (12,)) + self.assertEqual(ac_tester.long_long_converter(-1234), (-1234,)) + + def test_unsigned_long_long_converter(self): + from _testcapi import ULLONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(ULLONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(0, ULLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_py_ssize_t_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.py_ssize_t_converter([]) + self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None), (1, 2, 56)) + + def test_slice_index_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(TypeError): + ac_tester.slice_index_converter([]) + self.assertEqual(ac_tester.slice_index_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.slice_index_converter(1, 2, None), (1, 2, 56)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MAX, PY_SSIZE_T_MAX + 1, PY_SSIZE_T_MAX + 1234), + (PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MIN, PY_SSIZE_T_MIN - 1, PY_SSIZE_T_MIN - 1234), + (PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, PY_SSIZE_T_MIN)) + + def test_size_t_converter(self): + with self.assertRaises(ValueError): + ac_tester.size_t_converter(-1) + with self.assertRaises(TypeError): + ac_tester.size_t_converter([]) + self.assertEqual(ac_tester.size_t_converter(), (12,)) + + def test_float_converter(self): + with self.assertRaises(TypeError): + ac_tester.float_converter([]) + self.assertEqual(ac_tester.float_converter(), (12.5,)) + self.assertEqual(ac_tester.float_converter(-0.5), (-0.5,)) + + def test_double_converter(self): + with self.assertRaises(TypeError): + ac_tester.double_converter([]) + self.assertEqual(ac_tester.double_converter(), (12.5,)) + self.assertEqual(ac_tester.double_converter(-0.5), (-0.5,)) + + def test_py_complex_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_complex_converter([]) + self.assertEqual(ac_tester.py_complex_converter(complex(1, 2)), (complex(1, 2),)) + self.assertEqual(ac_tester.py_complex_converter(complex('-1-2j')), (complex('-1-2j'),)) + self.assertEqual(ac_tester.py_complex_converter(-0.5), (-0.5,)) + self.assertEqual(ac_tester.py_complex_converter(10), (10,)) + + def test_str_converter(self): + with self.assertRaises(TypeError): + ac_tester.str_converter(1) + with self.assertRaises(TypeError): + ac_tester.str_converter('a', 'b', 'c') + with self.assertRaises(ValueError): + ac_tester.str_converter('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter('a', b'b', 'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + + def test_py_buffer_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_buffer_converter('a', 'b') + self.assertEqual(ac_tester.py_buffer_converter('abc', bytearray([1, 2, 3])), (b'abc', b'\x01\x02\x03')) + + def test_keywords(self): + self.assertEqual(ac_tester.keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.keywords(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords(a=1, b=2), (1, 2)) + + def test_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly(1, 2) + self.assertEqual(ac_tester.keywords_kwonly(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords_kwonly(a=1, b=2), (1, 2)) + + def test_keywords_opt(self): + self.assertEqual(ac_tester.keywords_opt(1), (1, None, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, 3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_keywords_opt_kwonly(self): + self.assertEqual(ac_tester.keywords_opt_kwonly(1), (1, None, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_opt_kwonly(1, 2, 3) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, c=3), (1, None, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_keywords_kwonly_opt(self): + self.assertEqual(ac_tester.keywords_kwonly_opt(1), (1, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly_opt(1, 2) + self.assertEqual(ac_tester.keywords_kwonly_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.posonly_keywords(1, b=2), (1, 2)) + + def test_posonly_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(a=1, b=2) + self.assertEqual(ac_tester.posonly_kwonly(1, b=2), (1, 2)) + + def test_posonly_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(a=1, b=2, c=3, d=4) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_opt_keywords_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(a=1, b=2, c=3, d=4) + + def test_posonly_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1, 2) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(a=1, b=2, c=3, d=4) + + def test_posonly_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_keywords_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, b=2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_keywords_opt_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2), (1, 2, None, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, 4) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4, e=5), (1, 2, 3, 4, 5)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_opt_keywords_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, 4) + + def test_keyword_only_parameter(self): + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter() + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter(1) + self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst new file mode 100644 index 000000000000..dc86e1d70f12 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-08-22-15-49-14.gh-issue-96002.4UE9UE.rst @@ -0,0 +1 @@ +Add functional test for Argument Clinic. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c new file mode 100644 index 000000000000..63427ae6373e --- /dev/null +++ b/Modules/_testclinic.c @@ -0,0 +1,952 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +/* Always enable assertions */ +#undef NDEBUG + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" + +#include "clinic/_testclinic.c.h" + + +/* Pack arguments to a tuple, implicitly increase all the arguments' refcount. + * NULL arguments will be replaced to Py_None. */ +static PyObject * +pack_arguments_newref(int argc, ...) +{ + assert(!PyErr_Occurred()); + PyObject *tuple = PyTuple_New(argc); + if (!tuple) { + return NULL; + } + + va_list vargs; + va_start(vargs, argc); + for (int i = 0; i < argc; i++) { + PyObject *arg = va_arg(vargs, PyObject *); + if (arg) { + if (_PyObject_IsFreed(arg)) { + PyErr_Format(PyExc_AssertionError, + "argument %d at %p is freed or corrupted!", + i, arg); + va_end(vargs); + Py_DECREF(tuple); + return NULL; + } + } + else { + arg = Py_None; + } + PyTuple_SET_ITEM(tuple, i, Py_NewRef(arg)); + } + va_end(vargs); + return tuple; +} + +/* Pack arguments to a tuple. + * `wrapper` is function which converts primitive type to PyObject. + * `arg_type` is type that arguments should be converted to before wrapped. */ +#define RETURN_PACKED_ARGS(argc, wrapper, arg_type, ...) do { \ + assert(!PyErr_Occurred()); \ + arg_type in[argc] = {__VA_ARGS__}; \ + PyObject *out[argc] = {NULL,}; \ + for (int _i = 0; _i < argc; _i++) { \ + out[_i] = wrapper(in[_i]); \ + assert(out[_i] || PyErr_Occurred()); \ + if (!out[_i]) { \ + for (int _j = 0; _j < _i; _j++) { \ + Py_DECREF(out[_j]); \ + } \ + return NULL; \ + } \ + } \ + PyObject *tuple = PyTuple_New(argc); \ + if (!tuple) { \ + for (int _i = 0; _i < argc; _i++) { \ + Py_DECREF(out[_i]); \ + } \ + return NULL; \ + } \ + for (int _i = 0; _i < argc; _i++) { \ + PyTuple_SET_ITEM(tuple, _i, out[_i]); \ + } \ + return tuple; \ + } while (0) + + +/*[clinic input] +module _testclinic +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d4981b80d6efdb12]*/ + + +/*[clinic input] +test_empty_function + +[clinic start generated code]*/ + +static PyObject * +test_empty_function_impl(PyObject *module) +/*[clinic end generated code: output=0f8aeb3ddced55cb input=0dd7048651ad4ae4]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +objects_converter + + a: object + b: object = NULL + / + +[clinic start generated code]*/ + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=3f9c9415ec86c695 input=1533b1bd94187de4]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +bytes_object_converter + + a: PyBytesObject + / + +[clinic start generated code]*/ + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a) +/*[clinic end generated code: output=7732da869d74b784 input=94211751e7996236]*/ +{ + if (!PyBytes_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyBytesObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +byte_array_object_converter + + a: PyByteArrayObject + / + +[clinic start generated code]*/ + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a) +/*[clinic end generated code: output=51f15c76f302b1f7 input=b04d253db51c6f56]*/ +{ + if (!PyByteArray_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a PyByteArrayObject"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +unicode_converter + + a: unicode + / + +[clinic start generated code]*/ + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=1b4a4adbb6ac6e34 input=de7b5adbf07435ba]*/ +{ + if (!PyUnicode_Check(a)) { + PyErr_SetString(PyExc_AssertionError, + "argument a is not a unicode object"); + return NULL; + } + return pack_arguments_newref(1, a); +} + + +/*[clinic input] +bool_converter + + a: bool = True + b: bool(accept={object}) = True + c: bool(accept={int}) = True + / + +[clinic start generated code]*/ + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=17005b0c29afd590 input=7f6537705b2f32f4]*/ +{ + PyObject *obj_a = a ? Py_True : Py_False; + PyObject *obj_b = b ? Py_True : Py_False; + PyObject *obj_c = c ? Py_True : Py_False; + return pack_arguments_newref(3, obj_a, obj_b, obj_c); +} + + +/*[clinic input] +char_converter + + a: char = b'A' + b: char = b'\a' + c: char = b'\b' + d: char = b'\t' + e: char = b'\n' + f: char = b'\v' + g: char = b'\f' + h: char = b'\r' + i: char = b'"' + j: char = b"'" + k: char = b'?' + l: char = b'\\' + m: char = b'\000' + n: char = b'\377' + / + +[clinic start generated code]*/ + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n) +/*[clinic end generated code: output=f929dbd2e55a9871 input=b601bc5bc7fe85e3]*/ +{ + RETURN_PACKED_ARGS(14, PyLong_FromUnsignedLong, unsigned char, + a, b, c, d, e, f, g, h, i, j, k, l, m, n); +} + + +/*[clinic input] +unsigned_char_converter + + a: unsigned_char = 12 + b: unsigned_char(bitwise=False) = 34 + c: unsigned_char(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c) +/*[clinic end generated code: output=490af3b39ce0b199 input=e859502fbe0b3185]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned char, a, b, c); +} + + +/*[clinic input] +short_converter + + a: short = 12 + / + +[clinic start generated code]*/ + +static PyObject * +short_converter_impl(PyObject *module, short a) +/*[clinic end generated code: output=1ebb7ddb64248988 input=b4e2309a66f650ae]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_short_converter + + a: unsigned_short = 12 + b: unsigned_short(bitwise=False) = 34 + c: unsigned_short(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c) +/*[clinic end generated code: output=5f92cc72fc8707a7 input=9d15cd11e741d0c6]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +int_converter + + a: int = 12 + b: int(accept={int}) = 34 + c: int(accept={str}) = 45 + / + +[clinic start generated code]*/ + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c) +/*[clinic end generated code: output=8e56b59be7d0c306 input=a1dbc6344853db7a]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromLong, long, a, b, c); +} + + +/*[clinic input] +unsigned_int_converter + + a: unsigned_int = 12 + b: unsigned_int(bitwise=False) = 34 + c: unsigned_int(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c) +/*[clinic end generated code: output=399a57a05c494cc7 input=8427ed9a3f96272d]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_converter + + a: long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_converter_impl(PyObject *module, long a) +/*[clinic end generated code: output=9663d936a652707a input=84ad0ef28f24bd85]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLong, long, a); +} + + +/*[clinic input] +unsigned_long_converter + + a: unsigned_long = 12 + b: unsigned_long(bitwise=False) = 34 + c: unsigned_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c) +/*[clinic end generated code: output=120b82ea9ebd93a8 input=440dd6f1817f5d91]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLong, unsigned long, a, b, c); +} + + +/*[clinic input] +long_long_converter + + a: long_long = 12 + / + +[clinic start generated code]*/ + +static PyObject * +long_long_converter_impl(PyObject *module, long long a) +/*[clinic end generated code: output=5fb5f2220770c3e1 input=730fcb3eecf4d993]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromLongLong, long long, a); +} + + +/*[clinic input] +unsigned_long_long_converter + + a: unsigned_long_long = 12 + b: unsigned_long_long(bitwise=False) = 34 + c: unsigned_long_long(bitwise=True) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c) +/*[clinic end generated code: output=65b7273e63501762 input=300737b0bdb230e9]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromUnsignedLongLong, unsigned long long, + a, b, c); +} + + +/*[clinic input] +py_ssize_t_converter + + a: Py_ssize_t = 12 + b: Py_ssize_t(accept={int}) = 34 + c: Py_ssize_t(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=ce252143e0ed0372 input=76d0f342e9317a1f]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +slice_index_converter + + a: slice_index = 12 + b: slice_index(accept={int}) = 34 + c: slice_index(accept={int, NoneType}) = 56 + / + +[clinic start generated code]*/ + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c) +/*[clinic end generated code: output=923c6cac77666a6b input=64f99f3f83265e47]*/ +{ + RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); +} + + +/*[clinic input] +size_t_converter + + a: size_t = 12 + / + +[clinic start generated code]*/ + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a) +/*[clinic end generated code: output=412b5b7334ab444d input=83ae7d9171fbf208]*/ +{ + RETURN_PACKED_ARGS(1, PyLong_FromSize_t, size_t, a); +} + + +/*[clinic input] +float_converter + + a: float = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +float_converter_impl(PyObject *module, float a) +/*[clinic end generated code: output=1c98f64f2cf1d55c input=a625b59ad68047d8]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +double_converter + + a: double = 12.5 + / + +[clinic start generated code]*/ + +static PyObject * +double_converter_impl(PyObject *module, double a) +/*[clinic end generated code: output=a4e8532d284d035d input=098df188f24e7c62]*/ +{ + RETURN_PACKED_ARGS(1, PyFloat_FromDouble, double, a); +} + + +/*[clinic input] +py_complex_converter + + a: Py_complex + / + +[clinic start generated code]*/ + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a) +/*[clinic end generated code: output=9e6ca2eb53b14846 input=e9148a8ca1dbf195]*/ +{ + RETURN_PACKED_ARGS(1, PyComplex_FromCComplex, Py_complex, a); +} + + +/*[clinic input] +str_converter + + a: str = "a" + b: str(accept={robuffer}) = "b" + c: str(accept={robuffer, str}, zeroes=True) = "c" + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_clean_t c_length) +/*[clinic end generated code: output=203880b5c6d9fbb4 input=bff2656c92ee25de]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + +static PyObject * +bytes_from_buffer(Py_buffer *buf) +{ + PyObject *bytes_obj = PyBytes_FromStringAndSize(NULL, buf->len); + if (!bytes_obj) { + return NULL; + } + void *bytes_obj_buf = ((PyBytesObject *)bytes_obj)->ob_sval; + if (PyBuffer_ToContiguous(bytes_obj_buf, buf, buf->len, 'C') < 0) { + Py_DECREF(bytes_obj); + return NULL; + } + return bytes_obj; +} + +/*[clinic input] +py_buffer_converter + + a: Py_buffer(accept={str, buffer, NoneType}) + b: Py_buffer(accept={rwbuffer}) + / + +[clinic start generated code]*/ + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b) +/*[clinic end generated code: output=52fb13311e3d6d03 input=775de727de5c7421]*/ +{ + RETURN_PACKED_ARGS(2, bytes_from_buffer, Py_buffer *, a, b); +} + + +/*[clinic input] +keywords + + a: object + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=850aaed53e26729e input=f44b89e718c1a93b]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_kwonly + + a: object + * + b: object + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=a45c48241da584dc input=1f08e39c3312b015]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +keywords_opt + + a: object + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) +/*[clinic end generated code: output=25e4b67d91c76a66 input=b0ba0e4f04904556]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +keywords_opt_kwonly + + a: object + b: object = None + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=6aa5b655a6e9aeb0 input=f79da689d6c51076]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keywords_kwonly_opt + + a: object + * + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=707f78eb0f55c2b1 input=e0fa1a0e46dca791]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords + + a: object + / + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=6ac88f4a5f0bfc8d input=fde0a2f79fe82b06]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_kwonly + + a: object + / + * + b: object + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=483e6790d3482185 input=78b3712768da9a19]*/ +{ + return pack_arguments_newref(2, a, b); +} + + +/*[clinic input] +posonly_keywords_kwonly + + a: object + / + b: object + * + c: object + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=2fae573e8cc3fad8 input=a1ad5d2295eb803c]*/ +{ + return pack_arguments_newref(3, a, b, c); +} + + +/*[clinic input] +posonly_keywords_opt + + a: object + / + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=f5eb66241bcf68fb input=51c10de2a120e279]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_keywords_opt + + a: object + b: object = None + / + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=d54a30e549296ffd input=f408a1de7dfaf31f]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_kwonly_opt + + a: object + / + * + b: object + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=a20503fe36b4fd62 input=3494253975272f52]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_opt_kwonly_opt + + a: object + b: object = None + / + * + c: object = None + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=64f3204a3a0413b6 input=d17516581e478412]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +posonly_keywords_kwonly_opt + + a: object + / + b: object + * + c: object + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=dbd7e7ddd6257fa0 input=33529f29e97e5adb]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_keywords_opt_kwonly_opt + + a: object + / + b: object + c: object = None + * + d: object = None + e: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e) +/*[clinic end generated code: output=775d12ae44653045 input=4d4cc62f11441301]*/ +{ + return pack_arguments_newref(5, a, b, c, d, e); +} + + +/*[clinic input] +posonly_opt_keywords_opt_kwonly_opt + + a: object + b: object = None + / + c: object = None + * + d: object = None + +[clinic start generated code]*/ + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=40c6dc422591eade input=3964960a68622431]*/ +{ + return pack_arguments_newref(4, a, b, c, d); +} + + +/*[clinic input] +keyword_only_parameter + + * + a: object + +[clinic start generated code]*/ + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=c454b6ce98232787 input=8d2868b8d0b27bdb]*/ +{ + return pack_arguments_newref(1, a); +} + + +static PyMethodDef tester_methods[] = { + TEST_EMPTY_FUNCTION_METHODDEF + OBJECTS_CONVERTER_METHODDEF + BYTES_OBJECT_CONVERTER_METHODDEF + BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF + UNICODE_CONVERTER_METHODDEF + BOOL_CONVERTER_METHODDEF + CHAR_CONVERTER_METHODDEF + UNSIGNED_CHAR_CONVERTER_METHODDEF + SHORT_CONVERTER_METHODDEF + UNSIGNED_SHORT_CONVERTER_METHODDEF + INT_CONVERTER_METHODDEF + UNSIGNED_INT_CONVERTER_METHODDEF + LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_CONVERTER_METHODDEF + LONG_LONG_CONVERTER_METHODDEF + UNSIGNED_LONG_LONG_CONVERTER_METHODDEF + PY_SSIZE_T_CONVERTER_METHODDEF + SLICE_INDEX_CONVERTER_METHODDEF + SIZE_T_CONVERTER_METHODDEF + FLOAT_CONVERTER_METHODDEF + DOUBLE_CONVERTER_METHODDEF + PY_COMPLEX_CONVERTER_METHODDEF + STR_CONVERTER_METHODDEF + PY_BUFFER_CONVERTER_METHODDEF + KEYWORDS_METHODDEF + KEYWORDS_KWONLY_METHODDEF + KEYWORDS_OPT_METHODDEF + KEYWORDS_OPT_KWONLY_METHODDEF + KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_METHODDEF + POSONLY_KWONLY_METHODDEF + POSONLY_KEYWORDS_KWONLY_METHODDEF + POSONLY_KEYWORDS_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_METHODDEF + POSONLY_KWONLY_OPT_METHODDEF + POSONLY_OPT_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF + POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF + KEYWORD_ONLY_PARAMETER_METHODDEF + {NULL, NULL} +}; + +static struct PyModuleDef _testclinic_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testclinic", + .m_size = 0, + .m_methods = tester_methods, +}; + +PyMODINIT_FUNC +PyInit__testclinic(void) +{ + return PyModule_Create(&_testclinic_module); +} + +#undef RETURN_PACKED_ARGS diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h new file mode 100644 index 000000000000..a8273a5e6e2f --- /dev/null +++ b/Modules/clinic/_testclinic.c.h @@ -0,0 +1,1917 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(test_empty_function__doc__, +"test_empty_function($module, /)\n" +"--\n" +"\n"); + +#define TEST_EMPTY_FUNCTION_METHODDEF \ + {"test_empty_function", (PyCFunction)test_empty_function, METH_NOARGS, test_empty_function__doc__}, + +static PyObject * +test_empty_function_impl(PyObject *module); + +static PyObject * +test_empty_function(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_empty_function_impl(module); +} + +PyDoc_STRVAR(objects_converter__doc__, +"objects_converter($module, a, b=<unrepresentable>, /)\n" +"--\n" +"\n"); + +#define OBJECTS_CONVERTER_METHODDEF \ + {"objects_converter", (PyCFunction)(void(*)(void))objects_converter, METH_FASTCALL, objects_converter__doc__}, + +static PyObject * +objects_converter_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +objects_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b = NULL; + + if (!_PyArg_CheckPositional("objects_converter", nargs, 1, 2)) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional; + } + b = args[1]; +skip_optional: + return_value = objects_converter_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_object_converter__doc__, +"bytes_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTES_OBJECT_CONVERTER_METHODDEF \ + {"bytes_object_converter", (PyCFunction)bytes_object_converter, METH_O, bytes_object_converter__doc__}, + +static PyObject * +bytes_object_converter_impl(PyObject *module, PyBytesObject *a); + +static PyObject * +bytes_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyBytesObject *a; + + if (!PyBytes_Check(arg)) { + _PyArg_BadArgument("bytes_object_converter", "argument", "bytes", arg); + goto exit; + } + a = (PyBytesObject *)arg; + return_value = bytes_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(byte_array_object_converter__doc__, +"byte_array_object_converter($module, a, /)\n" +"--\n" +"\n"); + +#define BYTE_ARRAY_OBJECT_CONVERTER_METHODDEF \ + {"byte_array_object_converter", (PyCFunction)byte_array_object_converter, METH_O, byte_array_object_converter__doc__}, + +static PyObject * +byte_array_object_converter_impl(PyObject *module, PyByteArrayObject *a); + +static PyObject * +byte_array_object_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyByteArrayObject *a; + + if (!PyByteArray_Check(arg)) { + _PyArg_BadArgument("byte_array_object_converter", "argument", "bytearray", arg); + goto exit; + } + a = (PyByteArrayObject *)arg; + return_value = byte_array_object_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_converter__doc__, +"unicode_converter($module, a, /)\n" +"--\n" +"\n"); + +#define UNICODE_CONVERTER_METHODDEF \ + {"unicode_converter", (PyCFunction)unicode_converter, METH_O, unicode_converter__doc__}, + +static PyObject * +unicode_converter_impl(PyObject *module, PyObject *a); + +static PyObject * +unicode_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("unicode_converter", "argument", "str", arg); + goto exit; + } + if (PyUnicode_READY(arg) == -1) { + goto exit; + } + a = arg; + return_value = unicode_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(bool_converter__doc__, +"bool_converter($module, a=True, b=True, c=True, /)\n" +"--\n" +"\n"); + +#define BOOL_CONVERTER_METHODDEF \ + {"bool_converter", (PyCFunction)(void(*)(void))bool_converter, METH_FASTCALL, bool_converter__doc__}, + +static PyObject * +bool_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +bool_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 1; + int b = 1; + int c = 1; + + if (!_PyArg_CheckPositional("bool_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyObject_IsTrue(args[0]); + if (a < 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = PyObject_IsTrue(args[1]); + if (b < 0) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = _PyLong_AsInt(args[2]); + if (c == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = bool_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(char_converter__doc__, +"char_converter($module, a=b\'A\', b=b\'\\x07\', c=b\'\\x08\', d=b\'\\t\', e=b\'\\n\',\n" +" f=b\'\\x0b\', g=b\'\\x0c\', h=b\'\\r\', i=b\'\"\', j=b\"\'\", k=b\'?\',\n" +" l=b\'\\\\\', m=b\'\\x00\', n=b\'\\xff\', /)\n" +"--\n" +"\n"); + +#define CHAR_CONVERTER_METHODDEF \ + {"char_converter", (PyCFunction)(void(*)(void))char_converter, METH_FASTCALL, char_converter__doc__}, + +static PyObject * +char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, + char f, char g, char h, char i, char j, char k, char l, + char m, char n); + +static PyObject * +char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char a = 'A'; + char b = '\x07'; + char c = '\x08'; + char d = '\t'; + char e = '\n'; + char f = '\x0b'; + char g = '\x0c'; + char h = '\r'; + char i = '"'; + char j = '\''; + char k = '?'; + char l = '\\'; + char m = '\x00'; + char n = '\xff'; + + if (!_PyArg_CheckPositional("char_converter", nargs, 0, 14)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + a = PyBytes_AS_STRING(args[0])[0]; + } + else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + a = PyByteArray_AS_STRING(args[0])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 1", "a byte string of length 1", args[0]); + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + b = PyBytes_AS_STRING(args[1])[0]; + } + else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + b = PyByteArray_AS_STRING(args[1])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 2", "a byte string of length 1", args[1]); + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + c = PyBytes_AS_STRING(args[2])[0]; + } + else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + c = PyByteArray_AS_STRING(args[2])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 3", "a byte string of length 1", args[2]); + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + d = PyBytes_AS_STRING(args[3])[0]; + } + else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + d = PyByteArray_AS_STRING(args[3])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 4", "a byte string of length 1", args[3]); + goto exit; + } + if (nargs < 5) { + goto skip_optional; + } + if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + e = PyBytes_AS_STRING(args[4])[0]; + } + else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + e = PyByteArray_AS_STRING(args[4])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 5", "a byte string of length 1", args[4]); + goto exit; + } + if (nargs < 6) { + goto skip_optional; + } + if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + f = PyBytes_AS_STRING(args[5])[0]; + } + else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + f = PyByteArray_AS_STRING(args[5])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 6", "a byte string of length 1", args[5]); + goto exit; + } + if (nargs < 7) { + goto skip_optional; + } + if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + g = PyBytes_AS_STRING(args[6])[0]; + } + else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + g = PyByteArray_AS_STRING(args[6])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 7", "a byte string of length 1", args[6]); + goto exit; + } + if (nargs < 8) { + goto skip_optional; + } + if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + h = PyBytes_AS_STRING(args[7])[0]; + } + else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + h = PyByteArray_AS_STRING(args[7])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 8", "a byte string of length 1", args[7]); + goto exit; + } + if (nargs < 9) { + goto skip_optional; + } + if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + i = PyBytes_AS_STRING(args[8])[0]; + } + else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + i = PyByteArray_AS_STRING(args[8])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 9", "a byte string of length 1", args[8]); + goto exit; + } + if (nargs < 10) { + goto skip_optional; + } + if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + j = PyBytes_AS_STRING(args[9])[0]; + } + else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + j = PyByteArray_AS_STRING(args[9])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 10", "a byte string of length 1", args[9]); + goto exit; + } + if (nargs < 11) { + goto skip_optional; + } + if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + k = PyBytes_AS_STRING(args[10])[0]; + } + else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + k = PyByteArray_AS_STRING(args[10])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 11", "a byte string of length 1", args[10]); + goto exit; + } + if (nargs < 12) { + goto skip_optional; + } + if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + l = PyBytes_AS_STRING(args[11])[0]; + } + else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + l = PyByteArray_AS_STRING(args[11])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 12", "a byte string of length 1", args[11]); + goto exit; + } + if (nargs < 13) { + goto skip_optional; + } + if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + m = PyBytes_AS_STRING(args[12])[0]; + } + else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + m = PyByteArray_AS_STRING(args[12])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 13", "a byte string of length 1", args[12]); + goto exit; + } + if (nargs < 14) { + goto skip_optional; + } + if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + n = PyBytes_AS_STRING(args[13])[0]; + } + else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + n = PyByteArray_AS_STRING(args[13])[0]; + } + else { + _PyArg_BadArgument("char_converter", "argument 14", "a byte string of length 1", args[13]); + goto exit; + } +skip_optional: + return_value = char_converter_impl(module, a, b, c, d, e, f, g, h, i, j, k, l, m, n); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_char_converter__doc__, +"unsigned_char_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_CHAR_CONVERTER_METHODDEF \ + {"unsigned_char_converter", (PyCFunction)(void(*)(void))unsigned_char_converter, METH_FASTCALL, unsigned_char_converter__doc__}, + +static PyObject * +unsigned_char_converter_impl(PyObject *module, unsigned char a, + unsigned char b, unsigned char c); + +static PyObject * +unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned char a = 12; + unsigned char b = 34; + unsigned char c = 56; + + if (!_PyArg_CheckPositional("unsigned_char_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + a = (unsigned char) ival; + } + } + if (nargs < 2) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + b = (unsigned char) ival; + } + } + if (nargs < 3) { + goto skip_optional; + } + { + unsigned long ival = PyLong_AsUnsignedLongMask(args[2]); + if (ival == (unsigned long)-1 && PyErr_Occurred()) { + goto exit; + } + else { + c = (unsigned char) ival; + } + } +skip_optional: + return_value = unsigned_char_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(short_converter__doc__, +"short_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SHORT_CONVERTER_METHODDEF \ + {"short_converter", (PyCFunction)(void(*)(void))short_converter, METH_FASTCALL, short_converter__doc__}, + +static PyObject * +short_converter_impl(PyObject *module, short a); + +static PyObject * +short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + short a = 12; + + if (!_PyArg_CheckPositional("short_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + a = (short) ival; + } + } +skip_optional: + return_value = short_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_short_converter__doc__, +"unsigned_short_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_SHORT_CONVERTER_METHODDEF \ + {"unsigned_short_converter", (PyCFunction)(void(*)(void))unsigned_short_converter, METH_FASTCALL, unsigned_short_converter__doc__}, + +static PyObject * +unsigned_short_converter_impl(PyObject *module, unsigned short a, + unsigned short b, unsigned short c); + +static PyObject * +unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned short a = 12; + unsigned short b = 34; + unsigned short c = 56; + + if (!_PyArg_CheckPositional("unsigned_short_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedShort_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned short)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned short)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_short_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(int_converter__doc__, +"int_converter($module, a=12, b=34, c=45, /)\n" +"--\n" +"\n"); + +#define INT_CONVERTER_METHODDEF \ + {"int_converter", (PyCFunction)(void(*)(void))int_converter, METH_FASTCALL, int_converter__doc__}, + +static PyObject * +int_converter_impl(PyObject *module, int a, int b, int c); + +static PyObject * +int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int a = 12; + int b = 34; + int c = 45; + + if (!_PyArg_CheckPositional("int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = _PyLong_AsInt(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + b = _PyLong_AsInt(args[1]); + if (b == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyUnicode_Check(args[2])) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + if (PyUnicode_READY(args[2])) { + goto exit; + } + if (PyUnicode_GET_LENGTH(args[2]) != 1) { + _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + goto exit; + } + c = PyUnicode_READ_CHAR(args[2], 0); +skip_optional: + return_value = int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_int_converter__doc__, +"unsigned_int_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_INT_CONVERTER_METHODDEF \ + {"unsigned_int_converter", (PyCFunction)(void(*)(void))unsigned_int_converter, METH_FASTCALL, unsigned_int_converter__doc__}, + +static PyObject * +unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, + unsigned int c); + +static PyObject * +unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int a = 12; + unsigned int b = 34; + unsigned int c = 56; + + if (!_PyArg_CheckPositional("unsigned_int_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + c = (unsigned int)PyLong_AsUnsignedLongMask(args[2]); + if (c == (unsigned int)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = unsigned_int_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_converter__doc__, +"long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_CONVERTER_METHODDEF \ + {"long_converter", (PyCFunction)(void(*)(void))long_converter, METH_FASTCALL, long_converter__doc__}, + +static PyObject * +long_converter_impl(PyObject *module, long a); + +static PyObject * +long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long a = 12; + + if (!_PyArg_CheckPositional("long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_converter__doc__, +"unsigned_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_converter", (PyCFunction)(void(*)(void))unsigned_long_converter, METH_FASTCALL, unsigned_long_converter__doc__}, + +static PyObject * +unsigned_long_converter_impl(PyObject *module, unsigned long a, + unsigned long b, unsigned long c); + +static PyObject * +unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long a = 12; + unsigned long b = 34; + unsigned long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongMask(args[2]); +skip_optional: + return_value = unsigned_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(long_long_converter__doc__, +"long_long_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define LONG_LONG_CONVERTER_METHODDEF \ + {"long_long_converter", (PyCFunction)(void(*)(void))long_long_converter, METH_FASTCALL, long_long_converter__doc__}, + +static PyObject * +long_long_converter_impl(PyObject *module, long long a); + +static PyObject * +long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + long long a = 12; + + if (!_PyArg_CheckPositional("long_long_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + a = PyLong_AsLongLong(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = long_long_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(unsigned_long_long_converter__doc__, +"unsigned_long_long_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define UNSIGNED_LONG_LONG_CONVERTER_METHODDEF \ + {"unsigned_long_long_converter", (PyCFunction)(void(*)(void))unsigned_long_long_converter, METH_FASTCALL, unsigned_long_long_converter__doc__}, + +static PyObject * +unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, + unsigned long long b, unsigned long long c); + +static PyObject * +unsigned_long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned long long a = 12; + unsigned long long b = 34; + unsigned long long c = 56; + + if (!_PyArg_CheckPositional("unsigned_long_long_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedLongLong_Converter(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("unsigned_long_long_converter", "argument 3", "int", args[2]); + goto exit; + } + c = PyLong_AsUnsignedLongLongMask(args[2]); +skip_optional: + return_value = unsigned_long_long_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_ssize_t_converter__doc__, +"py_ssize_t_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define PY_SSIZE_T_CONVERTER_METHODDEF \ + {"py_ssize_t_converter", (PyCFunction)(void(*)(void))py_ssize_t_converter, METH_FASTCALL, py_ssize_t_converter__doc__}, + +static PyObject * +py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("py_ssize_t_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + a = ival; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + b = ival; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = py_ssize_t_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(slice_index_converter__doc__, +"slice_index_converter($module, a=12, b=34, c=56, /)\n" +"--\n" +"\n"); + +#define SLICE_INDEX_CONVERTER_METHODDEF \ + {"slice_index_converter", (PyCFunction)(void(*)(void))slice_index_converter, METH_FASTCALL, slice_index_converter__doc__}, + +static PyObject * +slice_index_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, + Py_ssize_t c); + +static PyObject * +slice_index_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t a = 12; + Py_ssize_t b = 34; + Py_ssize_t c = 56; + + if (!_PyArg_CheckPositional("slice_index_converter", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[0], &a)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &b)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &c)) { + goto exit; + } +skip_optional: + return_value = slice_index_converter_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(size_t_converter__doc__, +"size_t_converter($module, a=12, /)\n" +"--\n" +"\n"); + +#define SIZE_T_CONVERTER_METHODDEF \ + {"size_t_converter", (PyCFunction)(void(*)(void))size_t_converter, METH_FASTCALL, size_t_converter__doc__}, + +static PyObject * +size_t_converter_impl(PyObject *module, size_t a); + +static PyObject * +size_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + size_t a = 12; + + if (!_PyArg_CheckPositional("size_t_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_PyLong_Size_t_Converter(args[0], &a)) { + goto exit; + } +skip_optional: + return_value = size_t_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(float_converter__doc__, +"float_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define FLOAT_CONVERTER_METHODDEF \ + {"float_converter", (PyCFunction)(void(*)(void))float_converter, METH_FASTCALL, float_converter__doc__}, + +static PyObject * +float_converter_impl(PyObject *module, float a); + +static PyObject * +float_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float a = 12.5; + + if (!_PyArg_CheckPositional("float_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + a = (float) PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = float_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(double_converter__doc__, +"double_converter($module, a=12.5, /)\n" +"--\n" +"\n"); + +#define DOUBLE_CONVERTER_METHODDEF \ + {"double_converter", (PyCFunction)(void(*)(void))double_converter, METH_FASTCALL, double_converter__doc__}, + +static PyObject * +double_converter_impl(PyObject *module, double a); + +static PyObject * +double_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double a = 12.5; + + if (!_PyArg_CheckPositional("double_converter", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (PyFloat_CheckExact(args[0])) { + a = PyFloat_AS_DOUBLE(args[0]); + } + else + { + a = PyFloat_AsDouble(args[0]); + if (a == -1.0 && PyErr_Occurred()) { + goto exit; + } + } +skip_optional: + return_value = double_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_complex_converter__doc__, +"py_complex_converter($module, a, /)\n" +"--\n" +"\n"); + +#define PY_COMPLEX_CONVERTER_METHODDEF \ + {"py_complex_converter", (PyCFunction)py_complex_converter, METH_O, py_complex_converter__doc__}, + +static PyObject * +py_complex_converter_impl(PyObject *module, Py_complex a); + +static PyObject * +py_complex_converter(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_complex a; + + a = PyComplex_AsCComplex(arg); + if (PyErr_Occurred()) { + goto exit; + } + return_value = py_complex_converter_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(str_converter__doc__, +"str_converter($module, a=\'a\', b=\'b\', c=\'c\', /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_METHODDEF \ + {"str_converter", (PyCFunction)(void(*)(void))str_converter, METH_FASTCALL, str_converter__doc__}, + +static PyObject * +str_converter_impl(PyObject *module, const char *a, const char *b, + const char *c, Py_ssize_clean_t c_length); + +static PyObject * +str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + const char *a = "a"; + const char *b = "b"; + const char *c = "c"; + Py_ssize_clean_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "|sys#:str_converter", + &a, &b, &c, &c_length)) { + goto exit; + } + return_value = str_converter_impl(module, a, b, c, c_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_buffer_converter__doc__, +"py_buffer_converter($module, a, b, /)\n" +"--\n" +"\n"); + +#define PY_BUFFER_CONVERTER_METHODDEF \ + {"py_buffer_converter", (PyCFunction)(void(*)(void))py_buffer_converter, METH_FASTCALL, py_buffer_converter__doc__}, + +static PyObject * +py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b); + +static PyObject * +py_buffer_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer a = {NULL, NULL}; + Py_buffer b = {NULL, NULL}; + + if (!_PyArg_ParseStack(args, nargs, "z*w*:py_buffer_converter", + &a, &b)) { + goto exit; + } + return_value = py_buffer_converter_impl(module, &a, &b); + +exit: + /* Cleanup for a */ + if (a.obj) { + PyBuffer_Release(&a); + } + /* Cleanup for b */ + if (b.obj) { + PyBuffer_Release(&b); + } + + return return_value; +} + +PyDoc_STRVAR(keywords__doc__, +"keywords($module, /, a, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_METHODDEF \ + {"keywords", (PyCFunction)(void(*)(void))keywords, METH_FASTCALL|METH_KEYWORDS, keywords__doc__}, + +static PyObject * +keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly__doc__, +"keywords_kwonly($module, /, a, *, b)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_METHODDEF \ + {"keywords_kwonly", (PyCFunction)(void(*)(void))keywords_kwonly, METH_FASTCALL|METH_KEYWORDS, keywords_kwonly__doc__}, + +static PyObject * +keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = keywords_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt__doc__, +"keywords_opt($module, /, a, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_METHODDEF \ + {"keywords_opt", (PyCFunction)(void(*)(void))keywords_opt, METH_FASTCALL|METH_KEYWORDS, keywords_opt__doc__}, + +static PyObject * +keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); + +static PyObject * +keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = keywords_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_opt_kwonly__doc__, +"keywords_opt_kwonly($module, /, a, b=None, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_OPT_KWONLY_METHODDEF \ + {"keywords_opt_kwonly", (PyCFunction)(void(*)(void))keywords_opt_kwonly, METH_FASTCALL|METH_KEYWORDS, keywords_opt_kwonly__doc__}, + +static PyObject * +keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_opt_kwonly", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = keywords_opt_kwonly_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keywords_kwonly_opt__doc__, +"keywords_kwonly_opt($module, /, a, *, b=None, c=None)\n" +"--\n" +"\n"); + +#define KEYWORDS_KWONLY_OPT_METHODDEF \ + {"keywords_kwonly_opt", (PyCFunction)(void(*)(void))keywords_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, keywords_kwonly_opt__doc__}, + +static PyObject * +keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keywords_kwonly_opt", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[2]; +skip_optional_kwonly: + return_value = keywords_kwonly_opt_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords__doc__, +"posonly_keywords($module, a, /, b)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_METHODDEF \ + {"posonly_keywords", (PyCFunction)(void(*)(void))posonly_keywords, METH_FASTCALL|METH_KEYWORDS, posonly_keywords__doc__}, + +static PyObject * +posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_keywords_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly__doc__, +"posonly_kwonly($module, a, /, *, b)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_METHODDEF \ + {"posonly_kwonly", (PyCFunction)(void(*)(void))posonly_kwonly, METH_FASTCALL|METH_KEYWORDS, posonly_kwonly__doc__}, + +static PyObject * +posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = posonly_kwonly_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly__doc__, +"posonly_keywords_kwonly($module, a, /, b, *, c)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_METHODDEF \ + {"posonly_keywords_kwonly", (PyCFunction)(void(*)(void))posonly_keywords_kwonly, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly__doc__}, + +static PyObject * +posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly", 0}; + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = posonly_keywords_kwonly_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt__doc__, +"posonly_keywords_opt($module, a, /, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_METHODDEF \ + {"posonly_keywords_opt", (PyCFunction)(void(*)(void))posonly_keywords_opt, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt__doc__}, + +static PyObject * +posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt__doc__, +"posonly_opt_keywords_opt($module, a, b=None, /, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_METHODDEF \ + {"posonly_opt_keywords_opt", (PyCFunction)(void(*)(void))posonly_opt_keywords_opt, METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + d = args[3]; +skip_optional_pos: + return_value = posonly_opt_keywords_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_kwonly_opt__doc__, +"posonly_kwonly_opt($module, a, /, *, b, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_KWONLY_OPT_METHODDEF \ + {"posonly_kwonly_opt", (PyCFunction)(void(*)(void))posonly_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_kwonly_opt__doc__}, + +static PyObject * +posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_kwonly_opt__doc__, +"posonly_opt_kwonly_opt($module, a, b=None, /, *, c=None, d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_kwonly_opt", (PyCFunction)(void(*)(void))posonly_opt_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_kwonly_opt__doc__, +"posonly_keywords_kwonly_opt($module, a, /, b, *, c, d=None, e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_kwonly_opt", (PyCFunction)(void(*)(void))posonly_keywords_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_keywords_opt_kwonly_opt__doc__, +"posonly_keywords_opt_kwonly_opt($module, a, /, b, c=None, *, d=None,\n" +" e=None)\n" +"--\n" +"\n"); + +#define POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_keywords_opt_kwonly_opt", (PyCFunction)(void(*)(void))posonly_keywords_opt_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, + PyObject *e); + +static PyObject * +posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + PyObject *d = Py_None; + PyObject *e = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + d = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + e = args[4]; +skip_optional_kwonly: + return_value = posonly_keywords_opt_kwonly_opt_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(posonly_opt_keywords_opt_kwonly_opt__doc__, +"posonly_opt_keywords_opt_kwonly_opt($module, a, b=None, /, c=None, *,\n" +" d=None)\n" +"--\n" +"\n"); + +#define POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF \ + {"posonly_opt_keywords_opt_kwonly_opt", (PyCFunction)(void(*)(void))posonly_opt_keywords_opt_kwonly_opt, METH_FASTCALL|METH_KEYWORDS, posonly_opt_keywords_opt_kwonly_opt__doc__}, + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d); + +static PyObject * +posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "c", "d", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_opt_keywords_opt_kwonly_opt", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + PyObject *d = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + noptargs--; + b = args[1]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + c = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + d = args[3]; +skip_optional_kwonly: + return_value = posonly_opt_keywords_opt_kwonly_opt_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(keyword_only_parameter__doc__, +"keyword_only_parameter($module, /, *, a)\n" +"--\n" +"\n"); + +#define KEYWORD_ONLY_PARAMETER_METHODDEF \ + {"keyword_only_parameter", (PyCFunction)(void(*)(void))keyword_only_parameter, METH_FASTCALL|METH_KEYWORDS, keyword_only_parameter__doc__}, + +static PyObject * +keyword_only_parameter_impl(PyObject *module, PyObject *a); + +static PyObject * +keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "keyword_only_parameter", 0}; + PyObject *argsbuf[1]; + PyObject *a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = keyword_only_parameter_impl(module, a); + +exit: + return return_value; +} +/*[clinic end generated code: output=2d5ac6b41daadf6f input=a9049054013a1b77]*/ diff --git a/Tools/scripts/generate_stdlib_module_names.py b/Tools/scripts/generate_stdlib_module_names.py index dc6a07427471..efea9a7b20a1 100644 --- a/Tools/scripts/generate_stdlib_module_names.py +++ b/Tools/scripts/generate_stdlib_module_names.py @@ -24,6 +24,7 @@ '_ctypes_test', '_testbuffer', '_testcapi', + '_testclinic', '_testconsole', '_testimportmultiple', '_testinternalcapi', diff --git a/setup.py b/setup.py index 85a2b26357db..a39610a1c7cd 100644 --- a/setup.py +++ b/setup.py @@ -1043,6 +1043,9 @@ def detect_test_extensions(self): self.add(Extension('_testcapi', ['_testcapimodule.c'], depends=['testcapi_long.h'])) + # Python Argument Clinc functional test module + self.add(Extension('_testclinic', ['_testclinic.c'])) + # Python Internal C API test module self.add(Extension('_testinternalcapi', ['_testinternalcapi.c'], extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) From webhook-mailer at python.org Sat Dec 17 01:40:56 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 06:40:56 -0000 Subject: [Python-checkins] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (#99890) Message-ID: <mailman.2954.1671259256.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a6f82f1fc68cb24e2d88d35fde4cfb663213a744 commit: a6f82f1fc68cb24e2d88d35fde4cfb663213a744 branch: main author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:10:51+05:30 summary: gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (#99890) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: From webhook-mailer at python.org Sat Dec 17 01:53:25 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 06:53:25 -0000 Subject: [Python-checkins] Document that zipfile's pwd parameter is a `bytes` object (GH-100209) Message-ID: <mailman.2955.1671260006.3313.python-checkins@python.org> https://github.com/python/cpython/commit/697a07a8549fa117c047796ec8a59eb7f1fd5741 commit: 697a07a8549fa117c047796ec8a59eb7f1fd5741 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T22:53:20-08:00 summary: Document that zipfile's pwd parameter is a `bytes` object (GH-100209) (cherry picked from commit 5ee7eb9debb12914f36c5ccee92460a681516fd6) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index f6288709fec1..10fffb70dbdb 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -251,7 +251,8 @@ ZipFile Objects Access a member of the archive as a binary file-like object. *name* can be either the name of a file within the archive or a :class:`ZipInfo` object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files. + or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files as a + :class:`bytes` object. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -303,7 +304,7 @@ ZipFile Objects must be its full name or a :class:`ZipInfo` object. Its file information is extracted as accurately as possible. *path* specifies a different directory to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files. + *pwd* is the password used for encrypted files as a :class:`bytes` object. Returns the normalized path created (a directory or new file). @@ -330,7 +331,7 @@ ZipFile Objects Extract all members from the archive to the current working directory. *path* specifies a different directory to extract to. *members* is optional and must be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files. + used for encrypted files as a :class:`bytes` object. .. warning:: @@ -355,16 +356,16 @@ ZipFile Objects .. method:: ZipFile.setpassword(pwd) - Set *pwd* as default password to extract encrypted files. + Set *pwd* (a :class:`bytes` object) as default password to extract encrypted files. .. method:: ZipFile.read(name, pwd=None) Return the bytes of the file *name* in the archive. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files and, if specified, - it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a ZipFile that uses a compression method other than + read or append. *pwd* is the password used for encrypted files as a :class:`bytes` + object and, if specified, overrides the default password set with :meth:`setpassword`. + Calling :meth:`read` on a ZipFile that uses a compression method other than :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also be raised if the corresponding compression module is not available. From webhook-mailer at python.org Sat Dec 17 01:54:38 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 06:54:38 -0000 Subject: [Python-checkins] Document that zipfile's pwd parameter is a `bytes` object (GH-100209) Message-ID: <mailman.2956.1671260080.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e3d85783e56d9621b432819805c452f2f65669e0 commit: e3d85783e56d9621b432819805c452f2f65669e0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T22:54:33-08:00 summary: Document that zipfile's pwd parameter is a `bytes` object (GH-100209) (cherry picked from commit 5ee7eb9debb12914f36c5ccee92460a681516fd6) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4dd9fa961a8d..d02e32d320bc 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -273,7 +273,8 @@ ZipFile Objects Access a member of the archive as a binary file-like object. *name* can be either the name of a file within the archive or a :class:`ZipInfo` object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files. + or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files as a + :class:`bytes` object. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -325,7 +326,7 @@ ZipFile Objects must be its full name or a :class:`ZipInfo` object. Its file information is extracted as accurately as possible. *path* specifies a different directory to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files. + *pwd* is the password used for encrypted files as a :class:`bytes` object. Returns the normalized path created (a directory or new file). @@ -352,7 +353,7 @@ ZipFile Objects Extract all members from the archive to the current working directory. *path* specifies a different directory to extract to. *members* is optional and must be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files. + used for encrypted files as a :class:`bytes` object. .. warning:: @@ -377,16 +378,16 @@ ZipFile Objects .. method:: ZipFile.setpassword(pwd) - Set *pwd* as default password to extract encrypted files. + Set *pwd* (a :class:`bytes` object) as default password to extract encrypted files. .. method:: ZipFile.read(name, pwd=None) Return the bytes of the file *name* in the archive. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files and, if specified, - it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a ZipFile that uses a compression method other than + read or append. *pwd* is the password used for encrypted files as a :class:`bytes` + object and, if specified, overrides the default password set with :meth:`setpassword`. + Calling :meth:`read` on a ZipFile that uses a compression method other than :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also be raised if the corresponding compression module is not available. From webhook-mailer at python.org Sat Dec 17 02:12:45 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 07:12:45 -0000 Subject: [Python-checkins] gh-97909: Fix markup for `PyMethodDef` members (#100089) Message-ID: <mailman.2957.1671261165.3313.python-checkins@python.org> https://github.com/python/cpython/commit/8edcb30c3f8bdd8099a093146fedbd9b63a3f667 commit: 8edcb30c3f8bdd8099a093146fedbd9b63a3f667 branch: main author: ram vikram singh <ramvikrams243 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T12:42:39+05:30 summary: gh-97909: Fix markup for `PyMethodDef` members (#100089) files: M Doc/c-api/structures.rst diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 827d624fc99e..be6e7b5faf9f 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -228,29 +228,30 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`ml_name` | const char \* | name of the method | - +------------------+---------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+---------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+---------------+-------------------------------+ - | :attr:`ml_doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ - -The :attr:`ml_meth` is a C function pointer. The functions may be of different + .. c:member:: const char* ml_name + + name of the method + + .. c:member:: PyCFunction ml_meth + + pointer to the C implementation + + .. c:member:: int ml_flags + + flags bits indicating how the call should be constructed + + .. c:member:: const char* ml_doc + + points to the contents of the docstring + +The :c:member:`ml_meth` is a C function pointer. The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :attr:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding convention. From webhook-mailer at python.org Sat Dec 17 02:19:50 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 07:19:50 -0000 Subject: [Python-checkins] gh-97909: Fix markup for `PyMethodDef` members (GH-100089) Message-ID: <mailman.2958.1671261591.3313.python-checkins@python.org> https://github.com/python/cpython/commit/370498b12ee9ce9edb93da0498a5cec1806147fc commit: 370498b12ee9ce9edb93da0498a5cec1806147fc branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T23:19:45-08:00 summary: gh-97909: Fix markup for `PyMethodDef` members (GH-100089) (cherry picked from commit 8edcb30c3f8bdd8099a093146fedbd9b63a3f667) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/c-api/structures.rst diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 2b5d9a84822b..30760c0db88c 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -247,29 +247,30 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`ml_name` | const char \* | name of the method | - +------------------+---------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+---------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+---------------+-------------------------------+ - | :attr:`ml_doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ + .. c:member:: const char* ml_name + + name of the method + + .. c:member:: PyCFunction ml_meth + + pointer to the C implementation + + .. c:member:: int ml_flags + + flags bits indicating how the call should be constructed + + .. c:member:: const char* ml_doc + + points to the contents of the docstring -The :attr:`ml_meth` is a C function pointer. The functions may be of different +The :c:member:`ml_meth` is a C function pointer. The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :attr:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding convention. From webhook-mailer at python.org Sat Dec 17 02:20:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 07:20:00 -0000 Subject: [Python-checkins] gh-97909: Fix markup for `PyMethodDef` members (GH-100089) Message-ID: <mailman.2959.1671261601.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e38a525c20f0fe68cfade983a472a1f011d5e3df commit: e38a525c20f0fe68cfade983a472a1f011d5e3df branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T23:19:54-08:00 summary: gh-97909: Fix markup for `PyMethodDef` members (GH-100089) (cherry picked from commit 8edcb30c3f8bdd8099a093146fedbd9b63a3f667) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/c-api/structures.rst diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 069774745654..4ad85d0428a0 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -235,29 +235,30 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`ml_name` | const char \* | name of the method | - +------------------+---------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+---------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+---------------+-------------------------------+ - | :attr:`ml_doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ + .. c:member:: const char* ml_name + + name of the method + + .. c:member:: PyCFunction ml_meth + + pointer to the C implementation + + .. c:member:: int ml_flags + + flags bits indicating how the call should be constructed + + .. c:member:: const char* ml_doc + + points to the contents of the docstring -The :attr:`ml_meth` is a C function pointer. The functions may be of different +The :c:member:`ml_meth` is a C function pointer. The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :attr:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding convention. From webhook-mailer at python.org Sat Dec 17 02:42:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 17 Dec 2022 07:42:24 -0000 Subject: [Python-checkins] Docs: Use `PY_VERSION_HEX` for version comparison (#100179) Message-ID: <mailman.2960.1671262945.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0264f634f720fbf12afaf1715a53cd1495fbd85b commit: 0264f634f720fbf12afaf1715a53cd1495fbd85b branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-17T13:12:18+05:30 summary: Docs: Use `PY_VERSION_HEX` for version comparison (#100179) files: M Doc/c-api/apiabiversion.rst M Doc/whatsnew/3.11.rst diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 85b6e2f37387..62d542966622 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -58,6 +58,8 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is hexversion ``0x030a00f0``. + Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. + This version is also available via the symbol :data:`Py_Version`. .. c:var:: const unsigned long Py_Version diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7931988ed0b0..6f283f1d8002 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2319,7 +2319,7 @@ Porting to Python 3.11 can define the following macros and use them throughout the code (credit: these were copied from the ``mypy`` codebase):: - #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 + #if PY_VERSION_HEX >= 0x03080000 # define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) # define CPy_TRASHCAN_END(op) Py_TRASHCAN_END #else From webhook-mailer at python.org Sat Dec 17 02:49:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 17 Dec 2022 07:49:59 -0000 Subject: [Python-checkins] Docs: Use `PY_VERSION_HEX` for version comparison (GH-100179) Message-ID: <mailman.2961.1671263401.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bf0a334c4e46041c5621f0271cb70feef934c501 commit: bf0a334c4e46041c5621f0271cb70feef934c501 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-16T23:49:49-08:00 summary: Docs: Use `PY_VERSION_HEX` for version comparison (GH-100179) (cherry picked from commit 0264f634f720fbf12afaf1715a53cd1495fbd85b) Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> files: M Doc/c-api/apiabiversion.rst M Doc/whatsnew/3.11.rst diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 85b6e2f37387..62d542966622 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -58,6 +58,8 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is hexversion ``0x030a00f0``. + Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. + This version is also available via the symbol :data:`Py_Version`. .. c:var:: const unsigned long Py_Version diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index f09ccb133fbc..4244875f6c69 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2320,7 +2320,7 @@ Porting to Python 3.11 can define the following macros and use them throughout the code (credit: these were copied from the ``mypy`` codebase):: - #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 + #if PY_VERSION_HEX >= 0x03080000 # define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) # define CPy_TRASHCAN_END(op) Py_TRASHCAN_END #else From webhook-mailer at python.org Sat Dec 17 03:53:42 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 17 Dec 2022 08:53:42 -0000 Subject: [Python-checkins] gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551) Message-ID: <mailman.2962.1671267223.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b38a9aa747785f6d540e95e6846d6510de6b306 commit: 2b38a9aa747785f6d540e95e6846d6510de6b306 branch: main author: Erlend E. Aasland <erlend.aasland at protonmail.com> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-17T09:53:36+01:00 summary: gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551) files: A Lib/test/test_capi/test_mem.py M Lib/test/test_capi/test_misc.py M Modules/_testcapi/mem.c M Modules/_testcapimodule.c diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py new file mode 100644 index 000000000000..a9ff410cb93a --- /dev/null +++ b/Lib/test/test_capi/test_mem.py @@ -0,0 +1,169 @@ +import re +import textwrap +import unittest + + +from test import support +from test.support import import_helper, requires_subprocess +from test.support.script_helper import assert_python_failure, assert_python_ok + + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + + at requires_subprocess() +class PyMemDebugTests(unittest.TestCase): + PYTHONMALLOC = 'debug' + # '0x04c06e0' or '04C06E0' + PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + # FreeBSD: instruct jemalloc to not fill freed() memory + # with junk byte 0x5a, see JEMALLOC(3) + MALLOC_CONF="junk:false", + ) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE $0x[0-9a-f]{{2}}$:\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfd\n" + r" at tail\+2: 0xfd\n" + r" .*\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") + regex = regex.format(ptr=self.PTR_REGEX) + regex = re.compile(regex, flags=re.DOTALL) + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") + regex = regex.format(ptr=self.PTR_REGEX) + self.assertRegex(out, regex) + + def check_malloc_without_gil(self, code): + out = self.check(code) + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without holding the GIL') + self.assertIn(expected, out) + + def test_pymem_malloc_without_gil(self): + # Debug hooks must raise an error if PyMem_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def test_pyobject_malloc_without_gil(self): + # Debug hooks must raise an error if PyObject_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def check_pyobject_is_freed(self, func_name): + code = textwrap.dedent(f''' + import gc, os, sys, _testcapi + # Disable the GC to avoid crash on GC collection + gc.disable() + try: + _testcapi.{func_name}() + # Exit immediately to avoid a crash while deallocating + # the invalid object + os._exit(0) + except _testcapi.error: + os._exit(1) + ''') + assert_python_ok( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + MALLOC_CONF="junk:false", + ) + + def test_pyobject_null_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_null_is_freed') + + def test_pyobject_uninitialized_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') + + def test_pyobject_forbidden_bytes_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') + + def test_pyobject_freed_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_freed_is_freed') + + def test_set_nomemory(self): + code = """if 1: + import _testcapi + + class C(): pass + + # The first loop tests both functions and that remove_mem_hooks() + # can be called twice in a row. The second loop checks a call to + # set_nomemory() after a call to remove_mem_hooks(). The third + # loop checks the start and stop arguments of set_nomemory(). + for outer_cnt in range(1, 4): + start = 10 * outer_cnt + for j in range(100): + if j == 0: + if outer_cnt != 3: + _testcapi.set_nomemory(start) + else: + _testcapi.set_nomemory(start, start + 1) + try: + C() + except MemoryError as e: + if outer_cnt != 3: + _testcapi.remove_mem_hooks() + print('MemoryError', outer_cnt, j) + _testcapi.remove_mem_hooks() + break + """ + rc, out, err = assert_python_ok('-c', code) + lines = out.splitlines() + for i, line in enumerate(lines, 1): + self.assertIn(b'MemoryError', out) + *_, count = line.split(b' ') + count = int(count) + self.assertLessEqual(count, i*5) + self.assertGreaterEqual(count, i*5-2) + + +class PyMemMallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'malloc_debug' + + + at unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') +class PyMemPymallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'pymalloc_debug' + + + at unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') +class PyMemDefaultTests(PyMemDebugTests): + # test default allocator of Python compiled in debug mode + PYTHONMALLOC = '' + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 06a51aa3cc21..dace37c362e5 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -323,42 +323,6 @@ def test_getitem_with_error(self): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() - def test_set_nomemory(self): - code = """if 1: - import _testcapi - - class C(): pass - - # The first loop tests both functions and that remove_mem_hooks() - # can be called twice in a row. The second loop checks a call to - # set_nomemory() after a call to remove_mem_hooks(). The third - # loop checks the start and stop arguments of set_nomemory(). - for outer_cnt in range(1, 4): - start = 10 * outer_cnt - for j in range(100): - if j == 0: - if outer_cnt != 3: - _testcapi.set_nomemory(start) - else: - _testcapi.set_nomemory(start, start + 1) - try: - C() - except MemoryError as e: - if outer_cnt != 3: - _testcapi.remove_mem_hooks() - print('MemoryError', outer_cnt, j) - _testcapi.remove_mem_hooks() - break - """ - rc, out, err = assert_python_ok('-c', code) - lines = out.splitlines() - for i, line in enumerate(lines, 1): - self.assertIn(b'MemoryError', out) - *_, count = line.split(b' ') - count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) - def test_mapping_keys_values_items(self): class Mapping1(dict): def keys(self): @@ -1470,124 +1434,6 @@ class Test_testinternalcapi(unittest.TestCase): if name.startswith('test_')) - at support.requires_subprocess() -class PyMemDebugTests(unittest.TestCase): - PYTHONMALLOC = 'debug' - # '0x04c06e0' or '04C06E0' - PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' - - def check(self, code): - with support.SuppressCrashReport(): - out = assert_python_failure( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - # FreeBSD: instruct jemalloc to not fill freed() memory - # with junk byte 0x5a, see JEMALLOC(3) - MALLOC_CONF="junk:false", - ) - stderr = out.err - return stderr.decode('ascii', 'replace') - - def test_buffer_overflow(self): - out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE $0x[0-9a-f]{{2}}$:\n" - r" at tail\+0: 0x78 \*\*\* OUCH\n" - r" at tail\+1: 0xfd\n" - r" at tail\+2: 0xfd\n" - r" .*\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") - regex = regex.format(ptr=self.PTR_REGEX) - regex = re.compile(regex, flags=re.DOTALL) - self.assertRegex(out, regex) - - def test_api_misuse(self): - out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") - regex = regex.format(ptr=self.PTR_REGEX) - self.assertRegex(out, regex) - - def check_malloc_without_gil(self, code): - out = self.check(code) - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without holding the GIL') - self.assertIn(expected, out) - - def test_pymem_malloc_without_gil(self): - # Debug hooks must raise an error if PyMem_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def test_pyobject_malloc_without_gil(self): - # Debug hooks must raise an error if PyObject_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def check_pyobject_is_freed(self, func_name): - code = textwrap.dedent(f''' - import gc, os, sys, _testcapi - # Disable the GC to avoid crash on GC collection - gc.disable() - try: - _testcapi.{func_name}() - # Exit immediately to avoid a crash while deallocating - # the invalid object - os._exit(0) - except _testcapi.error: - os._exit(1) - ''') - assert_python_ok( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - MALLOC_CONF="junk:false", - ) - - def test_pyobject_null_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_null_is_freed') - - def test_pyobject_uninitialized_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') - - def test_pyobject_forbidden_bytes_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') - - def test_pyobject_freed_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_freed_is_freed') - - -class PyMemMallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'malloc_debug' - - - at unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') -class PyMemPymallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'pymalloc_debug' - - - at unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') -class PyMemDefaultTests(PyMemDebugTests): - # test default allocator of Python compiled in debug mode - PYTHONMALLOC = '' - - @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") class Test_ModuleStateAccess(unittest.TestCase): """Test access to module start (PEP 573)""" diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index ef723bf06585..ae3f7a4372dc 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -596,6 +596,82 @@ check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) #endif } +// Tracemalloc tests +static PyObject * +tracemalloc_track(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + Py_ssize_t size; + int release_gil = 0; + + if (!PyArg_ParseTuple(args, "IOn|i", + &domain, &ptr_obj, &size, &release_gil)) + { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + int res; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); + Py_END_ALLOW_THREADS + } + else { + res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); + } + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_untrack(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_get_traceback(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); +} + static PyMethodDef test_methods[] = { {"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, @@ -617,6 +693,11 @@ static PyMethodDef test_methods[] = { {"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS}, {"test_pyobject_new", test_pyobject_new, METH_NOARGS}, {"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS}, + + // Tracemalloc tests + {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, + {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, + {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 83eef73a875d..35c895d9ceb2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2065,78 +2065,6 @@ getitem_with_error(PyObject *self, PyObject *args) return PyObject_GetItem(map, key); } -static PyObject * -tracemalloc_track(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - Py_ssize_t size; - int release_gil = 0; - int res; - - if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - if (release_gil) { - Py_BEGIN_ALLOW_THREADS - res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); - Py_END_ALLOW_THREADS - } - else { - res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); - } - - if (res < 0) { - PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error"); - return NULL; - } - - Py_RETURN_NONE; -} - -static PyObject * -tracemalloc_untrack(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - int res; - - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); - if (res < 0) { - PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); - return NULL; - } - - Py_RETURN_NONE; -} - -static PyObject * -tracemalloc_get_traceback(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); -} - static PyObject * dict_get_version(PyObject *self, PyObject *args) { @@ -3301,9 +3229,6 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, - {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, - {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, - {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, From webhook-mailer at python.org Sat Dec 17 07:33:06 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 17 Dec 2022 12:33:06 -0000 Subject: [Python-checkins] gh-100272: Fix JSON serialization of OrderedDict (GH-100273) Message-ID: <mailman.2963.1671280387.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0fe61d0838218b18bcff4bf434beba3e8dbcc42b commit: 0fe61d0838218b18bcff4bf434beba3e8dbcc42b branch: main author: Serhiy Storchaka <storchaka at gmail.com> committer: serhiy-storchaka <storchaka at gmail.com> date: 2022-12-17T14:32:48+02:00 summary: gh-100272: Fix JSON serialization of OrderedDict (GH-100273) It now preserves the order of keys. files: A Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst M Lib/test/test_json/test_default.py M Modules/_json.c diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 9b8325e9c381..3ce16684a082 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -1,3 +1,4 @@ +import collections from test.test_json import PyTest, CTest @@ -7,6 +8,16 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + def test_ordereddict(self): + od = collections.OrderedDict(a=1, b=2, c=3, d=4) + od.move_to_end('b') + self.assertEqual( + self.dumps(od), + '{"a": 1, "c": 3, "d": 4, "b": 2}') + self.assertEqual( + self.dumps(od, sort_keys=True), + '{"a": 1, "b": 2, "c": 3, "d": 4}') + class TestPyDefault(TestDefault, PyTest): pass class TestCDefault(TestDefault, CTest): pass diff --git a/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst b/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst new file mode 100644 index 000000000000..2fb089261792 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst @@ -0,0 +1 @@ +Fix JSON serialization of OrderedDict. It now preserves the order of keys. diff --git a/Modules/_json.c b/Modules/_json.c index 429b4ee0fa8d..6879ad3d0722 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1570,10 +1570,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, */ } - if (s->sort_keys) { - - items = PyDict_Items(dct); - if (items == NULL || PyList_Sort(items) < 0) + if (s->sort_keys || !PyDict_CheckExact(dct)) { + items = PyMapping_Items(dct); + if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) goto bail; for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { From webhook-mailer at python.org Sat Dec 17 20:10:37 2022 From: webhook-mailer at python.org (rhettinger) Date: Sun, 18 Dec 2022 01:10:37 -0000 Subject: [Python-checkins] [3.11] GH-98363: Update batched() recipe in docs to match 3.12 (#100323) Message-ID: <mailman.2964.1671325838.3313.python-checkins@python.org> https://github.com/python/cpython/commit/09186676cf3d483d19d64f5f1ed23b5a7e6edbbb commit: 09186676cf3d483d19d64f5f1ed23b5a7e6edbbb branch: 3.11 author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-17T19:10:03-06:00 summary: [3.11] GH-98363: Update batched() recipe in docs to match 3.12 (#100323) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index ee70b16f42fb..8eb843ab0a67 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -881,12 +881,12 @@ which incur interpreter overhead. raise ValueError('Expected fill, strict, or ignore') def batched(iterable, n): - "Batch data into lists of length n. The last batch may be shorter." + "Batch data into tuples of length n. The last batch may be shorter." # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: raise ValueError('n must be at least one') it = iter(iterable) - while (batch := list(islice(it, n))): + while (batch := tuple(islice(it, n))): yield batch def triplewise(iterable): @@ -1248,25 +1248,25 @@ which incur interpreter overhead. [('a', 'b', 'c'), ('d', 'e', 'f')] >>> list(batched('ABCDEFG', 3)) - [['A', 'B', 'C'], ['D', 'E', 'F'], ['G']] + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)] >>> list(batched('ABCDEF', 3)) - [['A', 'B', 'C'], ['D', 'E', 'F']] + [('A', 'B', 'C'), ('D', 'E', 'F')] >>> list(batched('ABCDE', 3)) - [['A', 'B', 'C'], ['D', 'E']] + [('A', 'B', 'C'), ('D', 'E')] >>> list(batched('ABCD', 3)) - [['A', 'B', 'C'], ['D']] + [('A', 'B', 'C'), ('D',)] >>> list(batched('ABC', 3)) - [['A', 'B', 'C']] + [('A', 'B', 'C')] >>> list(batched('AB', 3)) - [['A', 'B']] + [('A', 'B')] >>> list(batched('A', 3)) - [['A']] + [('A',)] >>> list(batched('', 3)) [] >>> list(batched('ABCDEFG', 2)) - [['A', 'B'], ['C', 'D'], ['E', 'F'], ['G']] + [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)] >>> list(batched('ABCDEFG', 1)) - [['A'], ['B'], ['C'], ['D'], ['E'], ['F'], ['G']] + [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)] >>> s = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' >>> all(list(flatten(batched(s[:n], 5))) == list(s[:n]) for n in range(len(s))) True From webhook-mailer at python.org Sun Dec 18 02:11:34 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 18 Dec 2022 07:11:34 -0000 Subject: [Python-checkins] Correct CVE-2020-10735 documentation (#100306) Message-ID: <mailman.2965.1671347495.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1cf3d78c92eb07dc09d15cc2e773b0b1b9436825 commit: 1cf3d78c92eb07dc09d15cc2e773b0b1b9436825 branch: main author: Jeremy Paige <ucodery at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-18T12:41:21+05:30 summary: Correct CVE-2020-10735 documentation (#100306) files: M Doc/library/stdtypes.rst M Python/clinic/sysmodule.c.h M Python/sysmodule.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c785336944f5..5a6b7d83edf7 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5480,7 +5480,7 @@ to mitigate denial of service attacks. This limit *only* applies to decimal or other non-power-of-two number bases. Hexadecimal, octal, and binary conversions are unlimited. The limit can be configured. -The :class:`int` type in CPython is an abitrary length number stored in binary +The :class:`int` type in CPython is an arbitrary length number stored in binary form (commonly known as a "bignum"). There exists no algorithm that can convert a string to a binary integer or a binary integer to a string in linear time, *unless* the base is a power of 2. Even the best known algorithms for base 10 @@ -5544,7 +5544,7 @@ and :class:`str` or :class:`bytes`: * ``int(string)`` with default base 10. * ``int(string, base)`` for all bases that are not a power of 2. * ``str(integer)``. -* ``repr(integer)`` +* ``repr(integer)``. * any other string conversion to base 10, for example ``f"{integer}"``, ``"{}".format(integer)``, or ``b"%d" % integer``. @@ -5572,7 +5572,7 @@ command line flag to configure the limit: :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. If both the env var and the ``-X`` option are set, the ``-X`` option takes precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initilization. + :data:`sys.int_info.default_max_str_digits` was used during initialization. From code, you can inspect the current limit and set a new one using these :mod:`sys` APIs: diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 5678d0ac2a60..03eeda8126eb 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -749,7 +749,7 @@ PyDoc_STRVAR(sys_get_int_max_str_digits__doc__, "get_int_max_str_digits($module, /)\n" "--\n" "\n" -"Set the maximum string digits limit for non-binary int<->str conversions."); +"Return the maximum string digits limit for non-binary int<->str conversions."); #define SYS_GET_INT_MAX_STR_DIGITS_METHODDEF \ {"get_int_max_str_digits", (PyCFunction)sys_get_int_max_str_digits, METH_NOARGS, sys_get_int_max_str_digits__doc__}, @@ -1318,4 +1318,4 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=79228e569529129c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b32b444538dfd354 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 91f5c487c98f..3f0baf98890b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1699,12 +1699,12 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] sys.get_int_max_str_digits -Set the maximum string digits limit for non-binary int<->str conversions. +Return the maximum string digits limit for non-binary int<->str conversions. [clinic start generated code]*/ static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=8dab13e2023e60d5]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromLong(interp->long_state.max_str_digits); From webhook-mailer at python.org Sun Dec 18 14:13:36 2022 From: webhook-mailer at python.org (gvanrossum) Date: Sun, 18 Dec 2022 19:13:36 -0000 Subject: [Python-checkins] gh-94912: Added marker for non-standard coroutine function detection (#99247) Message-ID: <mailman.2966.1671390817.3313.python-checkins@python.org> https://github.com/python/cpython/commit/532aa4e4e019812d0388920768ede7c04232ebe1 commit: 532aa4e4e019812d0388920768ede7c04232ebe1 branch: main author: Carlton Gibson <carlton at noumenal.es> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-18T11:13:24-08:00 summary: gh-94912: Added marker for non-standard coroutine function detection (#99247) This introduces a new decorator `@inspect.markcoroutinefunction`, which, applied to a sync function, makes it appear async to `inspect.iscoroutinefunction()`. files: A Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst M Doc/library/inspect.rst M Doc/whatsnew/3.12.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 6705577551dc..58b84a35a890 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -343,8 +343,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: iscoroutinefunction(object) - Return ``True`` if the object is a :term:`coroutine function` - (a function defined with an :keyword:`async def` syntax). + Return ``True`` if the object is a :term:`coroutine function` (a function + defined with an :keyword:`async def` syntax), a :func:`functools.partial` + wrapping a :term:`coroutine function`, or a sync function marked with + :func:`markcoroutinefunction`. .. versionadded:: 3.5 @@ -352,6 +354,25 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a :term:`coroutine function`. + .. versionchanged:: 3.12 + Sync functions marked with :func:`markcoroutinefunction` now return + ``True``. + + +.. function:: markcoroutinefunction(func) + + Decorator to mark a callable as a :term:`coroutine function` if it would not + otherwise be detected by :func:`iscoroutinefunction`. + + This may be of use for sync functions that return a :term:`coroutine`, if + the function is passed to an API that requires :func:`iscoroutinefunction`. + + When possible, using an :keyword:`async def` function is preferred. Also + acceptable is calling the function and testing the return with + :func:`iscoroutine`. + + .. versionadded:: 3.12 + .. function:: iscoroutine(object) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 73dc462f0b33..0cc4471364b6 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -225,6 +225,12 @@ asyncio a custom event loop factory. (Contributed by Kumar Aditya in :gh:`99388`.) +inspect +------- + +* Add :func:`inspect.markcoroutinefunction` to mark sync functions that return + a :term:`coroutine` for use with :func:`iscoroutinefunction`. + (Contributed Carlton Gibson in :gh:`99247`.) pathlib ------- diff --git a/Lib/inspect.py b/Lib/inspect.py index e92c355220fd..052f0bfe64b6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -125,6 +125,7 @@ "ismodule", "isroutine", "istraceback", + "markcoroutinefunction", "signature", "stack", "trace", @@ -391,12 +392,33 @@ def isgeneratorfunction(obj): See help(isfunction) for a list of attributes.""" return _has_code_flag(obj, CO_GENERATOR) +# A marker for markcoroutinefunction and iscoroutinefunction. +_is_coroutine_marker = object() + +def _has_coroutine_mark(f): + while ismethod(f): + f = f.__func__ + f = functools._unwrap_partial(f) + if not (isfunction(f) or _signature_is_functionlike(f)): + return False + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker + +def markcoroutinefunction(func): + """ + Decorator to ensure callable is recognised as a coroutine function. + """ + if hasattr(func, '__func__'): + func = func.__func__ + func._is_coroutine_marker = _is_coroutine_marker + return func + def iscoroutinefunction(obj): """Return true if the object is a coroutine function. - Coroutine functions are defined with "async def" syntax. + Coroutine functions are normally defined with "async def" syntax, but may + be marked via markcoroutinefunction. """ - return _has_code_flag(obj, CO_COROUTINE) + return _has_code_flag(obj, CO_COROUTINE) or _has_coroutine_mark(obj) def isasyncgenfunction(obj): """Return true if the object is an asynchronous generator function. diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 0f8217ba3b75..78e6e9e34e50 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -202,6 +202,51 @@ def test_iscoroutine(self): gen_coroutine_function_example)))) self.assertTrue(inspect.isgenerator(gen_coro)) + async def _fn3(): + pass + + @inspect.markcoroutinefunction + def fn3(): + return _fn3() + + self.assertTrue(inspect.iscoroutinefunction(fn3)) + self.assertTrue( + inspect.iscoroutinefunction( + inspect.markcoroutinefunction(lambda: _fn3()) + ) + ) + + class Cl: + async def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl)) + # instances with async def __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl())) + + class Cl2: + @inspect.markcoroutinefunction + def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl2)) + # instances with marked __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl2())) + + class Cl3: + @inspect.markcoroutinefunction + @classmethod + def do_something_classy(cls): + pass + + @inspect.markcoroutinefunction + @staticmethod + def do_something_static(): + pass + + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_classy)) + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_static)) + self.assertFalse( inspect.iscoroutinefunction(unittest.mock.Mock())) self.assertTrue( diff --git a/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst b/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst new file mode 100644 index 000000000000..ee00f9d8d03f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst @@ -0,0 +1,2 @@ +Add :func:`inspect.markcoroutinefunction` decorator which manually marks +a function as a coroutine for the benefit of :func:`iscoroutinefunction`. From webhook-mailer at python.org Mon Dec 19 03:40:22 2022 From: webhook-mailer at python.org (hugovk) Date: Mon, 19 Dec 2022 08:40:22 -0000 Subject: [Python-checkins] Docs: Don't upload CI artifacts (#100330) Message-ID: <mailman.2967.1671439223.3313.python-checkins@python.org> https://github.com/python/cpython/commit/702a5bc4637c82dc011e98b84f0cede98eb08dda commit: 702a5bc4637c82dc011e98b84f0cede98eb08dda branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-19T10:40:11+02:00 summary: Docs: Don't upload CI artifacts (#100330) files: M .github/workflows/doc.yml diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 44a1f206df1e..465da12fa1be 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -50,18 +50,8 @@ jobs: run: make -C Doc/ venv - name: 'Check documentation' run: make -C Doc/ check - - name: 'Upload NEWS' - uses: actions/upload-artifact at v3 - with: - name: NEWS - path: Doc/build/NEWS - name: 'Build HTML documentation' run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - - name: 'Upload docs' - uses: actions/upload-artifact at v3 - with: - name: doc-html - path: Doc/build/html # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: From webhook-mailer at python.org Mon Dec 19 04:49:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 19 Dec 2022 09:49:23 -0000 Subject: [Python-checkins] [3.11] gh-64490: Fix refcount error when arguments are packed to tuple in argument clinic (GH-99233) (#100338) Message-ID: <mailman.2968.1671443363.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c42a4ad58749273841ae6f504a4e661fb7c5ca3b commit: c42a4ad58749273841ae6f504a4e661fb7c5ca3b branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-19T15:18:49+05:30 summary: [3.11] gh-64490: Fix refcount error when arguments are packed to tuple in argument clinic (GH-99233) (#100338) (cherry picked from commit 69f6cc77d0f1664f983a83b6ae707d99a99f5c4f) files: A Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 0228d6be0326..8643f3fe78ae 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3339,7 +3339,7 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg a = args[0]; __clinic_args = PyTuple_New(nargs - 1); for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, args[1 + i]); + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } return_value = test_vararg_and_posonly_impl(module, a, __clinic_args); @@ -3350,7 +3350,7 @@ exit: static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=548bca3a127c22c1 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=081a953b8cbe7617 input=08dc2bf7afbf1613]*/ /*[clinic input] test_vararg diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 86c4f94ac028..af86d685866c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1211,6 +1211,20 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + def test_vararg_and_posonly(self): + with self.assertRaises(TypeError): + ac_tester.vararg_and_posonly() + with self.assertRaises(TypeError): + ac_tester.vararg_and_posonly(1, b=2) + self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_gh_99233_refcount(self): + arg = '*A unique string is not referenced by anywhere else.*' + arg_refcount_origin = sys.getrefcount(arg) + ac_tester.gh_99233_refcount(arg) + arg_refcount_after = sys.getrefcount(arg) + self.assertEqual(arg_refcount_origin, arg_refcount_after) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst b/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst new file mode 100644 index 000000000000..f98c181cc9c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-08-11-18-51.gh-issue-64490.VcBgrN.rst @@ -0,0 +1 @@ +Fix refcount error when arguments are packed to tuple in Argument Clinic. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index c9858e964457..a23ece2ae035 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -892,6 +892,41 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) } +/*[clinic input] +vararg_and_posonly + + a: object + *args: object + / + +[clinic start generated code]*/ + +static PyObject * +vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=42792f799465a14d input=defe017b19ba52e8]*/ +{ + return pack_arguments_newref(2, a, args); +} + + +/*[clinic input] +gh_99233_refcount + + *args: object + / + +Proof-of-concept of GH-99233 refcount error bug. + +[clinic start generated code]*/ + +static PyObject * +gh_99233_refcount_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=585855abfbca9a7f input=85f5fb47ac91a626]*/ +{ + Py_RETURN_NONE; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -933,6 +968,8 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF + VARARG_AND_POSONLY_METHODDEF + GH_99233_REFCOUNT_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index d363dd3f358e..c108d9d626a1 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1914,4 +1914,69 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=83f439d06635a2e9 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(vararg_and_posonly__doc__, +"vararg_and_posonly($module, a, /, *args)\n" +"--\n" +"\n"); + +#define VARARG_AND_POSONLY_METHODDEF \ + {"vararg_and_posonly", _PyCFunction_CAST(vararg_and_posonly), METH_FASTCALL, vararg_and_posonly__doc__}, + +static PyObject * +vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + __clinic_args = PyTuple_New(nargs - 1); + for (Py_ssize_t i = 0; i < nargs - 1; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); + } + return_value = vararg_and_posonly_impl(module, a, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(gh_99233_refcount__doc__, +"gh_99233_refcount($module, /, *args)\n" +"--\n" +"\n" +"Proof-of-concept of GH-99233 refcount error bug."); + +#define GH_99233_REFCOUNT_METHODDEF \ + {"gh_99233_refcount", _PyCFunction_CAST(gh_99233_refcount), METH_FASTCALL, gh_99233_refcount__doc__}, + +static PyObject * +gh_99233_refcount_impl(PyObject *module, PyObject *args); + +static PyObject * +gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("gh_99233_refcount", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = gh_99233_refcount_impl(module, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} +/*[clinic end generated code: output=16848e04bd5ddf7d input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 97d8d0a9410b..acd463bdd894 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -897,7 +897,7 @@ def parser_body(prototype, *fields, declarations=''): parser_code.append(normalize_snippet(""" %s = PyTuple_New(%s); for (Py_ssize_t i = 0; i < %s; ++i) {{ - PyTuple_SET_ITEM(%s, i, args[%d + i]); + PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); }} """ % ( p.converter.parser_name, From webhook-mailer at python.org Mon Dec 19 13:59:07 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Mon, 19 Dec 2022 18:59:07 -0000 Subject: [Python-checkins] gh-89727: Fix os.walk RecursionError on deep trees (#99803) Message-ID: <mailman.2969.1671476348.3313.python-checkins@python.org> https://github.com/python/cpython/commit/797edb28c3dd02a5727f0374e937e906a389ab77 commit: 797edb28c3dd02a5727f0374e937e906a389ab77 branch: main author: Jon Burdo <jon at jonburdo.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-19T10:59:01-08:00 summary: gh-89727: Fix os.walk RecursionError on deep trees (#99803) Use a stack to implement os.walk iteratively instead of recursively to avoid hitting recursion limits on deeply nested trees. files: A Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst M Lib/os.py M Lib/test/support/__init__.py M Lib/test/test_os.py diff --git a/Lib/os.py b/Lib/os.py index fd1e774fdcbc..73a5442ee8b8 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -340,89 +340,95 @@ def walk(top, topdown=True, onerror=None, followlinks=False): """ sys.audit("os.walk", top, topdown, onerror, followlinks) - return _walk(fspath(top), topdown, onerror, followlinks) - -def _walk(top, topdown, onerror, followlinks): - dirs = [] - nondirs = [] - walk_dirs = [] - - # We may not have read permission for top, in which case we can't - # get a list of the files the directory contains. os.walk - # always suppressed the exception then, rather than blow up for a - # minor reason when (say) a thousand readable directories are still - # left to visit. That logic is copied here. - try: - # Note that scandir is global in this module due - # to earlier import-*. - scandir_it = scandir(top) - except OSError as error: - if onerror is not None: - onerror(error) - return - with scandir_it: - while True: - try: + stack = [(False, fspath(top))] + islink, join = path.islink, path.join + while stack: + must_yield, top = stack.pop() + if must_yield: + yield top + continue + + dirs = [] + nondirs = [] + walk_dirs = [] + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. + # We suppress the exception here, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. + try: + scandir_it = scandir(top) + except OSError as error: + if onerror is not None: + onerror(error) + continue + + cont = False + with scandir_it: + while True: try: - entry = next(scandir_it) - except StopIteration: + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + cont = True break - except OSError as error: - if onerror is not None: - onerror(error) - return - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True + if is_dir: + dirs.append(entry.name) else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - - # Yield before recursion if going top down - if topdown: - yield top, dirs, nondirs - - # Recurse into sub-directories - islink, join = path.islink, path.join - for dirname in dirs: - new_path = join(top, dirname) - # Issue #23605: os.path.islink() is used instead of caching - # entry.is_symlink() result during the loop on os.scandir() because - # the caller can replace the directory entry during the "yield" - # above. - if followlinks or not islink(new_path): - yield from _walk(new_path, topdown, onerror, followlinks) - else: - # Recurse into sub-directories - for new_path in walk_dirs: - yield from _walk(new_path, topdown, onerror, followlinks) - # Yield after recursion if going bottom up - yield top, dirs, nondirs + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) + if cont: + continue + + if topdown: + # Yield before sub-directory traversal if going top down + yield top, dirs, nondirs + # Traverse into sub-directories + for dirname in reversed(dirs): + new_path = join(top, dirname) + # bpo-23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + stack.append((False, new_path)) + else: + # Yield after sub-directory traversal if going bottom up + stack.append((True, (top, dirs, nondirs))) + # Traverse into sub-directories + for new_path in reversed(walk_dirs): + stack.append((False, new_path)) __all__.append("walk") diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a631bfc80cfa..b7186057990a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2178,19 +2178,23 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) @contextlib.contextmanager +def set_recursion_limit(limit): + """Temporarily change the recursion limit.""" + original_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(limit) + yield + finally: + sys.setrecursionlimit(original_limit) + def infinite_recursion(max_depth=75): """Set a lower limit for tests that interact with infinite recursions (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some debug windows builds, due to not enough functions being inlined the stack size might not handle the default recursion limit (1000). See bpo-11105 for details.""" + return set_recursion_limit(max_depth) - original_depth = sys.getrecursionlimit() - try: - sys.setrecursionlimit(max_depth) - yield - finally: - sys.setrecursionlimit(original_depth) def ignore_deprecations_from(module: str, *, like: str) -> object: token = object() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e0577916428a..e6e25b507de0 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -33,6 +33,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper +from test.support import set_recursion_limit from test.support import warnings_helper from platform import win32_is_iot @@ -1471,6 +1472,46 @@ def test_walk_many_open_files(self): self.assertEqual(next(it), expected) p = os.path.join(p, 'd') + def test_walk_above_recursion_limit(self): + depth = 50 + os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) + with set_recursion_limit(depth - 5): + all = list(self.walk(self.walk_path)) + + sub2_path = self.sub2_tree[0] + for root, dirs, files in all: + if root == sub2_path: + dirs.sort() + files.sort() + + d_entries = [] + d_path = self.walk_path + for _ in range(depth): + d_path = os.path.join(d_path, "d") + d_entries.append((d_path, ["d"], [])) + d_entries[-1][1].clear() + + # Sub-sequences where the order is known + sections = { + "SUB1": [ + (self.sub1_path, ["SUB11"], ["tmp2"]), + (self.sub11_path, [], []), + ], + "SUB2": [self.sub2_tree], + "d": d_entries, + } + + # The ordering of sub-dirs is arbitrary but determines the order in + # which sub-sequences appear + dirs = all[0][1] + expected = [(self.walk_path, dirs, ["tmp1"])] + for d in dirs: + expected.extend(sections[d]) + + self.assertEqual(len(all), depth + 4) + self.assertEqual(sorted(dirs), ["SUB1", "SUB2", "d"]) + self.assertEqual(all, expected) + @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): @@ -1545,6 +1586,8 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None + # fwalk() still uses recursion + test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): diff --git a/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst new file mode 100644 index 000000000000..8a5fdb64b87f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.walk` where a :exc:`RecursionError` would occur on +deep directory structures by adjusting the implementation of +:func:`os.walk` to be iterative instead of recursive. From webhook-mailer at python.org Mon Dec 19 22:07:57 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Tue, 20 Dec 2022 03:07:57 -0000 Subject: [Python-checkins] gh-69929: re docs: Add more specific definition of \w (#92015) Message-ID: <mailman.2970.1671505677.3313.python-checkins@python.org> https://github.com/python/cpython/commit/36a0b1d0dddbdf324d98071ea31e7e9151eea6d5 commit: 36a0b1d0dddbdf324d98071ea31e7e9151eea6d5 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-19T19:07:31-08:00 summary: gh-69929: re docs: Add more specific definition of \w (#92015) Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index f7d46586cf75..cbee70b01d9f 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -591,10 +591,9 @@ character ``'$'``. ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched. + Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + as well as the underscore (``_``). + If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; From webhook-mailer at python.org Mon Dec 19 22:15:02 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 03:15:02 -0000 Subject: [Python-checkins] gh-69929: re docs: Add more specific definition of \w (GH-92015) Message-ID: <mailman.2971.1671506104.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cfa78ecc12255dd648c0d96bdeb4b74a1a686a95 commit: cfa78ecc12255dd648c0d96bdeb4b74a1a686a95 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-19T19:14:52-08:00 summary: gh-69929: re docs: Add more specific definition of \w (GH-92015) (cherry picked from commit 36a0b1d0dddbdf324d98071ea31e7e9151eea6d5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 8e279049a1c2..670569956522 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -589,10 +589,9 @@ character ``'$'``. ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched. + Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + as well as the underscore (``_``). + If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; From webhook-mailer at python.org Mon Dec 19 22:15:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 03:15:11 -0000 Subject: [Python-checkins] gh-69929: re docs: Add more specific definition of \w (GH-92015) Message-ID: <mailman.2972.1671506111.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b81d1c3be360c5f9a9dff0761e7f1f70a200f0aa commit: b81d1c3be360c5f9a9dff0761e7f1f70a200f0aa branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-19T19:14:48-08:00 summary: gh-69929: re docs: Add more specific definition of \w (GH-92015) (cherry picked from commit 36a0b1d0dddbdf324d98071ea31e7e9151eea6d5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index a40c99bf702f..716d3401a2bb 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -526,10 +526,9 @@ character ``'$'``. ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched. + Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + as well as the underscore (``_``). + If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; From webhook-mailer at python.org Tue Dec 20 02:10:39 2022 From: webhook-mailer at python.org (hugovk) Date: Tue, 20 Dec 2022 07:10:39 -0000 Subject: [Python-checkins] gh-89051: Add ssl.OP_LEGACY_SERVER_CONNECT (#93927) Message-ID: <mailman.2973.1671520239.3313.python-checkins@python.org> https://github.com/python/cpython/commit/79ccc03b62d819d83e592c6c8038545d9263a0d4 commit: 79ccc03b62d819d83e592c6c8038545d9263a0d4 branch: main author: Thomas Grainger <tagrain at gmail.com> committer: hugovk <hugovk at users.noreply.github.com> date: 2022-12-20T09:10:30+02:00 summary: gh-89051: Add ssl.OP_LEGACY_SERVER_CONNECT (#93927) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Christian Heimes <christian at python.org> Co-authored-by: Hugo van Kemenade <hugovk at users.noreply.github.com> Fixes https://github.com/python/cpython/issues/89051 files: A Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst M Doc/library/ssl.rst M Lib/test/test_ssl.py M Modules/_ssl.c diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 08824feeb395..78d44a23a83b 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -823,6 +823,13 @@ Constants .. versionadded:: 3.12 +.. data:: OP_LEGACY_SERVER_CONNECT + + Allow legacy insecure renegotiation between OpenSSL and unpatched servers + only. + + .. versionadded:: 3.12 + .. data:: HAS_ALPN Whether the OpenSSL library has built-in support for the *Application-Layer diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e926fc5e88e5..d4eb2d2e81fe 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1461,6 +1461,8 @@ def _assert_context_options(self, ctx): if OP_CIPHER_SERVER_PREFERENCE != 0: self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, OP_CIPHER_SERVER_PREFERENCE) + self.assertEqual(ctx.options & ssl.OP_LEGACY_SERVER_CONNECT, + 0 if IS_OPENSSL_3_0_0 else ssl.OP_LEGACY_SERVER_CONNECT) def test_create_default_context(self): ctx = ssl.create_default_context() @@ -3815,6 +3817,20 @@ def test_compression_disabled(self): sni_name=hostname) self.assertIs(stats['compression'], None) + def test_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_no_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options &= ~ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") def test_dh_params(self): # Check we can get a connection with ephemeral Diffie-Hellman diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst new file mode 100644 index 000000000000..5c8164863b81 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-06-17-08-00-34.gh-issue-89051.yP4Na0.rst @@ -0,0 +1 @@ +Add :data:`ssl.OP_LEGACY_SERVER_CONNECT` diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 591eb91dd0f3..8f03a846aed0 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -5845,6 +5845,8 @@ sslmodule_init_constants(PyObject *m) SSL_OP_CIPHER_SERVER_PREFERENCE); PyModule_AddIntConstant(m, "OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE); PyModule_AddIntConstant(m, "OP_NO_TICKET", SSL_OP_NO_TICKET); + PyModule_AddIntConstant(m, "OP_LEGACY_SERVER_CONNECT", + SSL_OP_LEGACY_SERVER_CONNECT); #ifdef SSL_OP_SINGLE_ECDH_USE PyModule_AddIntConstant(m, "OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE); #endif From webhook-mailer at python.org Tue Dec 20 05:10:37 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:10:37 -0000 Subject: [Python-checkins] gh-88211: Change lower-case and upper-case to match recommendations in imaplib docs (#99625) Message-ID: <mailman.2974.1671531038.3313.python-checkins@python.org> https://github.com/python/cpython/commit/39dfbb2d5dc47635f332bc13ca667293de6989ab commit: 39dfbb2d5dc47635f332bc13ca667293de6989ab branch: main author: Brad Wolfe <brad.wolfe at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T15:40:31+05:30 summary: gh-88211: Change lower-case and upper-case to match recommendations in imaplib docs (#99625) files: M Doc/library/imaplib.rst diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 0c10e7afee40..8c28fce99ff9 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -187,7 +187,7 @@ IMAP4 Objects ------------- All IMAP4rev1 commands are represented by methods of the same name, either -upper-case or lower-case. +uppercase or lowercase. All arguments to commands are converted to strings, except for ``AUTHENTICATE``, and the last argument to ``APPEND`` which is passed as an IMAP4 literal. If From webhook-mailer at python.org Tue Dec 20 05:20:16 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:20:16 -0000 Subject: [Python-checkins] [3.11] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100352) Message-ID: <mailman.2975.1671531616.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ba8e30c56ba4833f572318e1cf4108d9f206d1a0 commit: ba8e30c56ba4833f572318e1cf4108d9f206d1a0 branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T15:49:53+05:30 summary: [3.11] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100352) (cherry picked from commit 8dbe08eb7c807f484fe9870f5b7f5ae2881fd966) Fix double-free bug mentioned at GH-99240, by moving memory clean up out of "exit" label. files: A Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 8643f3fe78ae..991e82d4f966 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -1740,29 +1740,18 @@ test_str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } return_value = test_str_converter_encoding_impl(module, a, b, c, d, d_length, e, e_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + /* Post parse cleanup for d */ + PyMem_FREE(d); + /* Post parse cleanup for e */ + PyMem_FREE(e); exit: - /* Cleanup for a */ - if (a) { - PyMem_FREE(a); - } - /* Cleanup for b */ - if (b) { - PyMem_FREE(b); - } - /* Cleanup for c */ - if (c) { - PyMem_FREE(c); - } - /* Cleanup for d */ - if (d) { - PyMem_FREE(d); - } - /* Cleanup for e */ - if (e) { - PyMem_FREE(e); - } - return return_value; } @@ -1770,7 +1759,7 @@ static PyObject * test_str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, char *d, Py_ssize_t d_length, char *e, Py_ssize_t e_length) -/*[clinic end generated code: output=8acb886a3843f3bc input=eb4c38e1f898f402]*/ +/*[clinic end generated code: output=999c1deecfa15b0a input=eb4c38e1f898f402]*/ /*[clinic input] diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index af86d685866c..3f6cc6069669 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1045,6 +1045,17 @@ def test_str_converter(self): self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + def test_str_converter_encoding(self): + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding(1) + self.assertEqual(ac_tester.str_converter_encoding('a', 'b', 'c'), ('a', 'b', 'c')) + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c')])), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c'), 0, ord('c')])), + ('a', 'b', 'c\x00c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', b'c\x00c'), ('a', 'b', 'c\x00c')) + def test_py_buffer_converter(self): with self.assertRaises(TypeError): ac_tester.py_buffer_converter('a', 'b') @@ -1225,6 +1236,10 @@ def test_gh_99233_refcount(self): arg_refcount_after = sys.getrefcount(arg) self.assertEqual(arg_refcount_origin, arg_refcount_after) + def test_gh_99240_double_free(self): + expected_error = r'gh_99240_double_free argument 2 must be encoded string without null bytes, not str' + with self.assertRaisesRegex(TypeError, expected_error): + ac_tester.gh_99240_double_free('a', '\0b') if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst new file mode 100644 index 000000000000..0a4db052755f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst @@ -0,0 +1,2 @@ +Fix double-free bug in Argument Clinic ``str_converter`` by +extracting memory clean up to a new ``post_parsing`` section. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index a23ece2ae035..56eddfd6fdbf 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -551,6 +551,64 @@ str_converter_impl(PyObject *module, const char *a, const char *b, } +/*[clinic input] +str_converter_encoding + + a: str(encoding="idna") + b: str(encoding="idna", accept={bytes, bytearray, str}) + c: str(encoding="idna", accept={bytes, bytearray, str}, zeroes=True) + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_t c_length) +/*[clinic end generated code: output=af68766049248a1c input=0c5cf5159d0e870d]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + static PyObject * bytes_from_buffer(Py_buffer *buf) { @@ -927,6 +985,25 @@ gh_99233_refcount_impl(PyObject *module, PyObject *args) } +/*[clinic input] +gh_99240_double_free + + a: str(encoding="idna") + b: str(encoding="idna") + / + +Proof-of-concept of GH-99240 double-free bug. + +[clinic start generated code]*/ + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b) +/*[clinic end generated code: output=586dc714992fe2ed input=23db44aa91870fc7]*/ +{ + Py_RETURN_NONE; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -951,6 +1028,7 @@ static PyMethodDef tester_methods[] = { DOUBLE_CONVERTER_METHODDEF PY_COMPLEX_CONVERTER_METHODDEF STR_CONVERTER_METHODDEF + STR_CONVERTER_ENCODING_METHODDEF PY_BUFFER_CONVERTER_METHODDEF KEYWORDS_METHODDEF KEYWORDS_KWONLY_METHODDEF @@ -970,6 +1048,7 @@ static PyMethodDef tester_methods[] = { KEYWORD_ONLY_PARAMETER_METHODDEF VARARG_AND_POSONLY_METHODDEF GH_99233_REFCOUNT_METHODDEF + GH_99240_DOUBLE_FREE_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index c108d9d626a1..222ea783486e 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1159,6 +1159,43 @@ str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(str_converter_encoding__doc__, +"str_converter_encoding($module, a, b, c, /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_ENCODING_METHODDEF \ + {"str_converter_encoding", _PyCFunction_CAST(str_converter_encoding), METH_FASTCALL, str_converter_encoding__doc__}, + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_t c_length); + +static PyObject * +str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + char *c = NULL; + Py_ssize_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "esetet#:str_converter_encoding", + "idna", &a, "idna", &b, "idna", &c, &c_length)) { + goto exit; + } + return_value = str_converter_encoding_impl(module, a, b, c, c_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + +exit: + return return_value; +} + PyDoc_STRVAR(py_buffer_converter__doc__, "py_buffer_converter($module, a, b, /)\n" "--\n" @@ -1979,4 +2016,37 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_XDECREF(__clinic_args); return return_value; } -/*[clinic end generated code: output=16848e04bd5ddf7d input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gh_99240_double_free__doc__, +"gh_99240_double_free($module, a, b, /)\n" +"--\n" +"\n" +"Proof-of-concept of GH-99240 double-free bug."); + +#define GH_99240_DOUBLE_FREE_METHODDEF \ + {"gh_99240_double_free", _PyCFunction_CAST(gh_99240_double_free), METH_FASTCALL, gh_99240_double_free__doc__}, + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b); + +static PyObject * +gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + + if (!_PyArg_ParseStack(args, nargs, "eses:gh_99240_double_free", + "idna", &a, "idna", &b)) { + goto exit; + } + return_value = gh_99240_double_free_impl(module, a, b); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + +exit: + return return_value; +} +/*[clinic end generated code: output=4147b953eebc3c82 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index acd463bdd894..c49805ce1a7e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -349,6 +349,12 @@ def __init__(self): # "goto exit" if there are any. self.return_conversion = [] + # The C statements required to do some operations + # after the end of parsing but before cleaning up. + # These operations may be, for example, memory deallocations which + # can only be done without any error happening during argument parsing. + self.post_parsing = [] + # The C statements required to clean up after the impl call. self.cleanup = [] @@ -761,6 +767,7 @@ def parser_body(prototype, *fields, declarations=''): {modifications} {return_value} = {c_basename}_impl({impl_arguments}); {return_conversion} + {post_parsing} {exit_label} {cleanup} @@ -1405,6 +1412,7 @@ def render_function(self, clinic, f): template_dict['impl_parameters'] = ", ".join(data.impl_parameters) template_dict['impl_arguments'] = ", ".join(data.impl_arguments) template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) template_dict['cleanup'] = format_escape("".join(data.cleanup)) template_dict['return_value'] = data.return_value @@ -1429,6 +1437,7 @@ def render_function(self, clinic, f): return_conversion=template_dict['return_conversion'], initializers=template_dict['initializers'], modifications=template_dict['modifications'], + post_parsing=template_dict['post_parsing'], cleanup=template_dict['cleanup'], ) @@ -2660,6 +2669,10 @@ def _render_non_self(self, parameter, data): # parse_arguments self.parse_argument(data.parse_arguments) + # post_parsing + if post_parsing := self.post_parsing(): + data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') + # cleanup cleanup = self.cleanup() if cleanup: @@ -2755,6 +2768,14 @@ def modify(self): """ return "" + def post_parsing(self): + """ + The C statements required to do some operations after the end of parsing but before cleaning up. + Return a string containing this code indented at column 0. + If no operation is necessary, return an empty string. + """ + return "" + def cleanup(self): """ The C statements required to clean up after this variable. @@ -3351,10 +3372,10 @@ def converter_init(self, *, accept={str}, encoding=None, zeroes=False): if NoneType in accept and self.c_default == "Py_None": self.c_default = "NULL" - def cleanup(self): + def post_parsing(self): if self.encoding: name = self.name - return "".join(["if (", name, ") {\n PyMem_FREE(", name, ");\n}\n"]) + return f"PyMem_FREE({name});\n" def parse_arg(self, argname, displayname): if self.format_unit == 's': From webhook-mailer at python.org Tue Dec 20 05:20:48 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:20:48 -0000 Subject: [Python-checkins] [3.10] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100353) Message-ID: <mailman.2976.1671531649.3313.python-checkins@python.org> https://github.com/python/cpython/commit/53063b7ffa8f8edf13dd7c81f89f034a3b3b71b6 commit: 53063b7ffa8f8edf13dd7c81f89f034a3b3b71b6 branch: 3.10 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T15:50:42+05:30 summary: [3.10] gh-99240: Fix double-free bug in Argument Clinic str_converter generated code (GH-99241) (#100353) (cherry picked from commit 8dbe08eb7c807f484fe9870f5b7f5ae2881fd966) Fix double-free bug mentioned at GH-99240, by moving memory clean up out of "exit" label. files: A Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 3a3fbd174d86..af111dbee2b8 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -1740,29 +1740,18 @@ test_str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } return_value = test_str_converter_encoding_impl(module, a, b, c, d, d_length, e, e_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + /* Post parse cleanup for d */ + PyMem_FREE(d); + /* Post parse cleanup for e */ + PyMem_FREE(e); exit: - /* Cleanup for a */ - if (a) { - PyMem_FREE(a); - } - /* Cleanup for b */ - if (b) { - PyMem_FREE(b); - } - /* Cleanup for c */ - if (c) { - PyMem_FREE(c); - } - /* Cleanup for d */ - if (d) { - PyMem_FREE(d); - } - /* Cleanup for e */ - if (e) { - PyMem_FREE(e); - } - return return_value; } @@ -1770,7 +1759,7 @@ static PyObject * test_str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, char *d, Py_ssize_clean_t d_length, char *e, Py_ssize_clean_t e_length) -/*[clinic end generated code: output=f579dd9e795a364e input=eb4c38e1f898f402]*/ +/*[clinic end generated code: output=2d7e8b6203db31aa input=eb4c38e1f898f402]*/ /*[clinic input] diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 86c4f94ac028..feebb6add09a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1045,6 +1045,17 @@ def test_str_converter(self): self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + def test_str_converter_encoding(self): + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding(1) + self.assertEqual(ac_tester.str_converter_encoding('a', 'b', 'c'), ('a', 'b', 'c')) + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c')])), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c'), 0, ord('c')])), + ('a', 'b', 'c\x00c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', b'c\x00c'), ('a', 'b', 'c\x00c')) + def test_py_buffer_converter(self): with self.assertRaises(TypeError): ac_tester.py_buffer_converter('a', 'b') @@ -1211,6 +1222,11 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + def test_gh_99240_double_free(self): + expected_error = r'gh_99240_double_free argument 2 must be encoded string without null bytes, not str' + with self.assertRaisesRegex(TypeError, expected_error): + ac_tester.gh_99240_double_free('a', '\0b') + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst new file mode 100644 index 000000000000..0a4db052755f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-08-15-54-43.gh-issue-99240.MhYwcz.rst @@ -0,0 +1,2 @@ +Fix double-free bug in Argument Clinic ``str_converter`` by +extracting memory clean up to a new ``post_parsing`` section. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 63427ae6373e..079a40130b1d 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -551,6 +551,64 @@ str_converter_impl(PyObject *module, const char *a, const char *b, } +/*[clinic input] +str_converter_encoding + + a: str(encoding="idna") + b: str(encoding="idna", accept={bytes, bytearray, str}) + c: str(encoding="idna", accept={bytes, bytearray, str}, zeroes=True) + / + +[clinic start generated code]*/ + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_clean_t c_length) +/*[clinic end generated code: output=ebef1a3f9b730a24 input=0c5cf5159d0e870d]*/ +{ + assert(!PyErr_Occurred()); + PyObject *out[3] = {NULL,}; + int i = 0; + PyObject *arg; + + arg = PyUnicode_FromString(a); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromString(b); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + arg = PyUnicode_FromStringAndSize(c, c_length); + assert(arg || PyErr_Occurred()); + if (!arg) { + goto error; + } + out[i++] = arg; + + PyObject *tuple = PyTuple_New(3); + if (!tuple) { + goto error; + } + for (int j = 0; j < 3; j++) { + PyTuple_SET_ITEM(tuple, j, out[j]); + } + return tuple; + +error: + for (int j = 0; j < i; j++) { + Py_DECREF(out[j]); + } + return NULL; +} + + static PyObject * bytes_from_buffer(Py_buffer *buf) { @@ -892,6 +950,25 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) } +/*[clinic input] +gh_99240_double_free + + a: str(encoding="idna") + b: str(encoding="idna") + / + +Proof-of-concept of GH-99240 double-free bug. + +[clinic start generated code]*/ + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b) +/*[clinic end generated code: output=586dc714992fe2ed input=23db44aa91870fc7]*/ +{ + Py_RETURN_NONE; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -916,6 +993,7 @@ static PyMethodDef tester_methods[] = { DOUBLE_CONVERTER_METHODDEF PY_COMPLEX_CONVERTER_METHODDEF STR_CONVERTER_METHODDEF + STR_CONVERTER_ENCODING_METHODDEF PY_BUFFER_CONVERTER_METHODDEF KEYWORDS_METHODDEF KEYWORDS_KWONLY_METHODDEF @@ -933,6 +1011,7 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF + GH_99240_DOUBLE_FREE_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index a8273a5e6e2f..ed79fac50869 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1159,6 +1159,43 @@ str_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(str_converter_encoding__doc__, +"str_converter_encoding($module, a, b, c, /)\n" +"--\n" +"\n"); + +#define STR_CONVERTER_ENCODING_METHODDEF \ + {"str_converter_encoding", (PyCFunction)(void(*)(void))str_converter_encoding, METH_FASTCALL, str_converter_encoding__doc__}, + +static PyObject * +str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, + Py_ssize_clean_t c_length); + +static PyObject * +str_converter_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + char *c = NULL; + Py_ssize_clean_t c_length; + + if (!_PyArg_ParseStack(args, nargs, "esetet#:str_converter_encoding", + "idna", &a, "idna", &b, "idna", &c, &c_length)) { + goto exit; + } + return_value = str_converter_encoding_impl(module, a, b, c, c_length); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + /* Post parse cleanup for c */ + PyMem_FREE(c); + +exit: + return return_value; +} + PyDoc_STRVAR(py_buffer_converter__doc__, "py_buffer_converter($module, a, b, /)\n" "--\n" @@ -1914,4 +1951,37 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=2d5ac6b41daadf6f input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gh_99240_double_free__doc__, +"gh_99240_double_free($module, a, b, /)\n" +"--\n" +"\n" +"Proof-of-concept of GH-99240 double-free bug."); + +#define GH_99240_DOUBLE_FREE_METHODDEF \ + {"gh_99240_double_free", (PyCFunction)(void(*)(void))gh_99240_double_free, METH_FASTCALL, gh_99240_double_free__doc__}, + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b); + +static PyObject * +gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + + if (!_PyArg_ParseStack(args, nargs, "eses:gh_99240_double_free", + "idna", &a, "idna", &b)) { + goto exit; + } + return_value = gh_99240_double_free_impl(module, a, b); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + +exit: + return return_value; +} +/*[clinic end generated code: output=ecc55cbeac4adae6 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1a59fa57ac41..b0d1717596f6 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -346,6 +346,12 @@ def __init__(self): # "goto exit" if there are any. self.return_conversion = [] + # The C statements required to do some operations + # after the end of parsing but before cleaning up. + # These operations may be, for example, memory deallocations which + # can only be done without any error happening during argument parsing. + self.post_parsing = [] + # The C statements required to clean up after the impl call. self.cleanup = [] @@ -751,6 +757,7 @@ def parser_body(prototype, *fields, declarations=''): {modifications} {return_value} = {c_basename}_impl({impl_arguments}); {return_conversion} + {post_parsing} {exit_label} {cleanup} @@ -1343,6 +1350,7 @@ def render_function(self, clinic, f): template_dict['impl_parameters'] = ", ".join(data.impl_parameters) template_dict['impl_arguments'] = ", ".join(data.impl_arguments) template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) template_dict['cleanup'] = format_escape("".join(data.cleanup)) template_dict['return_value'] = data.return_value @@ -1367,6 +1375,7 @@ def render_function(self, clinic, f): return_conversion=template_dict['return_conversion'], initializers=template_dict['initializers'], modifications=template_dict['modifications'], + post_parsing=template_dict['post_parsing'], cleanup=template_dict['cleanup'], ) @@ -2595,6 +2604,10 @@ def _render_non_self(self, parameter, data): # parse_arguments self.parse_argument(data.parse_arguments) + # post_parsing + if post_parsing := self.post_parsing(): + data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') + # cleanup cleanup = self.cleanup() if cleanup: @@ -2686,6 +2699,14 @@ def modify(self): """ return "" + def post_parsing(self): + """ + The C statements required to do some operations after the end of parsing but before cleaning up. + Return a string containing this code indented at column 0. + If no operation is necessary, return an empty string. + """ + return "" + def cleanup(self): """ The C statements required to clean up after this variable. @@ -3278,10 +3299,10 @@ def converter_init(self, *, accept={str}, encoding=None, zeroes=False): if NoneType in accept and self.c_default == "Py_None": self.c_default = "NULL" - def cleanup(self): + def post_parsing(self): if self.encoding: name = self.name - return "".join(["if (", name, ") {\n PyMem_FREE(", name, ");\n}\n"]) + return f"PyMem_FREE({name});\n" def parse_arg(self, argname, displayname): if self.format_unit == 's': From webhook-mailer at python.org Tue Dec 20 05:41:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 20 Dec 2022 10:41:23 -0000 Subject: [Python-checkins] gh-100348: Fix ref cycle in `asyncio._SelectorSocketTransport` with `_read_ready_cb` (#100349) Message-ID: <mailman.2977.1671532884.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a6331b605e8044a205a113e1db87d2b0a53d0222 commit: a6331b605e8044a205a113e1db87d2b0a53d0222 branch: main author: Richard Kojedzinszky <rkojedzinszky at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-20T16:10:56+05:30 summary: gh-100348: Fix ref cycle in `asyncio._SelectorSocketTransport` with `_read_ready_cb` (#100349) files: A Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst M Lib/asyncio/selector_events.py diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 3d30006198f6..74f289f0e6f8 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1133,6 +1133,10 @@ def _make_empty_waiter(self): def _reset_empty_waiter(self): self._empty_waiter = None + def close(self): + self._read_ready_cb = None + super().close() + class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): diff --git a/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst b/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst new file mode 100644 index 000000000000..b5d4f7ca998c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-19-30-06.gh-issue-100348.o7IAHh.rst @@ -0,0 +1,2 @@ +Fix ref cycle in :class:`!asyncio._SelectorSocketTransport` by removing ``_read_ready_cb`` in ``close``. + From webhook-mailer at python.org Tue Dec 20 05:55:10 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Tue, 20 Dec 2022 10:55:10 -0000 Subject: [Python-checkins] gh-99925: Fix inconsistency in `json.dumps()` error messages (GH-99926) Message-ID: <mailman.2978.1671533711.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d98ca8172c39326bb200308a5191ceeb4a262d53 commit: d98ca8172c39326bb200308a5191ceeb4a262d53 branch: main author: Franti?ek Nesveda <fnesveda at users.noreply.github.com> committer: serhiy-storchaka <storchaka at gmail.com> date: 2022-12-20T12:54:56+02:00 summary: gh-99925: Fix inconsistency in `json.dumps()` error messages (GH-99926) files: A Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst M Lib/test/test_json/test_float.py M Modules/_json.c diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index d0c7214334d6..61540a3a02c2 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -26,7 +26,8 @@ def test_allow_nan(self): res = self.loads(out) self.assertEqual(len(res), 1) self.assertNotEqual(res[0], res[0]) - self.assertRaises(ValueError, self.dumps, [val], allow_nan=False) + msg = f'Out of range float values are not JSON compliant: {val}' + self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False) class TestPyFloat(TestFloat, PyTest): pass diff --git a/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst new file mode 100644 index 000000000000..660635a03963 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-01-15-44-58.gh-issue-99925.x4y6pF.rst @@ -0,0 +1,4 @@ +Unify error messages in JSON serialization between +``json.dumps(float('nan'), allow_nan=False)`` and ``json.dumps(float('nan'), +allow_nan=False, indent=<SOMETHING>)``. Now both include the representation +of the value that could not be serialized. diff --git a/Modules/_json.c b/Modules/_json.c index 6879ad3d0722..fa8e2a936d2c 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1319,9 +1319,10 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) double i = PyFloat_AS_DOUBLE(obj); if (!Py_IS_FINITE(i)) { if (!s->allow_nan) { - PyErr_SetString( + PyErr_Format( PyExc_ValueError, - "Out of range float values are not JSON compliant" + "Out of range float values are not JSON compliant: %R", + obj ); return NULL; } From webhook-mailer at python.org Tue Dec 20 06:36:26 2022 From: webhook-mailer at python.org (pablogsal) Date: Tue, 20 Dec 2022 11:36:26 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (#99246) Message-ID: <mailman.2979.1671536188.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cb60b6131bc2bb11c48a15f808914d8b242b9fc5 commit: cb60b6131bc2bb11c48a15f808914d8b242b9fc5 branch: main author: Pablo Galindo Salgado <Pablogsal at gmail.com> committer: pablogsal <Pablogsal at gmail.com> date: 2022-12-20T11:35:48Z summary: Clarify that every thread has its own default context in contextvars (#99246) files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 08a7c7d74eab..0ac2f3d85749 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 06:45:01 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 11:45:01 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (GH-99246) Message-ID: <mailman.2980.1671536702.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1332fdabbab75bc9e4bced064dc4daab2d7acb47 commit: 1332fdabbab75bc9e4bced064dc4daab2d7acb47 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T03:44:55-08:00 summary: Clarify that every thread has its own default context in contextvars (GH-99246) (cherry picked from commit cb60b6131bc2bb11c48a15f808914d8b242b9fc5) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 08a7c7d74eab..0ac2f3d85749 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 06:45:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 11:45:55 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (GH-99246) Message-ID: <mailman.2981.1671536756.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a757c1b7d3dfec875be15f8bd83777142d75b376 commit: a757c1b7d3dfec875be15f8bd83777142d75b376 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T03:45:48-08:00 summary: Clarify that every thread has its own default context in contextvars (GH-99246) (cherry picked from commit cb60b6131bc2bb11c48a15f808914d8b242b9fc5) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 08a7c7d74eab..0ac2f3d85749 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 06:57:14 2022 From: webhook-mailer at python.org (miss-islington) Date: Tue, 20 Dec 2022 11:57:14 -0000 Subject: [Python-checkins] Clarify that every thread has its own default context in contextvars (GH-99246) Message-ID: <mailman.2982.1671537435.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e8f61ede93ecbda3b2055a2e1d8c0b43b2049737 commit: e8f61ede93ecbda3b2055a2e1d8c0b43b2049737 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T03:57:08-08:00 summary: Clarify that every thread has its own default context in contextvars (GH-99246) (cherry picked from commit cb60b6131bc2bb11c48a15f808914d8b242b9fc5) Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com> files: M Doc/library/contextvars.rst diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index be1dd0c9eb57..3580b3537348 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,6 +144,11 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. + Every thread will have a different top-level :class:`~contextvars.Context` + object. This means that a :class:`ContextVar` object behaves in a similar + fashion to :func:`threading.local()` when values are assigned in different + threads. + Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) From webhook-mailer at python.org Tue Dec 20 11:51:32 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 20 Dec 2022 16:51:32 -0000 Subject: [Python-checkins] gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) Message-ID: <mailman.2983.1671555094.3313.python-checkins@python.org> https://github.com/python/cpython/commit/44892d45b038f919b0378590a776580a9d73b291 commit: 44892d45b038f919b0378590a776580a9d73b291 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: ambv <lukasz at langa.pl> date: 2022-12-20T17:51:26+01:00 summary: gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst M Lib/http/cookiejar.py M Lib/test/test_http_cookiejar.py diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index b0161a86fdbb..e3df007033b3 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1890,7 +1890,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. @@ -2081,7 +2084,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index f8291c2aa32c..97e9c82cde9e 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -397,6 +397,32 @@ def test_mozilla_filepermissions(self): finally: os_helper.unlink(filename) + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_cookie_files_are_truncated(self): + filename = os_helper.TESTFN + for cookiejar_class in (LWPCookieJar, MozillaCookieJar): + c = cookiejar_class(filename) + + req = urllib.request.Request("http://www.acme.com/") + headers = ["Set-Cookie: pll_lang=en; Max-Age=31536000; path=/"] + res = FakeResponse(headers, "http://www.acme.com/") + c.extract_cookies(res, req) + self.assertEqual(len(c), 1) + + try: + # Save the first version with contents: + c.save() + # Now, clear cookies and re-save: + c.clear() + c.save() + # Check that file was truncated: + c.load() + finally: + os_helper.unlink(filename) + + self.assertEqual(len(c), 0) + def test_bad_magic(self): # OSErrors (eg. file doesn't exist) are allowed to propagate filename = os_helper.TESTFN diff --git a/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst new file mode 100644 index 000000000000..9cbeb64b5625 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst @@ -0,0 +1,2 @@ +Fix ``.save()`` method for ``LWPCookieJar`` and ``MozillaCookieJar``: saved +file was not truncated on repeated save. From webhook-mailer at python.org Tue Dec 20 12:22:55 2022 From: webhook-mailer at python.org (ambv) Date: Tue, 20 Dec 2022 17:22:55 -0000 Subject: [Python-checkins] [3.11] gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) (GH-100377) Message-ID: <mailman.2984.1671556976.3313.python-checkins@python.org> https://github.com/python/cpython/commit/73d2b15bd278e84e89de83dbd2ee64eac2e63cda commit: 73d2b15bd278e84e89de83dbd2ee64eac2e63cda branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv <lukasz at langa.pl> date: 2022-12-20T18:22:39+01:00 summary: [3.11] gh-99576: Fix cookiejar file that was not truncated for some classes (GH-99616) (GH-100377) (cherry picked from commit 44892d45b038f919b0378590a776580a9d73b291) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst M Lib/http/cookiejar.py M Lib/test/test_http_cookiejar.py diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 65c45e2b17df..e622fc36cbfc 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1890,7 +1890,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. @@ -2086,7 +2089,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with os.fdopen(os.open(filename, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index f0d205a60a4c..b90612867fef 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -398,6 +398,32 @@ def test_mozilla_filepermissions(self): finally: os_helper.unlink(filename) + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_cookie_files_are_truncated(self): + filename = os_helper.TESTFN + for cookiejar_class in (LWPCookieJar, MozillaCookieJar): + c = cookiejar_class(filename) + + req = urllib.request.Request("http://www.acme.com/") + headers = ["Set-Cookie: pll_lang=en; Max-Age=31536000; path=/"] + res = FakeResponse(headers, "http://www.acme.com/") + c.extract_cookies(res, req) + self.assertEqual(len(c), 1) + + try: + # Save the first version with contents: + c.save() + # Now, clear cookies and re-save: + c.clear() + c.save() + # Check that file was truncated: + c.load() + finally: + os_helper.unlink(filename) + + self.assertEqual(len(c), 0) + def test_bad_magic(self): # OSErrors (eg. file doesn't exist) are allowed to propagate filename = os_helper.TESTFN diff --git a/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst new file mode 100644 index 000000000000..9cbeb64b5625 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-20-11-59-54.gh-issue-99576.ZD7jU6.rst @@ -0,0 +1,2 @@ +Fix ``.save()`` method for ``LWPCookieJar`` and ``MozillaCookieJar``: saved +file was not truncated on repeated save. From webhook-mailer at python.org Tue Dec 20 15:46:26 2022 From: webhook-mailer at python.org (sweeneyde) Date: Tue, 20 Dec 2022 20:46:26 -0000 Subject: [Python-checkins] gh-100188: Reduce misses in BINARY_SUBSCR_(LIST/TUPLE)_INT (#100189) Message-ID: <mailman.2985.1671569188.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c18d83118881333b9a0afd0add83afb2ba7300f7 commit: c18d83118881333b9a0afd0add83afb2ba7300f7 branch: main author: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com> committer: sweeneyde <36520290+sweeneyde at users.noreply.github.com> date: 2022-12-20T15:46:16-05:00 summary: gh-100188: Reduce misses in BINARY_SUBSCR_(LIST/TUPLE)_INT (#100189) Don't specialize if the index is negative. files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst M Python/specialize.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst new file mode 100644 index 000000000000..ec62fbd582fb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-05-30-12.gh-issue-100188.sGCSMR.rst @@ -0,0 +1,3 @@ +The ``BINARY_SUBSCR_LIST_INT`` and ``BINARY_SUBSCR_TUPLE_INT`` +instructions are no longer used for negative integers because +those instructions always miss when encountering negative integers. diff --git a/Python/specialize.c b/Python/specialize.c index a1666ccc9159..c6c502716478 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1302,8 +1302,12 @@ _Py_Specialize_BinarySubscr( PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opcode(instr, BINARY_SUBSCR_LIST_INT); - goto success; + if (Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) { + _py_set_opcode(instr, BINARY_SUBSCR_LIST_INT); + goto success; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; } SPECIALIZATION_FAIL(BINARY_SUBSCR, PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER); @@ -1311,8 +1315,12 @@ _Py_Specialize_BinarySubscr( } if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { - _py_set_opcode(instr, BINARY_SUBSCR_TUPLE_INT); - goto success; + if (Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) { + _py_set_opcode(instr, BINARY_SUBSCR_TUPLE_INT); + goto success; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; } SPECIALIZATION_FAIL(BINARY_SUBSCR, PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER); From webhook-mailer at python.org Tue Dec 20 21:02:20 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 02:02:20 -0000 Subject: [Python-checkins] gh-99991: improve docs on str.encode and bytes.decode (#100198) Message-ID: <mailman.2986.1671588141.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a2bb3b7f9d8d15c81b724726454d68357fb31d1c commit: a2bb3b7f9d8d15c81b724726454d68357fb31d1c branch: main author: Bisola Olasehinde <horlasehinde at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T18:02:14-08:00 summary: gh-99991: improve docs on str.encode and bytes.decode (#100198) Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 5a6b7d83edf7..624284f7092f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1624,25 +1624,28 @@ expression support in the :mod:`re` module). .. method:: str.encode(encoding="utf-8", errors="strict") - Return an encoded version of the string as a bytes object. Default encoding - is ``'utf-8'``. *errors* may be given to set a different error handling scheme. - The default for *errors* is ``'strict'``, meaning that encoding errors raise - a :exc:`UnicodeError`. Other possible - values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, - ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - By default, the *errors* argument is not checked for best performances, but - only used at the first encoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check - *errors*. + Return the string encoded to :class:`bytes`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how encoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, + ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any + other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. + + For performance reasons, the value of *errors* is not checked for validity + unless an encoding error actually occurs, + :ref:`devmode` is enabled + or a :ref:`debug build <debug-build>` is used. .. versionchanged:: 3.1 - Support for keyword arguments added. + Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. @@ -2759,29 +2762,32 @@ arbitrary binary data. .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") - Return a string decoded from the given bytes. Default encoding is - ``'utf-8'``. *errors* may be given to set a different - error handling scheme. The default for *errors* is ``'strict'``, meaning - that encoding errors raise a :exc:`UnicodeError`. Other possible values are - ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. + Return the bytes decoded to a :class:`str`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how decoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, ``'replace'``, + and any other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. - By default, the *errors* argument is not checked for best performances, but - only used at the first decoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check *errors*. + For performance reasons, the value of *errors* is not checked for validity + unless a decoding error actually occurs, + :ref:`devmode` is enabled or a :ref:`debug build <debug-build>` is used. .. note:: Passing the *encoding* argument to :class:`str` allows decoding any :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. + :class:`!bytes` or :class:`!bytearray` object. .. versionchanged:: 3.1 Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. From webhook-mailer at python.org Tue Dec 20 21:11:26 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 02:11:26 -0000 Subject: [Python-checkins] gh-99991: improve docs on str.encode and bytes.decode (GH-100198) Message-ID: <mailman.2987.1671588687.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b3d39c75990722aadd15586478d5dbe6882a2b76 commit: b3d39c75990722aadd15586478d5dbe6882a2b76 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T18:11:20-08:00 summary: gh-99991: improve docs on str.encode and bytes.decode (GH-100198) (cherry picked from commit a2bb3b7f9d8d15c81b724726454d68357fb31d1c) Co-authored-by: Bisola Olasehinde <horlasehinde at gmail.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a6a7eea9f071..d71d6936c568 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1585,25 +1585,28 @@ expression support in the :mod:`re` module). .. method:: str.encode(encoding="utf-8", errors="strict") - Return an encoded version of the string as a bytes object. Default encoding - is ``'utf-8'``. *errors* may be given to set a different error handling scheme. - The default for *errors* is ``'strict'``, meaning that encoding errors raise - a :exc:`UnicodeError`. Other possible - values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, - ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - By default, the *errors* argument is not checked for best performances, but - only used at the first encoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check - *errors*. + Return the string encoded to :class:`bytes`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how encoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, + ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any + other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. + + For performance reasons, the value of *errors* is not checked for validity + unless an encoding error actually occurs, + :ref:`devmode` is enabled + or a :ref:`debug build <debug-build>` is used. .. versionchanged:: 3.1 - Support for keyword arguments added. + Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. @@ -2715,29 +2718,32 @@ arbitrary binary data. .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") - Return a string decoded from the given bytes. Default encoding is - ``'utf-8'``. *errors* may be given to set a different - error handling scheme. The default for *errors* is ``'strict'``, meaning - that encoding errors raise a :exc:`UnicodeError`. Other possible values are - ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. + Return the bytes decoded to a :class:`str`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how decoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, ``'replace'``, + and any other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. - By default, the *errors* argument is not checked for best performances, but - only used at the first decoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check *errors*. + For performance reasons, the value of *errors* is not checked for validity + unless a decoding error actually occurs, + :ref:`devmode` is enabled or a :ref:`debug build <debug-build>` is used. .. note:: Passing the *encoding* argument to :class:`str` allows decoding any :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. + :class:`!bytes` or :class:`!bytearray` object. .. versionchanged:: 3.1 Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. From webhook-mailer at python.org Tue Dec 20 21:12:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 02:12:59 -0000 Subject: [Python-checkins] gh-99991: improve docs on str.encode and bytes.decode (GH-100198) Message-ID: <mailman.2988.1671588779.3313.python-checkins@python.org> https://github.com/python/cpython/commit/23fa1667b364a73ed29074f257d2bcf41bd35019 commit: 23fa1667b364a73ed29074f257d2bcf41bd35019 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T18:12:53-08:00 summary: gh-99991: improve docs on str.encode and bytes.decode (GH-100198) (cherry picked from commit a2bb3b7f9d8d15c81b724726454d68357fb31d1c) Co-authored-by: Bisola Olasehinde <horlasehinde at gmail.com> Co-authored-by: C.A.M. Gerlach <CAM.Gerlach at Gerlach.CAM> files: M Doc/library/stdtypes.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 426eec620566..a1514f611d4f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1623,25 +1623,28 @@ expression support in the :mod:`re` module). .. method:: str.encode(encoding="utf-8", errors="strict") - Return an encoded version of the string as a bytes object. Default encoding - is ``'utf-8'``. *errors* may be given to set a different error handling scheme. - The default for *errors* is ``'strict'``, meaning that encoding errors raise - a :exc:`UnicodeError`. Other possible - values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, - ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - By default, the *errors* argument is not checked for best performances, but - only used at the first encoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check - *errors*. + Return the string encoded to :class:`bytes`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how encoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, + ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any + other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. + + For performance reasons, the value of *errors* is not checked for validity + unless an encoding error actually occurs, + :ref:`devmode` is enabled + or a :ref:`debug build <debug-build>` is used. .. versionchanged:: 3.1 - Support for keyword arguments added. + Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. @@ -2755,29 +2758,32 @@ arbitrary binary data. .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") - Return a string decoded from the given bytes. Default encoding is - ``'utf-8'``. *errors* may be given to set a different - error handling scheme. The default for *errors* is ``'strict'``, meaning - that encoding errors raise a :exc:`UnicodeError`. Other possible values are - ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. + Return the bytes decoded to a :class:`str`. + + *encoding* defaults to ``'utf-8'``; + see :ref:`standard-encodings` for possible values. + + *errors* controls how decoding errors are handled. + If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. + Other possible values are ``'ignore'``, ``'replace'``, + and any other name registered via :func:`codecs.register_error`. + See :ref:`error-handlers` for details. - By default, the *errors* argument is not checked for best performances, but - only used at the first decoding error. Enable the :ref:`Python Development - Mode <devmode>`, or use a :ref:`debug build <debug-build>` to check *errors*. + For performance reasons, the value of *errors* is not checked for validity + unless a decoding error actually occurs, + :ref:`devmode` is enabled or a :ref:`debug build <debug-build>` is used. .. note:: Passing the *encoding* argument to :class:`str` allows decoding any :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. + :class:`!bytes` or :class:`!bytearray` object. .. versionchanged:: 3.1 Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and + The value of the *errors* argument is now checked in :ref:`devmode` and in :ref:`debug mode <debug-build>`. From webhook-mailer at python.org Tue Dec 20 22:27:07 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 03:27:07 -0000 Subject: [Python-checkins] gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (#91499) Message-ID: <mailman.2989.1671593228.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c615286e8576f2555d4380f38a966c300805b1a5 commit: c615286e8576f2555d4380f38a966c300805b1a5 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T19:27:02-08:00 summary: gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (#91499) Co-authored-by: Pieter Eendebak <P.T.eendebak at tudelft.nl> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/weakref.rst diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 73e7b21ae405..1406b663c6a8 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -172,6 +172,30 @@ See :ref:`__slots__ documentation <slots>` for details. application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. + Note that when a key with equal value to an existing key (but not equal identity) + is inserted into the dictionary, it replaces the value but does not replace the + existing key. Due to this, when the reference to the original key is deleted, it + also deletes the entry in the dictionary:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> d[k2] = 2 # d = {k1: 2} + >>> del k1 # d = {} + + A workaround would be to remove the key prior to reassignment:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> del d[k1] + >>> d[k2] = 2 # d = {k2: 2} + >>> del k1 # d = {k2: 2} + .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. From webhook-mailer at python.org Tue Dec 20 22:29:08 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 03:29:08 -0000 Subject: [Python-checkins] gh-85267: Improvements to inspect.signature __text_signature__ handling (#98796) Message-ID: <mailman.2990.1671593349.3313.python-checkins@python.org> https://github.com/python/cpython/commit/79311cbfe718f17c89bab67d7f89da3931bfa2ac commit: 79311cbfe718f17c89bab67d7f89da3931bfa2ac branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T19:29:02-08:00 summary: gh-85267: Improvements to inspect.signature __text_signature__ handling (#98796) This makes a couple related changes to inspect.signature's behaviour when parsing a signature from `__text_signature__`. First, `inspect.signature` is documented as only raising ValueError or TypeError. However, in some cases, we could raise RuntimeError. This PR changes that, thereby fixing #83685. (Note that the new ValueErrors in RewriteSymbolics are caught and then reraised with a message) Second, `inspect.signature` could randomly drop parameters that it didn't understand (corresponding to `return None` in the `p` function). This is the core issue in #85267. I think this is very surprising behaviour and it seems better to fail outright. Third, adding this new failure broke a couple tests. To fix them (and to e.g. allow `inspect.signature(select.epoll.register)` as in #85267), I add constant folding of a couple binary operations to RewriteSymbolics. (There's some discussion of making signature expression evaluation arbitrary powerful in #68155. I think that's out of scope. The additional constant folding here is pretty straightforward, useful, and not much of a slippery slope) Fourth, while #85267 is incorrect about the cause of the issue, it turns out if you had consecutive newlines in __text_signature__, you'd get `tokenize.TokenError`. Finally, the `if name is invalid:` code path was dead, since `parse_name` never returned `invalid`. files: A Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 052f0bfe64b6..3db7745e8a5e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2122,7 +2122,7 @@ def _signature_strip_non_python_syntax(signature): self_parameter = None last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) @@ -2221,11 +2221,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2235,7 +2235,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2245,6 +2245,21 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) if default_node and default_node is not _empty: @@ -2252,7 +2267,7 @@ def p(name_node, default_node, default=empty): default_node = RewriteSymbolics().visit(default_node) default = ast.literal_eval(default_node) except ValueError: - return None + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 78e6e9e34e50..1b589c8df2a2 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2525,7 +2525,7 @@ def p(name): return signature.parameters[name].default self.assertEqual(p('f'), False) self.assertEqual(p('local'), 3) self.assertEqual(p('sys'), sys.maxsize) - self.assertNotIn('exp', signature.parameters) + self.assertEqual(p('exp'), sys.maxsize - 1) test_callable(object) @@ -4323,10 +4323,29 @@ def func(*args, **kwargs): sig = inspect.signature(func) self.assertIsNotNone(sig) self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' sig = inspect.signature(func) self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + def test_base_class_have_text_signature(self): # see issue 43118 from test.ann_module7 import BufferedReader diff --git a/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst new file mode 100644 index 000000000000..e69fd1ca1c2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst @@ -0,0 +1,6 @@ +Several improvements to :func:`inspect.signature`'s handling of ``__text_signature``. +- Fixes a case where :func:`inspect.signature` dropped parameters +- Fixes a case where :func:`inspect.signature` raised :exc:`tokenize.TokenError` +- Allows :func:`inspect.signature` to understand defaults involving binary operations of constants +- :func:`inspect.signature` is documented as only raising :exc:`TypeError` or :exc:`ValueError`, but sometimes raised :exc:`RuntimeError`. These cases now raise :exc:`ValueError` +- Removed a dead code path From webhook-mailer at python.org Tue Dec 20 22:34:23 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 03:34:23 -0000 Subject: [Python-checkins] gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) Message-ID: <mailman.2991.1671593664.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fe828ec709ac9458697071b46c6d997cadacdbb5 commit: fe828ec709ac9458697071b46c6d997cadacdbb5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T19:34:17-08:00 summary: gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) (cherry picked from commit c615286e8576f2555d4380f38a966c300805b1a5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Pieter Eendebak <P.T.eendebak at tudelft.nl> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/weakref.rst diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 73e7b21ae405..1406b663c6a8 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -172,6 +172,30 @@ See :ref:`__slots__ documentation <slots>` for details. application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. + Note that when a key with equal value to an existing key (but not equal identity) + is inserted into the dictionary, it replaces the value but does not replace the + existing key. Due to this, when the reference to the original key is deleted, it + also deletes the entry in the dictionary:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> d[k2] = 2 # d = {k1: 2} + >>> del k1 # d = {} + + A workaround would be to remove the key prior to reassignment:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> del d[k1] + >>> d[k2] = 2 # d = {k2: 2} + >>> del k1 # d = {k2: 2} + .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. From webhook-mailer at python.org Tue Dec 20 22:34:29 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 03:34:29 -0000 Subject: [Python-checkins] gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) Message-ID: <mailman.2992.1671593669.3313.python-checkins@python.org> https://github.com/python/cpython/commit/561e15b85783b87cbc3023aa6cbebf5c76f60a64 commit: 561e15b85783b87cbc3023aa6cbebf5c76f60a64 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-20T19:34:24-08:00 summary: gh-91081: Add note on WeakKeyDictionary behavior when deleting a replaced entry (GH-91499) (cherry picked from commit c615286e8576f2555d4380f38a966c300805b1a5) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Pieter Eendebak <P.T.eendebak at tudelft.nl> Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Doc/library/weakref.rst diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 9a8289a7b533..ffe94be8087d 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -170,6 +170,30 @@ See :ref:`__slots__ documentation <slots>` for details. application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. + Note that when a key with equal value to an existing key (but not equal identity) + is inserted into the dictionary, it replaces the value but does not replace the + existing key. Due to this, when the reference to the original key is deleted, it + also deletes the entry in the dictionary:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> d[k2] = 2 # d = {k1: 2} + >>> del k1 # d = {} + + A workaround would be to remove the key prior to reassignment:: + + >>> class T(str): pass + ... + >>> k1, k2 = T(), T() + >>> d = weakref.WeakKeyDictionary() + >>> d[k1] = 1 # d = {k1: 1} + >>> del d[k1] + >>> d[k2] = 2 # d = {k2: 2} + >>> del k1 # d = {k2: 2} + .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. From webhook-mailer at python.org Wed Dec 21 00:25:19 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 05:25:19 -0000 Subject: [Python-checkins] [3.11] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100392) Message-ID: <mailman.2993.1671600321.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bee905184e9297e58d421ecfee556277dc01864d commit: bee905184e9297e58d421ecfee556277dc01864d branch: 3.11 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T21:25:13-08:00 summary: [3.11] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100392) This makes a couple related changes to inspect.signature's behaviour when parsing a signature from `__text_signature__`. First, `inspect.signature` is documented as only raising ValueError or TypeError. However, in some cases, we could raise RuntimeError. This PR changes that, thereby fixing GH-83685. (Note that the new ValueErrors in RewriteSymbolics are caught and then reraised with a message) Second, `inspect.signature` could randomly drop parameters that it didn't understand (corresponding to `return None` in the `p` function). This is the core issue in GH-85267. I think this is very surprising behaviour and it seems better to fail outright. Third, adding this new failure broke a couple tests. To fix them (and to e.g. allow `inspect.signature(select.epoll.register)` as in GH-85267), I add constant folding of a couple binary operations to RewriteSymbolics. (There's some discussion of making signature expression evaluation arbitrary powerful in GH-68155. I think that's out of scope. The additional constant folding here is pretty straightforward, useful, and not much of a slippery slope) Fourth, while GH-85267 is incorrect about the cause of the issue, it turns out if you had consecutive newlines in __text_signature__, you'd get `tokenize.TokenError`. Finally, the `if name is invalid:` code path was dead, since `parse_name` never returned `invalid`.. (cherry picked from commit 79311cbfe718f17c89bab67d7f89da3931bfa2ac) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index cab92b6b8fb8..bc49e680122d 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2116,7 +2116,7 @@ def _signature_strip_non_python_syntax(signature): self_parameter = None last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) @@ -2192,7 +2192,6 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): parameters = [] empty = Parameter.empty - invalid = object() module = None module_dict = {} @@ -2216,11 +2215,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2230,7 +2229,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2240,19 +2239,29 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) - if name is invalid: - return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) + default = ast.literal_eval(default_node) except ValueError: - o = invalid - if o is invalid: - return None - default = o if o is not invalid else default + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 6c5e674aa69f..9ea49854cbc6 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2480,7 +2480,7 @@ def p(name): return signature.parameters[name].default self.assertEqual(p('f'), False) self.assertEqual(p('local'), 3) self.assertEqual(p('sys'), sys.maxsize) - self.assertNotIn('exp', signature.parameters) + self.assertEqual(p('exp'), sys.maxsize - 1) test_callable(object) @@ -4245,10 +4245,29 @@ def func(*args, **kwargs): sig = inspect.signature(func) self.assertIsNotNone(sig) self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' sig = inspect.signature(func) self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + def test_base_class_have_text_signature(self): # see issue 43118 from test.ann_module7 import BufferedReader diff --git a/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst new file mode 100644 index 000000000000..e69fd1ca1c2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst @@ -0,0 +1,6 @@ +Several improvements to :func:`inspect.signature`'s handling of ``__text_signature``. +- Fixes a case where :func:`inspect.signature` dropped parameters +- Fixes a case where :func:`inspect.signature` raised :exc:`tokenize.TokenError` +- Allows :func:`inspect.signature` to understand defaults involving binary operations of constants +- :func:`inspect.signature` is documented as only raising :exc:`TypeError` or :exc:`ValueError`, but sometimes raised :exc:`RuntimeError`. These cases now raise :exc:`ValueError` +- Removed a dead code path From webhook-mailer at python.org Wed Dec 21 00:25:24 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 21 Dec 2022 05:25:24 -0000 Subject: [Python-checkins] [3.10] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100393) Message-ID: <mailman.2994.1671600325.3313.python-checkins@python.org> https://github.com/python/cpython/commit/919045cb73709a3933ec807c09b929c7ad7f02f4 commit: 919045cb73709a3933ec807c09b929c7ad7f02f4 branch: 3.10 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-20T21:25:18-08:00 summary: [3.10] gh-85267: Improvements to inspect.signature __text_signature__ handling (GH-98796) (#100393) This makes a couple related changes to inspect.signature's behaviour when parsing a signature from `__text_signature__`. First, `inspect.signature` is documented as only raising ValueError or TypeError. However, in some cases, we could raise RuntimeError. This PR changes that, thereby fixing GH-83685. (Note that the new ValueErrors in RewriteSymbolics are caught and then reraised with a message) Second, `inspect.signature` could randomly drop parameters that it didn't understand (corresponding to `return None` in the `p` function). This is the core issue in GH-85267. I think this is very surprising behaviour and it seems better to fail outright. Third, adding this new failure broke a couple tests. To fix them (and to e.g. allow `inspect.signature(select.epoll.register)` as in GH-85267), I add constant folding of a couple binary operations to RewriteSymbolics. (There's some discussion of making signature expression evaluation arbitrary powerful in GH-68155. I think that's out of scope. The additional constant folding here is pretty straightforward, useful, and not much of a slippery slope) Fourth, while GH-85267 is incorrect about the cause of the issue, it turns out if you had consecutive newlines in __text_signature__, you'd get `tokenize.TokenError`. Finally, the `if name is invalid:` code path was dead, since `parse_name` never returned `invalid`.. (cherry picked from commit 79311cbfe718f17c89bab67d7f89da3931bfa2ac) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 417371f5cf14..2999a6019e0f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2061,7 +2061,7 @@ def _signature_strip_non_python_syntax(signature): self_parameter = None last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) @@ -2141,7 +2141,6 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): parameters = [] empty = Parameter.empty - invalid = object() module = None module_dict = {} @@ -2165,11 +2164,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2179,7 +2178,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2189,19 +2188,29 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) - if name is invalid: - return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) + default = ast.literal_eval(default_node) except ValueError: - o = invalid - if o is invalid: - return None - default = o if o is not invalid else default + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 5aa4a60de0bf..5e430dad061b 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2477,7 +2477,7 @@ def p(name): return signature.parameters[name].default self.assertEqual(p('f'), False) self.assertEqual(p('local'), 3) self.assertEqual(p('sys'), sys.maxsize) - self.assertNotIn('exp', signature.parameters) + self.assertEqual(p('exp'), sys.maxsize - 1) test_callable(object) @@ -4246,10 +4246,29 @@ def func(*args, **kwargs): sig = inspect.signature(func) self.assertIsNotNone(sig) self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' sig = inspect.signature(func) self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + def test_base_class_have_text_signature(self): # see issue 43118 from test.ann_module7 import BufferedReader diff --git a/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst new file mode 100644 index 000000000000..e69fd1ca1c2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-28-07-24-34.gh-issue-85267.xUy_Wm.rst @@ -0,0 +1,6 @@ +Several improvements to :func:`inspect.signature`'s handling of ``__text_signature``. +- Fixes a case where :func:`inspect.signature` dropped parameters +- Fixes a case where :func:`inspect.signature` raised :exc:`tokenize.TokenError` +- Allows :func:`inspect.signature` to understand defaults involving binary operations of constants +- :func:`inspect.signature` is documented as only raising :exc:`TypeError` or :exc:`ValueError`, but sometimes raised :exc:`RuntimeError`. These cases now raise :exc:`ValueError` +- Removed a dead code path From webhook-mailer at python.org Wed Dec 21 04:49:51 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 09:49:51 -0000 Subject: [Python-checkins] GH-100363: Speed up `asyncio.get_running_loop` (#100364) Message-ID: <mailman.2995.1671616191.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4994f2488f8a436ebda3510c779cbfe292bb21a0 commit: 4994f2488f8a436ebda3510c779cbfe292bb21a0 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:19:44+05:30 summary: GH-100363: Speed up `asyncio.get_running_loop` (#100364) files: A Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst M Lib/asyncio/events.py M Modules/_asynciomodule.c diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 34a8869dff8d..6cff8c59ea47 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -836,6 +836,7 @@ def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _set_running_loop(None) signal.set_wakeup_fd(-1) os.register_at_fork(after_in_child=on_fork) diff --git a/Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst b/Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst new file mode 100644 index 000000000000..69bb52956137 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-20-11-07-30.gh-issue-100363.Wo_Beg.rst @@ -0,0 +1 @@ +Speed up :func:`asyncio.get_running_loop` by removing redundant ``getpid`` checks. Patch by Kumar Aditya. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 60369d89dc39..32be537c00a5 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -23,7 +23,6 @@ typedef struct { PyTypeObject *TaskStepMethWrapper_Type; PyTypeObject *FutureType; PyTypeObject *TaskType; - PyTypeObject *PyRunningLoopHolder_Type; PyObject *asyncio_mod; PyObject *context_kwname; @@ -59,8 +58,8 @@ typedef struct { /* Imports from traceback. */ PyObject *traceback_extract_stack; - PyObject *cached_running_holder; // Borrowed ref. - volatile uint64_t cached_running_holder_tsid; + PyObject *cached_running_loop; // Borrowed reference + volatile uint64_t cached_running_loop_tsid; /* Counter for autogenerated Task names */ uint64_t task_name_counter; @@ -138,14 +137,6 @@ typedef struct { PyObject *sw_arg; } TaskStepMethWrapper; -typedef struct { - PyObject_HEAD - PyObject *rl_loop; -#if defined(HAVE_GETPID) && !defined(MS_WINDOWS) - pid_t rl_pid; -#endif -} PyRunningLoopHolder; - #define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) #define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) @@ -165,8 +156,6 @@ class _asyncio.Future "FutureObj *" "&Future_Type" /* Get FutureIter from Future */ static PyObject * future_new_iter(PyObject *); -static PyRunningLoopHolder * new_running_loop_holder(asyncio_state *, PyObject *); - static int _is_coroutine(asyncio_state *state, PyObject *coro) @@ -264,11 +253,11 @@ get_running_loop(asyncio_state *state, PyObject **loop) PyThreadState *ts = _PyThreadState_GET(); uint64_t ts_id = PyThreadState_GetID(ts); - if (state->cached_running_holder_tsid == ts_id && - state->cached_running_holder != NULL) + if (state->cached_running_loop_tsid == ts_id && + state->cached_running_loop != NULL) { // Fast path, check the cache. - rl = state->cached_running_holder; // borrowed + rl = state->cached_running_loop; } else { PyObject *ts_dict = _PyThreadState_GetDict(ts); // borrowed @@ -287,27 +276,16 @@ get_running_loop(asyncio_state *state, PyObject **loop) } } - state->cached_running_holder = rl; // borrowed - state->cached_running_holder_tsid = ts_id; + state->cached_running_loop = rl; + state->cached_running_loop_tsid = ts_id; } - assert(Py_IS_TYPE(rl, state->PyRunningLoopHolder_Type)); - PyObject *running_loop = ((PyRunningLoopHolder *)rl)->rl_loop; - - if (running_loop == Py_None) { - goto not_found; - } -#if defined(HAVE_GETPID) && !defined(MS_WINDOWS) - /* On Windows there is no getpid, but there is also no os.fork(), - so there is no need for this check. - */ - if (getpid() != ((PyRunningLoopHolder *)rl)->rl_pid) { + if (rl == Py_None) { goto not_found; } -#endif - *loop = Py_NewRef(running_loop); + *loop = Py_NewRef(rl); return 0; not_found: @@ -335,22 +313,14 @@ set_running_loop(asyncio_state *state, PyObject *loop) PyExc_RuntimeError, "thread-local storage is not available"); return -1; } - - PyRunningLoopHolder *rl = new_running_loop_holder(state, loop); - if (rl == NULL) { - return -1; - } - if (PyDict_SetItem( - ts_dict, &_Py_ID(__asyncio_running_event_loop__), (PyObject *)rl) < 0) + ts_dict, &_Py_ID(__asyncio_running_event_loop__), loop) < 0) { - Py_DECREF(rl); // will cleanup loop & current_pid return -1; } - Py_DECREF(rl); - state->cached_running_holder = (PyObject *)rl; - state->cached_running_holder_tsid = PyThreadState_GetID(tstate); + state->cached_running_loop = loop; // borrowed, kept alive by ts_dict + state->cached_running_loop_tsid = PyThreadState_GetID(tstate); return 0; } @@ -3344,79 +3314,6 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) } -/*********************** PyRunningLoopHolder ********************/ - - -static PyRunningLoopHolder * -new_running_loop_holder(asyncio_state *state, PyObject *loop) -{ - PyRunningLoopHolder *rl = PyObject_GC_New( - PyRunningLoopHolder, state->PyRunningLoopHolder_Type); - if (rl == NULL) { - return NULL; - } - -#if defined(HAVE_GETPID) && !defined(MS_WINDOWS) - rl->rl_pid = getpid(); -#endif - rl->rl_loop = Py_NewRef(loop); - - PyObject_GC_Track(rl); - return rl; -} - - -static int -PyRunningLoopHolder_clear(PyRunningLoopHolder *rl) -{ - Py_CLEAR(rl->rl_loop); - return 0; -} - - -static int -PyRunningLoopHolder_traverse(PyRunningLoopHolder *rl, visitproc visit, - void *arg) -{ - Py_VISIT(Py_TYPE(rl)); - Py_VISIT(rl->rl_loop); - return 0; -} - - -static void -PyRunningLoopHolder_tp_dealloc(PyRunningLoopHolder *rl) -{ - asyncio_state *state = get_asyncio_state_by_def((PyObject *)rl); - if (state->cached_running_holder == (PyObject *)rl) { - state->cached_running_holder = NULL; - } - PyTypeObject *tp = Py_TYPE(rl); - PyObject_GC_UnTrack(rl); - PyRunningLoopHolder_clear(rl); - PyObject_GC_Del(rl); - Py_DECREF(tp); -} - - -static PyType_Slot PyRunningLoopHolder_slots[] = { - {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_dealloc, (destructor)PyRunningLoopHolder_tp_dealloc}, - {Py_tp_traverse, (traverseproc)PyRunningLoopHolder_traverse}, - {Py_tp_clear, PyRunningLoopHolder_clear}, - {0, NULL}, -}; - - -static PyType_Spec PyRunningLoopHolder_spec = { - .name = "_asyncio._RunningLoopHolder", - .basicsize = sizeof(PyRunningLoopHolder), - .slots = PyRunningLoopHolder_slots, - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), -}; - - /*********************** Module **************************/ @@ -3448,7 +3345,6 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->TaskStepMethWrapper_Type); Py_VISIT(state->FutureType); Py_VISIT(state->TaskType); - Py_VISIT(state->PyRunningLoopHolder_Type); Py_VISIT(state->asyncio_mod); Py_VISIT(state->traceback_extract_stack); @@ -3486,7 +3382,6 @@ module_clear(PyObject *mod) Py_CLEAR(state->TaskStepMethWrapper_Type); Py_CLEAR(state->FutureType); Py_CLEAR(state->TaskType); - Py_CLEAR(state->PyRunningLoopHolder_Type); Py_CLEAR(state->asyncio_mod); Py_CLEAR(state->traceback_extract_stack); @@ -3625,7 +3520,6 @@ module_exec(PyObject *mod) } while (0) CREATE_TYPE(mod, state->TaskStepMethWrapper_Type, &TaskStepMethWrapper_spec, NULL); - CREATE_TYPE(mod, state->PyRunningLoopHolder_Type, &PyRunningLoopHolder_spec, NULL); CREATE_TYPE(mod, state->FutureIterType, &FutureIter_spec, NULL); CREATE_TYPE(mod, state->FutureType, &Future_spec, NULL); CREATE_TYPE(mod, state->TaskType, &Task_spec, state->FutureType); From webhook-mailer at python.org Wed Dec 21 04:50:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 09:50:32 -0000 Subject: [Python-checkins] GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (#100154) Message-ID: <mailman.2996.1671616234.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a7715ccfba5b86ab09f86ec56ac3755c93b46b48 commit: a7715ccfba5b86ab09f86ec56ac3755c93b46b48 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:20:26+05:30 summary: GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (#100154) files: A Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst M Lib/asyncio/base_subprocess.py M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index e15bb4141fc0..4c9b0dd5653c 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -215,9 +215,6 @@ def _process_exited(self, returncode): # object. On Python 3.6, it is required to avoid a ResourceWarning. self._proc.returncode = returncode self._call(self._protocol.process_exited) - for p in self._pipes.values(): - if p is not None: - p.pipe.close() self._try_finish() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 7411f735da3b..3830dea7d9ba 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -686,6 +686,23 @@ async def execute(): self.assertIsNone(self.loop.run_until_complete(execute())) + def test_subprocess_communicate_stdout(self): + # See https://github.com/python/cpython/issues/100133 + async def get_command_stdout(cmd, *args): + proc = await asyncio.create_subprocess_exec( + cmd, *args, stdout=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout.decode().strip() + + async def main(): + outputs = [f'foo{i}' for i in range(10)] + res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', + f'print({out!r})') for out in outputs]) + self.assertEqual(res, outputs) + + self.loop.run_until_complete(main()) + if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst new file mode 100644 index 000000000000..881e6ed80fed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst @@ -0,0 +1 @@ +Fix regression in :mod:`asyncio` where a subprocess would sometimes lose data received from pipe. From webhook-mailer at python.org Wed Dec 21 05:02:46 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 10:02:46 -0000 Subject: [Python-checkins] [3.11] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100385) Message-ID: <mailman.2997.1671616967.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bed1d141a95e8d69f522e6f4e46e0647c68a2460 commit: bed1d141a95e8d69f522e6f4e46e0647c68a2460 branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:32:29+05:30 summary: [3.11] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100385) (cherry picked from commit efbb1eb9f54cad4f7bf5df03eed3a6aba02d99f4) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst M Lib/test/test_capi/test_getargs.py M Modules/_testcapimodule.c M Python/getargs.c diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 72b6d64a4986..552fa2135247 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1098,6 +1098,10 @@ def test_Z_hash(self): warnings.simplefilter('error', DeprecationWarning) self.assertRaises(DeprecationWarning, getargs_Z_hash, 'abc\xe9') + def test_gh_99240_clear_args(self): + from _testcapi import gh_99240_clear_args + self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') + class Object_TestCase(unittest.TestCase): def test_S(self): diff --git a/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst new file mode 100644 index 000000000000..9a1bb3cf5a73 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst @@ -0,0 +1,2 @@ +In argument parsing, after deallocating newly allocated memory, reset its +pointer to NULL. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1291eff481cb..70242d78ab8c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6192,6 +6192,7 @@ function_get_module(PyObject *self, PyObject *func) static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); static PyObject *getargs_s_hash_int2(PyObject *, PyObject *, PyObject*); +static PyObject *gh_99240_clear_args(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -6308,6 +6309,7 @@ static PyMethodDef TestMethods[] = { METH_VARARGS|METH_KEYWORDS}, {"getargs_s_hash_int2", _PyCFunction_CAST(getargs_s_hash_int2), METH_VARARGS|METH_KEYWORDS}, + {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, {"getargs_z", getargs_z, METH_VARARGS}, {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, @@ -8089,3 +8091,21 @@ getargs_s_hash_int2(PyObject *self, PyObject *args, PyObject *kwargs) PyBuffer_Release(&buf); Py_RETURN_NONE; } + +static PyObject * +gh_99240_clear_args(PyObject *self, PyObject *args) +{ + char *a = NULL; + char *b = NULL; + + if (!PyArg_ParseTuple(args, "eses", "idna", &a, "idna", &b)) { + if (a || b) { + PyErr_Clear(); + PyErr_SetString(PyExc_AssertionError, "Arguments are not cleared."); + } + return NULL; + } + PyMem_Free(a); + PyMem_Free(b); + Py_RETURN_NONE; +} diff --git a/Python/getargs.c b/Python/getargs.c index c3e3cf155b87..9d6483f4fe0f 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -202,9 +202,9 @@ _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) static int cleanup_ptr(PyObject *self, void *ptr) { - if (ptr) { - PyMem_Free(ptr); - } + void **pptr = (void **)ptr; + PyMem_Free(*pptr); + *pptr = NULL; return 0; } @@ -1169,7 +1169,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( "(cleanup problem)", @@ -1215,7 +1215,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr("(cleanup problem)", arg, msgbuf, bufsize); From webhook-mailer at python.org Wed Dec 21 05:03:26 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 10:03:26 -0000 Subject: [Python-checkins] [3.10] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100386) Message-ID: <mailman.2998.1671617007.3313.python-checkins@python.org> https://github.com/python/cpython/commit/591365cd49cf1065e27bb1358aa5f07bd854dd33 commit: 591365cd49cf1065e27bb1358aa5f07bd854dd33 branch: 3.10 author: colorfulappl <colorfulappl at qq.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T15:33:21+05:30 summary: [3.10] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (GH-99890) (#100386) (cherry picked from commit efbb1eb9f54cad4f7bf5df03eed3a6aba02d99f4) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland at protonmail.com> files: A Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst M Lib/test/test_capi/test_getargs.py M Modules/_testcapimodule.c M Python/getargs.c diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 72b6d64a4986..552fa2135247 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1098,6 +1098,10 @@ def test_Z_hash(self): warnings.simplefilter('error', DeprecationWarning) self.assertRaises(DeprecationWarning, getargs_Z_hash, 'abc\xe9') + def test_gh_99240_clear_args(self): + from _testcapi import gh_99240_clear_args + self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') + class Object_TestCase(unittest.TestCase): def test_S(self): diff --git a/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst new file mode 100644 index 000000000000..9a1bb3cf5a73 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst @@ -0,0 +1,2 @@ +In argument parsing, after deallocating newly allocated memory, reset its +pointer to NULL. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 20c90c441e6b..c0c86722eb3c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5671,6 +5671,7 @@ test_fatal_error(PyObject *self, PyObject *args) static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); static PyObject *getargs_s_hash_int2(PyObject *, PyObject *, PyObject*); +static PyObject *gh_99240_clear_args(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -5784,6 +5785,7 @@ static PyMethodDef TestMethods[] = { METH_VARARGS|METH_KEYWORDS}, {"getargs_s_hash_int2", (PyCFunction)(void(*)(void))getargs_s_hash_int2, METH_VARARGS|METH_KEYWORDS}, + {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, {"getargs_z", getargs_z, METH_VARARGS}, {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, @@ -7500,3 +7502,21 @@ getargs_s_hash_int2(PyObject *self, PyObject *args, PyObject *kwargs) PyBuffer_Release(&buf); Py_RETURN_NONE; } + +static PyObject * +gh_99240_clear_args(PyObject *self, PyObject *args) +{ + char *a = NULL; + char *b = NULL; + + if (!PyArg_ParseTuple(args, "eses", "idna", &a, "idna", &b)) { + if (a || b) { + PyErr_Clear(); + PyErr_SetString(PyExc_AssertionError, "Arguments are not cleared."); + } + return NULL; + } + PyMem_Free(a); + PyMem_Free(b); + Py_RETURN_NONE; +} diff --git a/Python/getargs.c b/Python/getargs.c index fd49ee0f05d4..3b65cfad54f9 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -201,9 +201,9 @@ _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) static int cleanup_ptr(PyObject *self, void *ptr) { - if (ptr) { - PyMem_Free(ptr); - } + void **pptr = (void **)ptr; + PyMem_Free(*pptr); + *pptr = NULL; return 0; } @@ -1168,7 +1168,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( "(cleanup problem)", @@ -1214,7 +1214,7 @@ _Py_COMP_DIAG_POP PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr("(cleanup problem)", arg, msgbuf, bufsize); From webhook-mailer at python.org Wed Dec 21 05:24:25 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 10:24:25 -0000 Subject: [Python-checkins] GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (GH-100154) Message-ID: <mailman.2999.1671618265.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ae8520c70992710903819f24dbce4e7dd05d7ea8 commit: ae8520c70992710903819f24dbce4e7dd05d7ea8 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T02:24:19-08:00 summary: GH-100133: fix `asyncio` subprocess losing `stderr` and `stdout` output (GH-100154) (cherry picked from commit a7715ccfba5b86ab09f86ec56ac3755c93b46b48) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst M Lib/asyncio/base_subprocess.py M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index e15bb4141fc0..4c9b0dd5653c 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -215,9 +215,6 @@ def _process_exited(self, returncode): # object. On Python 3.6, it is required to avoid a ResourceWarning. self._proc.returncode = returncode self._call(self._protocol.process_exited) - for p in self._pipes.values(): - if p is not None: - p.pipe.close() self._try_finish() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index f71ad72f999e..bea2314a5286 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -684,6 +684,23 @@ async def execute(): self.assertIsNone(self.loop.run_until_complete(execute())) + def test_subprocess_communicate_stdout(self): + # See https://github.com/python/cpython/issues/100133 + async def get_command_stdout(cmd, *args): + proc = await asyncio.create_subprocess_exec( + cmd, *args, stdout=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout.decode().strip() + + async def main(): + outputs = [f'foo{i}' for i in range(10)] + res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', + f'print({out!r})') for out in outputs]) + self.assertEqual(res, outputs) + + self.loop.run_until_complete(main()) + if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst new file mode 100644 index 000000000000..881e6ed80fed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-10-08-36-07.gh-issue-100133.g-zQlp.rst @@ -0,0 +1 @@ +Fix regression in :mod:`asyncio` where a subprocess would sometimes lose data received from pipe. From webhook-mailer at python.org Wed Dec 21 08:25:24 2022 From: webhook-mailer at python.org (corona10) Date: Wed, 21 Dec 2022 13:25:24 -0000 Subject: [Python-checkins] gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) Message-ID: <mailman.3000.1671629125.3313.python-checkins@python.org> https://github.com/python/cpython/commit/12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2 commit: 12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2 branch: main author: Dominic Socular <BBH at awsl.rip> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-21T22:25:04+09:00 summary: gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst M Lib/socket.py M Lib/test/test_socket.py diff --git a/Lib/socket.py b/Lib/socket.py index 1c8cef6ce658..3a4f94de9cc0 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -785,11 +785,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index d808f3f62b96..f1b4018c265e 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1769,6 +1769,10 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst new file mode 100644 index 000000000000..e78352fb188e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst @@ -0,0 +1 @@ +Fix incorrect result and delay in :func:`socket.getfqdn`. Patch by Dominic Socular. From webhook-mailer at python.org Wed Dec 21 09:06:23 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:06:23 -0000 Subject: [Python-checkins] gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) Message-ID: <mailman.3001.1671631584.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b2076b00710c4366dcfe6cd236e480d68a3c38b7 commit: b2076b00710c4366dcfe6cd236e480d68a3c38b7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:06:10-08:00 summary: gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) (cherry picked from commit 12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2) Co-authored-by: Dominic Socular <BBH at awsl.rip> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst M Lib/socket.py M Lib/test/test_socket.py diff --git a/Lib/socket.py b/Lib/socket.py index 0717c696b12e..5a896bee7c49 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -784,11 +784,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index b07954989fdd..13cb2a7945d1 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1760,6 +1760,10 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst new file mode 100644 index 000000000000..e78352fb188e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst @@ -0,0 +1 @@ +Fix incorrect result and delay in :func:`socket.getfqdn`. Patch by Dominic Socular. From webhook-mailer at python.org Wed Dec 21 09:06:33 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:06:33 -0000 Subject: [Python-checkins] gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) Message-ID: <mailman.3002.1671631594.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7f24056dfbcaa080ab825924667e9359d86570d9 commit: 7f24056dfbcaa080ab825924667e9359d86570d9 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:06:27-08:00 summary: gh-100374: Fixed a bug in socket.getfqdn() (gh-100375) (cherry picked from commit 12be23cf3c1301be2c6b8fd4cb2cd35a567d2ea2) Co-authored-by: Dominic Socular <BBH at awsl.rip> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst M Lib/socket.py M Lib/test/test_socket.py diff --git a/Lib/socket.py b/Lib/socket.py index 63ba0acc908c..310b074ca278 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -783,11 +783,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index e0e0f2437bb0..211fd8c02da0 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1743,6 +1743,10 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst new file mode 100644 index 000000000000..e78352fb188e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-16-14-19.gh-issue-100374.YRrVHT.rst @@ -0,0 +1 @@ +Fix incorrect result and delay in :func:`socket.getfqdn`. Patch by Dominic Socular. From webhook-mailer at python.org Wed Dec 21 09:31:28 2022 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 21 Dec 2022 14:31:28 -0000 Subject: [Python-checkins] gh-100129: Add tests for pickling all builtin types and functions (GH-100142) Message-ID: <mailman.3003.1671633090.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da commit: b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da branch: main author: Serhiy Storchaka <storchaka at gmail.com> committer: serhiy-storchaka <storchaka at gmail.com> date: 2022-12-21T16:31:22+02:00 summary: gh-100129: Add tests for pickling all builtin types and functions (GH-100142) files: M Lib/test/pickletester.py diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 499f80a15f34..6e87370c2065 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,3 +1,4 @@ +import builtins import collections import copyreg import dbm @@ -11,6 +12,7 @@ import struct import sys import threading +import types import unittest import weakref from textwrap import dedent @@ -1980,6 +1982,33 @@ def test_singleton_types(self): u = self.loads(s) self.assertIs(type(singleton), u) + def test_builtin_types(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and not issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + + def test_builtin_exceptions(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) + + def test_builtin_functions(self): + for t in builtins.__dict__.values(): + if isinstance(t, types.BuiltinFunctionType): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + # Tests for protocol 2 def test_proto(self): From webhook-mailer at python.org Wed Dec 21 09:58:39 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:58:39 -0000 Subject: [Python-checkins] gh-100129: Add tests for pickling all builtin types and functions (GH-100142) Message-ID: <mailman.3004.1671634720.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0397f040e239c82848f51dedef9af7b9bd99ef55 commit: 0397f040e239c82848f51dedef9af7b9bd99ef55 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:58:20-08:00 summary: gh-100129: Add tests for pickling all builtin types and functions (GH-100142) (cherry picked from commit b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/test/pickletester.py diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 499f80a15f34..6e87370c2065 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,3 +1,4 @@ +import builtins import collections import copyreg import dbm @@ -11,6 +12,7 @@ import struct import sys import threading +import types import unittest import weakref from textwrap import dedent @@ -1980,6 +1982,33 @@ def test_singleton_types(self): u = self.loads(s) self.assertIs(type(singleton), u) + def test_builtin_types(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and not issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + + def test_builtin_exceptions(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) + + def test_builtin_functions(self): + for t in builtins.__dict__.values(): + if isinstance(t, types.BuiltinFunctionType): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + # Tests for protocol 2 def test_proto(self): From webhook-mailer at python.org Wed Dec 21 09:58:39 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 14:58:39 -0000 Subject: [Python-checkins] gh-100129: Add tests for pickling all builtin types and functions (GH-100142) Message-ID: <mailman.3005.1671634721.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2dda6680ccb50bd1d8a8dcf0d524feb7daf5a3af commit: 2dda6680ccb50bd1d8a8dcf0d524feb7daf5a3af branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T06:58:31-08:00 summary: gh-100129: Add tests for pickling all builtin types and functions (GH-100142) (cherry picked from commit b98d2d31bffcaeb0c4c8848a8d1b35419c70b2da) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/test/pickletester.py diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 18d7f52ecffd..72d6f52c90de 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,3 +1,4 @@ +import builtins import collections import copyreg import dbm @@ -11,6 +12,7 @@ import struct import sys import threading +import types import unittest import weakref from textwrap import dedent @@ -1979,6 +1981,33 @@ def test_singleton_types(self): u = self.loads(s) self.assertIs(type(singleton), u) + def test_builtin_types(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and not issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + + def test_builtin_exceptions(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) + + def test_builtin_functions(self): + for t in builtins.__dict__.values(): + if isinstance(t, types.BuiltinFunctionType): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + # Tests for protocol 2 def test_proto(self): From webhook-mailer at python.org Wed Dec 21 11:55:10 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 16:55:10 -0000 Subject: [Python-checkins] Remove unused variable from `dis._find_imports` (#100396) Message-ID: <mailman.3006.1671641710.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e0b4d966a8d1867c4b535b043e08288ca49b3548 commit: e0b4d966a8d1867c4b535b043e08288ca49b3548 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T22:25:02+05:30 summary: Remove unused variable from `dis._find_imports` (#100396) files: M Lib/dis.py diff --git a/Lib/dis.py b/Lib/dis.py index 523bd01d9295..76104c6098d4 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -666,7 +666,6 @@ def _find_imports(co): the corresponding args to __import__. """ IMPORT_NAME = opmap['IMPORT_NAME'] - LOAD_CONST = opmap['LOAD_CONST'] consts = co.co_consts names = co.co_names From webhook-mailer at python.org Wed Dec 21 12:31:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 17:31:23 -0000 Subject: [Python-checkins] gh-78878: Fix crash when creating an instance of `_ctypes.CField` (#14837) Message-ID: <mailman.3007.1671643885.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d713c54ac8a2eba0616a5a07714696d935f1062e commit: d713c54ac8a2eba0616a5a07714696d935f1062e branch: main author: Hai Shi <shihai1992 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T23:01:17+05:30 summary: gh-78878: Fix crash when creating an instance of `_ctypes.CField` (#14837) files: A Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst M Lib/test/test_ctypes/test_struct_fields.py M Modules/_ctypes/cfield.c M Modules/_ctypes/stgdict.c diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index fefeaea1496a..e444f5e1f779 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -54,6 +54,12 @@ class X(Structure): x.char = b'a\0b\0' self.assertEqual(bytes(x), b'a\x00###') + def test_6(self): + class X(Structure): + _fields_ = [("x", c_int)] + CField = type(X.x) + self.assertRaises(TypeError, CField) + def test_gh99275(self): class BrokenStructure(Structure): def __init_subclass__(cls, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst new file mode 100644 index 000000000000..8b455fd2ef7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst @@ -0,0 +1 @@ +Fix crash when creating an instance of :class:`!_ctypes.CField`. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 791aeba66539..796a1bec966d 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -30,13 +30,6 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ -static PyObject * -PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CFieldObject *obj; - obj = (CFieldObject *)type->tp_alloc(type, 0); - return (PyObject *)obj; -} /* * Expects the size, index and offset for the current field in *psize and @@ -68,7 +61,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, #define CONT_BITFIELD 2 #define EXPAND_BITFIELD 3 - self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + self = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (self == NULL) return NULL; dict = PyType_stgdict(desc); @@ -341,7 +334,7 @@ PyTypeObject PyCField_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - PyCField_new, /* tp_new */ + 0, /* tp_new */ 0, /* tp_free */ }; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 099331ca8bdb..9a4041fb2528 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -257,7 +257,7 @@ MakeFields(PyObject *type, CFieldObject *descr, } continue; } - new_descr = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + new_descr = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (new_descr == NULL) { Py_DECREF(fdescr); Py_DECREF(fieldlist); From webhook-mailer at python.org Wed Dec 21 12:48:31 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 21 Dec 2022 17:48:31 -0000 Subject: [Python-checkins] GH-69564: Clarify use of octal format of mode argument in help(os.chmod) (#20621) Message-ID: <mailman.3008.1671644913.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bbf4a66a18cc0fa9121e1bbc90d75d10f5b3a389 commit: bbf4a66a18cc0fa9121e1bbc90d75d10f5b3a389 branch: main author: amaajemyfren <32741226+amaajemyfren at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-21T23:18:25+05:30 summary: GH-69564: Clarify use of octal format of mode argument in help(os.chmod) (#20621) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst M Modules/clinic/posixmodule.c.h M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst b/Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst new file mode 100644 index 000000000000..019a1c42d88e --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2020-06-17-14-47-48.bpo-25377.CTxC6o.rst @@ -0,0 +1 @@ +Clarify use of octal format of mode argument in help(os.chmod) as well as help(os.fchmod) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 86251008b1bd..d4722cc533cb 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -501,6 +501,9 @@ PyDoc_STRVAR(os_chmod__doc__, " If this functionality is unavailable, using it raises an exception.\n" " mode\n" " Operating-system mode bitfield.\n" +" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" +" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" +" Python.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -602,6 +605,14 @@ PyDoc_STRVAR(os_fchmod__doc__, "\n" "Change the access permissions of the file given by file descriptor fd.\n" "\n" +" fd\n" +" The file descriptor of the file to be modified.\n" +" mode\n" +" Operating-system mode bitfield.\n" +" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" +" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" +" Python.\n" +"\n" "Equivalent to os.chmod(fd, mode)."); #define OS_FCHMOD_METHODDEF \ @@ -11549,4 +11560,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=04fd23c89ab41f75 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=41eab6c3523792a9 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4817973262f4..607d40b59d96 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3173,6 +3173,9 @@ os.chmod mode: int Operating-system mode bitfield. + Be careful when using number literals for *mode*. The conventional UNIX notation for + numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in + Python. * @@ -3198,7 +3201,7 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=989081551c00293b]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=674a14bc998de09d]*/ { int result; @@ -3328,7 +3331,12 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, os.fchmod fd: int + The file descriptor of the file to be modified. mode: int + Operating-system mode bitfield. + Be careful when using number literals for *mode*. The conventional UNIX notation for + numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in + Python. Change the access permissions of the file given by file descriptor fd. @@ -3337,7 +3345,7 @@ Equivalent to os.chmod(fd, mode). static PyObject * os_fchmod_impl(PyObject *module, int fd, int mode) -/*[clinic end generated code: output=afd9bc05b4e426b3 input=8ab11975ca01ee5b]*/ +/*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ { int res; int async_err = 0; From webhook-mailer at python.org Wed Dec 21 13:02:12 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 21 Dec 2022 18:02:12 -0000 Subject: [Python-checkins] gh-78878: Fix crash when creating an instance of `_ctypes.CField` (GH-14837) Message-ID: <mailman.3009.1671645733.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3e715e0cc811ebccce4234c3e25ef30787151d00 commit: 3e715e0cc811ebccce4234c3e25ef30787151d00 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-21T10:02:05-08:00 summary: gh-78878: Fix crash when creating an instance of `_ctypes.CField` (GH-14837) (cherry picked from commit d713c54ac8a2eba0616a5a07714696d935f1062e) Co-authored-by: Hai Shi <shihai1992 at gmail.com> files: A Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst M Lib/ctypes/test/test_struct_fields.py M Modules/_ctypes/cfield.c M Modules/_ctypes/stgdict.c diff --git a/Lib/ctypes/test/test_struct_fields.py b/Lib/ctypes/test/test_struct_fields.py index fefeaea1496a..e444f5e1f779 100644 --- a/Lib/ctypes/test/test_struct_fields.py +++ b/Lib/ctypes/test/test_struct_fields.py @@ -54,6 +54,12 @@ class X(Structure): x.char = b'a\0b\0' self.assertEqual(bytes(x), b'a\x00###') + def test_6(self): + class X(Structure): + _fields_ = [("x", c_int)] + CField = type(X.x) + self.assertRaises(TypeError, CField) + def test_gh99275(self): class BrokenStructure(Structure): def __init_subclass__(cls, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst new file mode 100644 index 000000000000..8b455fd2ef7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-20-54-04.gh-issue-78878.JrkYqJ.rst @@ -0,0 +1 @@ +Fix crash when creating an instance of :class:`!_ctypes.CField`. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 13ed8b7eda65..9bdf1db856e7 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -30,13 +30,6 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ -static PyObject * -PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CFieldObject *obj; - obj = (CFieldObject *)type->tp_alloc(type, 0); - return (PyObject *)obj; -} /* * Expects the size, index and offset for the current field in *psize and @@ -68,7 +61,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, #define CONT_BITFIELD 2 #define EXPAND_BITFIELD 3 - self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + self = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (self == NULL) return NULL; dict = PyType_stgdict(desc); @@ -343,7 +336,7 @@ PyTypeObject PyCField_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - PyCField_new, /* tp_new */ + 0, /* tp_new */ 0, /* tp_free */ }; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 88999b8dff18..b7134ce2571b 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -258,7 +258,7 @@ MakeFields(PyObject *type, CFieldObject *descr, } continue; } - new_descr = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + new_descr = (CFieldObject *)PyCField_Type.tp_alloc((PyTypeObject *)&PyCField_Type, 0); if (new_descr == NULL) { Py_DECREF(fdescr); Py_DECREF(fieldlist); From webhook-mailer at python.org Wed Dec 21 19:41:24 2022 From: webhook-mailer at python.org (brandtbucher) Date: Thu, 22 Dec 2022 00:41:24 -0000 Subject: [Python-checkins] GH-99554: Pack location tables more effectively (GH-99556) Message-ID: <mailman.3010.1671669684.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3c033a2e6fbde56f904aeca138141be6564341cf commit: 3c033a2e6fbde56f904aeca138141be6564341cf branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: brandtbucher <brandtbucher at gmail.com> date: 2022-12-21T16:41:18-08:00 summary: GH-99554: Pack location tables more effectively (GH-99556) files: A Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst M Programs/test_frozenmain.h M Python/compile.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst new file mode 100644 index 000000000000..96ec47db461d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-16-05-57-24.gh-issue-99554.A_Ywd2.rst @@ -0,0 +1 @@ +Pack debugging location tables more efficiently during bytecode compilation. diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 96be3ce3c25c..95f78b19e65e 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -28,15 +28,12 @@ unsigned char M_test_frozenmain[] = { 115,114,3,0,0,0,218,3,107,101,121,169,0,243,0,0, 0,0,250,18,116,101,115,116,95,102,114,111,122,101,110,109, 97,105,110,46,112,121,250,8,60,109,111,100,117,108,101,62, - 114,18,0,0,0,1,0,0,0,115,154,0,0,0,240,3, - 1,1,1,240,8,0,1,11,128,10,128,10,128,10,216,0, - 24,208,0,24,208,0,24,208,0,24,224,0,5,128,5,208, - 6,26,212,0,27,208,0,27,216,0,5,128,5,128,106,144, - 35,151,40,145,40,212,0,27,208,0,27,216,9,38,208,9, - 26,215,9,38,209,9,38,212,9,40,168,24,212,9,50,128, - 6,240,2,6,12,2,240,0,7,1,42,241,0,7,1,42, - 128,67,240,14,0,5,10,128,69,208,10,40,144,67,208,10, - 40,208,10,40,152,54,160,35,156,59,208,10,40,208,10,40, - 212,4,41,208,4,41,208,4,41,240,15,7,1,42,240,0, - 7,1,42,240,0,7,1,42,114,16,0,0,0, + 114,18,0,0,0,1,0,0,0,115,100,0,0,0,240,3, + 1,1,1,243,8,0,1,11,219,0,24,225,0,5,208,6, + 26,213,0,27,217,0,5,128,106,144,35,151,40,145,40,213, + 0,27,216,9,38,208,9,26,215,9,38,209,9,38,212,9, + 40,168,24,212,9,50,128,6,240,2,6,12,2,242,0,7, + 1,42,128,67,241,14,0,5,10,208,10,40,144,67,209,10, + 40,152,54,160,35,156,59,209,10,40,214,4,41,242,15,7, + 1,42,114,16,0,0,0, }; diff --git a/Python/compile.c b/Python/compile.c index 09eb4016940d..023c13507d6a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -150,6 +150,15 @@ location_is_after(location loc1, location loc2) { (loc1.col_offset > loc2.end_col_offset)); } +static inline bool +same_location(location a, location b) +{ + return a.lineno == b.lineno && + a.end_lineno == b.end_lineno && + a.col_offset == b.col_offset && + a.end_col_offset == b.end_col_offset; +} + #define LOC(x) SRC_LOCATION_FROM_AST(x) typedef struct jump_target_label_ { @@ -7722,15 +7731,15 @@ write_location_info_oneline_form(struct assembler* a, int length, int line_delta } static void -write_location_info_long_form(struct assembler* a, struct instr* i, int length) +write_location_info_long_form(struct assembler* a, location loc, int length) { assert(length > 0 && length <= 8); write_location_first_byte(a, PY_CODE_LOCATION_INFO_LONG, length); - write_location_signed_varint(a, i->i_loc.lineno - a->a_lineno); - assert(i->i_loc.end_lineno >= i->i_loc.lineno); - write_location_varint(a, i->i_loc.end_lineno - i->i_loc.lineno); - write_location_varint(a, i->i_loc.col_offset + 1); - write_location_varint(a, i->i_loc.end_col_offset + 1); + write_location_signed_varint(a, loc.lineno - a->a_lineno); + assert(loc.end_lineno >= loc.lineno); + write_location_varint(a, loc.end_lineno - loc.lineno); + write_location_varint(a, loc.col_offset + 1); + write_location_varint(a, loc.end_col_offset + 1); } static void @@ -7749,7 +7758,7 @@ write_location_info_no_column(struct assembler* a, int length, int line_delta) #define THEORETICAL_MAX_ENTRY_SIZE 25 /* 1 + 6 + 6 + 6 + 6 */ static int -write_location_info_entry(struct assembler* a, struct instr* i, int isize) +write_location_info_entry(struct assembler* a, location loc, int isize) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_linetable); if (a->a_location_off + THEORETICAL_MAX_ENTRY_SIZE >= len) { @@ -7758,49 +7767,51 @@ write_location_info_entry(struct assembler* a, struct instr* i, int isize) return -1; } } - if (i->i_loc.lineno < 0) { + if (loc.lineno < 0) { write_location_info_none(a, isize); return 0; } - int line_delta = i->i_loc.lineno - a->a_lineno; - int column = i->i_loc.col_offset; - int end_column = i->i_loc.end_col_offset; + int line_delta = loc.lineno - a->a_lineno; + int column = loc.col_offset; + int end_column = loc.end_col_offset; assert(column >= -1); assert(end_column >= -1); if (column < 0 || end_column < 0) { - if (i->i_loc.end_lineno == i->i_loc.lineno || i->i_loc.end_lineno == -1) { + if (loc.end_lineno == loc.lineno || loc.end_lineno == -1) { write_location_info_no_column(a, isize, line_delta); - a->a_lineno = i->i_loc.lineno; + a->a_lineno = loc.lineno; return 0; } } - else if (i->i_loc.end_lineno == i->i_loc.lineno) { + else if (loc.end_lineno == loc.lineno) { if (line_delta == 0 && column < 80 && end_column - column < 16 && end_column >= column) { write_location_info_short_form(a, isize, column, end_column); return 0; } if (line_delta >= 0 && line_delta < 3 && column < 128 && end_column < 128) { write_location_info_oneline_form(a, isize, line_delta, column, end_column); - a->a_lineno = i->i_loc.lineno; + a->a_lineno = loc.lineno; return 0; } } - write_location_info_long_form(a, i, isize); - a->a_lineno = i->i_loc.lineno; + write_location_info_long_form(a, loc, isize); + a->a_lineno = loc.lineno; return 0; } static int -assemble_emit_location(struct assembler* a, struct instr* i) +assemble_emit_location(struct assembler* a, location loc, int isize) { - int isize = instr_size(i); + if (isize == 0) { + return 0; + } while (isize > 8) { - if (write_location_info_entry(a, i, 8) < 0) { + if (write_location_info_entry(a, loc, 8)) { return -1; } isize -= 8; } - return write_location_info_entry(a, i, isize); + return write_location_info_entry(a, loc, isize); } /* assemble_emit() @@ -8860,13 +8871,23 @@ assemble(struct compiler *c, int addNone) /* Emit location info */ a.a_lineno = c->u->u_firstlineno; + location loc = NO_LOCATION; + int size = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int j = 0; j < b->b_iused; j++) { - if (assemble_emit_location(&a, &b->b_instr[j]) < 0) { - goto error; + if (!same_location(loc, b->b_instr[j].i_loc)) { + if (assemble_emit_location(&a, loc, size)) { + goto error; + } + loc = b->b_instr[j].i_loc; + size = 0; } + size += instr_size(&b->b_instr[j]); } } + if (assemble_emit_location(&a, loc, size)) { + goto error; + } if (assemble_exception_table(&a, g->g_entryblock) < 0) { goto error; From webhook-mailer at python.org Thu Dec 22 06:21:03 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Thu, 22 Dec 2022 11:21:03 -0000 Subject: [Python-checkins] Correct typo in typing.py (#100423) Message-ID: <mailman.3011.1671708064.3313.python-checkins@python.org> https://github.com/python/cpython/commit/68981578eceee763da4163e93cf653cc6b1b6d35 commit: 68981578eceee763da4163e93cf653cc6b1b6d35 branch: main author: david-why <david_why at outlook.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-22T11:20:52Z summary: Correct typo in typing.py (#100423) In the docstring of `ParamSpec`, the name of `P = ParamSpec('P')` was mistakenly written as `'T'`. files: M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index d9d6fbcdb8f0..8bc38f98c867 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1194,7 +1194,7 @@ def add_two(x: float, y: float) -> float: Parameter specification variables can be introspected. e.g.: - P.__name__ == 'T' + P.__name__ == 'P' P.__bound__ == None P.__covariant__ == False P.__contravariant__ == False From webhook-mailer at python.org Thu Dec 22 06:30:28 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 22 Dec 2022 11:30:28 -0000 Subject: [Python-checkins] gh-99761: Add `_PyLong_IsPositiveSingleDigit` function to check for single digit integers (#100064) Message-ID: <mailman.3012.1671708629.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b82c36f17ada471e734c3ad93e6eff8b36a5ad9 commit: 2b82c36f17ada471e734c3ad93e6eff8b36a5ad9 branch: main author: Pieter Eendebak <pieter.eendebak at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-22T17:00:18+05:30 summary: gh-99761: Add `_PyLong_IsPositiveSingleDigit` function to check for single digit integers (#100064) files: M Include/internal/pycore_long.h M Python/bytecodes.c M Python/generated_cases.c.h diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 30c97b7edc98..8c1d017bb95e 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -110,6 +110,25 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); +/* Return 1 if the argument is positive single digit int */ +static inline int +_PyLong_IsPositiveSingleDigit(PyObject* sub) { + /* For a positive single digit int, the value of Py_SIZE(sub) is 0 or 1. + + We perform a fast check using a single comparison by casting from int + to uint which casts negative numbers to large positive numbers. + For details see Section 14.2 "Bounds Checking" in the Agner Fog + optimization manual found at: + https://www.agner.org/optimize/optimizing_cpp.pdf + + The function is not affected by -fwrapv, -fno-wrapv and -ftrapv + compiler options of GCC and clang + */ + assert(PyLong_CheckExact(sub)); + Py_ssize_t signed_size = Py_SIZE(sub); + return ((size_t)signed_size) <= 1; +} + #ifdef __cplusplus } #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b29e16e080e9..c0b625bd662c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -391,8 +391,7 @@ dummy_func( DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); @@ -410,8 +409,7 @@ dummy_func( DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); @@ -508,7 +506,7 @@ dummy_func( DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; // Ensure index < len(list) DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6d84a643b457..42b7ca086705 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -491,8 +491,7 @@ DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); @@ -517,8 +516,7 @@ DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - Py_ssize_t signed_magnitude = Py_SIZE(sub); - DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR); assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); @@ -642,7 +640,7 @@ DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; // Ensure index < len(list) DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); From webhook-mailer at python.org Thu Dec 22 06:44:06 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 11:44:06 -0000 Subject: [Python-checkins] Correct typo in typing.py (GH-100423) Message-ID: <mailman.3013.1671709447.3313.python-checkins@python.org> https://github.com/python/cpython/commit/58967d1abec9a88b1657c50b421795b4eb519e56 commit: 58967d1abec9a88b1657c50b421795b4eb519e56 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T03:43:55-08:00 summary: Correct typo in typing.py (GH-100423) In the docstring of `ParamSpec`, the name of `P = ParamSpec('P')` was mistakenly written as `'T'`. (cherry picked from commit 68981578eceee763da4163e93cf653cc6b1b6d35) Co-authored-by: david-why <david_why at outlook.com> files: M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 9f8710b9f773..03cf243366db 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -901,7 +901,7 @@ def add_two(x: float, y: float) -> float: Parameter specification variables can be introspected. e.g.: - P.__name__ == 'T' + P.__name__ == 'P' P.__bound__ == None P.__covariant__ == False P.__contravariant__ == False From webhook-mailer at python.org Thu Dec 22 06:49:24 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 11:49:24 -0000 Subject: [Python-checkins] Correct typo in typing.py (GH-100423) Message-ID: <mailman.3014.1671709765.3313.python-checkins@python.org> https://github.com/python/cpython/commit/6afc389e1d1fa969a6cbfe5d6e51d6c2f8637d7d commit: 6afc389e1d1fa969a6cbfe5d6e51d6c2f8637d7d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T03:49:18-08:00 summary: Correct typo in typing.py (GH-100423) In the docstring of `ParamSpec`, the name of `P = ParamSpec('P')` was mistakenly written as `'T'`. (cherry picked from commit 68981578eceee763da4163e93cf653cc6b1b6d35) Co-authored-by: david-why <david_why at outlook.com> files: M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 9818237de435..b287446169c2 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1174,7 +1174,7 @@ def add_two(x: float, y: float) -> float: Parameter specification variables can be introspected. e.g.: - P.__name__ == 'T' + P.__name__ == 'P' P.__bound__ == None P.__covariant__ == False P.__contravariant__ == False From webhook-mailer at python.org Thu Dec 22 07:45:05 2022 From: webhook-mailer at python.org (markshannon) Date: Thu, 22 Dec 2022 12:45:05 -0000 Subject: [Python-checkins] GH-99770: Make the correct call specialization fail kind show up in the stats (GH-99771) Message-ID: <mailman.3015.1671713105.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a02161286a67692758cac38d9dbe42625c211605 commit: a02161286a67692758cac38d9dbe42625c211605 branch: main author: penguin_wwy <940375606 at qq.com> committer: markshannon <mark at hotpy.org> date: 2022-12-22T12:44:59Z summary: GH-99770: Make the correct call specialization fail kind show up in the stats (GH-99771) files: M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Python/specialize.c b/Python/specialize.c index c6c502716478..69a7bf98cad0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -309,7 +309,8 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_OUT_OF_RANGE 4 #define SPEC_FAIL_EXPECTED_ERROR 5 #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6 -#define SPEC_FAIL_NOT_PY_FUNCTION 7 +#define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7 +#define SPEC_FAIL_CODE_NOT_OPTIMIZED 8 #define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17 @@ -317,18 +318,18 @@ _PyCode_Quicken(PyCodeObject *code) /* Attributes */ -#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 8 -#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 9 -#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 10 -#define SPEC_FAIL_ATTR_METHOD 11 -#define SPEC_FAIL_ATTR_MUTABLE_CLASS 12 -#define SPEC_FAIL_ATTR_PROPERTY 13 -#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 14 -#define SPEC_FAIL_ATTR_READ_ONLY 15 -#define SPEC_FAIL_ATTR_AUDITED_SLOT 16 -#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 17 -#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 18 -#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 19 +#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9 +#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10 +#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11 +#define SPEC_FAIL_ATTR_METHOD 12 +#define SPEC_FAIL_ATTR_MUTABLE_CLASS 13 +#define SPEC_FAIL_ATTR_PROPERTY 14 +#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15 +#define SPEC_FAIL_ATTR_READ_ONLY 16 +#define SPEC_FAIL_ATTR_AUDITED_SLOT 17 +#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18 +#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19 +#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20 #define SPEC_FAIL_ATTR_SHADOWED 21 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 @@ -346,12 +347,12 @@ _PyCode_Quicken(PyCodeObject *code) /* Binary subscr and store subscr */ -#define SPEC_FAIL_SUBSCR_ARRAY_INT 8 -#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 9 -#define SPEC_FAIL_SUBSCR_LIST_SLICE 10 -#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 11 -#define SPEC_FAIL_SUBSCR_STRING_INT 12 -#define SPEC_FAIL_SUBSCR_STRING_SLICE 13 +#define SPEC_FAIL_SUBSCR_ARRAY_INT 9 +#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10 +#define SPEC_FAIL_SUBSCR_LIST_SLICE 11 +#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12 +#define SPEC_FAIL_SUBSCR_STRING_INT 13 +#define SPEC_FAIL_SUBSCR_STRING_SLICE 14 #define SPEC_FAIL_SUBSCR_BUFFER_INT 15 #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16 #define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17 @@ -366,49 +367,48 @@ _PyCode_Quicken(PyCodeObject *code) /* Binary op */ -#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 8 -#define SPEC_FAIL_BINARY_OP_ADD_OTHER 9 -#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 10 -#define SPEC_FAIL_BINARY_OP_AND_INT 11 -#define SPEC_FAIL_BINARY_OP_AND_OTHER 12 -#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 13 -#define SPEC_FAIL_BINARY_OP_LSHIFT 14 -#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 15 -#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 16 -#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 17 -#define SPEC_FAIL_BINARY_OP_OR 18 -#define SPEC_FAIL_BINARY_OP_POWER 19 -#define SPEC_FAIL_BINARY_OP_REMAINDER 20 -#define SPEC_FAIL_BINARY_OP_RSHIFT 21 -#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 22 -#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 23 -#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24 -#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 25 -#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 26 -#define SPEC_FAIL_BINARY_OP_XOR 27 +#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 9 +#define SPEC_FAIL_BINARY_OP_ADD_OTHER 10 +#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 11 +#define SPEC_FAIL_BINARY_OP_AND_INT 12 +#define SPEC_FAIL_BINARY_OP_AND_OTHER 13 +#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 14 +#define SPEC_FAIL_BINARY_OP_LSHIFT 15 +#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 16 +#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 17 +#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 18 +#define SPEC_FAIL_BINARY_OP_OR 19 +#define SPEC_FAIL_BINARY_OP_POWER 20 +#define SPEC_FAIL_BINARY_OP_REMAINDER 21 +#define SPEC_FAIL_BINARY_OP_RSHIFT 22 +#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 23 +#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 24 +#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25 +#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26 +#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27 +#define SPEC_FAIL_BINARY_OP_XOR 28 /* Calls */ -#define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9 -#define SPEC_FAIL_CALL_CO_NOT_OPTIMIZED 10 -/* SPEC_FAIL_METHOD defined as 11 above */ #define SPEC_FAIL_CALL_INSTANCE_METHOD 11 #define SPEC_FAIL_CALL_CMETHOD 12 #define SPEC_FAIL_CALL_CFUNC_VARARGS 13 #define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14 -#define SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS 15 -#define SPEC_FAIL_CALL_CFUNC_NOARGS 16 -#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17 -#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 18 -#define SPEC_FAIL_CALL_PYTHON_CLASS 19 -#define SPEC_FAIL_CALL_PEP_523 20 -#define SPEC_FAIL_CALL_BOUND_METHOD 21 -#define SPEC_FAIL_CALL_STR 22 -#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23 -#define SPEC_FAIL_CALL_CLASS_MUTABLE 24 -#define SPEC_FAIL_CALL_KWNAMES 25 -#define SPEC_FAIL_CALL_METHOD_WRAPPER 26 -#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27 +#define SPEC_FAIL_CALL_CFUNC_NOARGS 15 +#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16 +#define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17 +#define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18 +#define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19 +#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20 +#define SPEC_FAIL_CALL_PYTHON_CLASS 21 +#define SPEC_FAIL_CALL_PEP_523 22 +#define SPEC_FAIL_CALL_BOUND_METHOD 23 +#define SPEC_FAIL_CALL_STR 24 +#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 25 +#define SPEC_FAIL_CALL_CLASS_MUTABLE 26 +#define SPEC_FAIL_CALL_KWNAMES 27 +#define SPEC_FAIL_CALL_METHOD_WRAPPER 28 +#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -449,8 +449,8 @@ _PyCode_Quicken(PyCodeObject *code) // UNPACK_SEQUENCE -#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 8 -#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 9 +#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9 +#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10 static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); @@ -1251,10 +1251,10 @@ static int function_kind(PyCodeObject *code) { int flags = code->co_flags; if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) { - return SPEC_FAIL_CALL_COMPLEX_PARAMETERS; + return SPEC_FAIL_CODE_COMPLEX_PARAMETERS; } if ((flags & CO_OPTIMIZED) == 0) { - return SPEC_FAIL_CALL_CO_NOT_OPTIMIZED; + return SPEC_FAIL_CODE_NOT_OPTIMIZED; } return SIMPLE_FUNCTION; } @@ -1526,8 +1526,6 @@ builtin_call_fail_kind(int ml_flags) return SPEC_FAIL_CALL_CFUNC_VARARGS; case METH_VARARGS | METH_KEYWORDS: return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_FASTCALL_KEYWORDS; case METH_NOARGS: return SPEC_FAIL_CALL_CFUNC_NOARGS; case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: @@ -1535,6 +1533,29 @@ builtin_call_fail_kind(int ml_flags) /* These cases should be optimized, but return "other" just in case */ case METH_O: case METH_FASTCALL: + case METH_FASTCALL | METH_KEYWORDS: + return SPEC_FAIL_OTHER; + default: + return SPEC_FAIL_CALL_BAD_CALL_FLAGS; + } +} + +static int +meth_descr_call_fail_kind(int ml_flags) +{ + switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | + METH_KEYWORDS | METH_METHOD)) { + case METH_VARARGS: + return SPEC_FAIL_CALL_METH_DESCR_VARARGS; + case METH_VARARGS | METH_KEYWORDS: + return SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + return SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS; + /* These cases should be optimized, but return "other" just in case */ + case METH_NOARGS: + case METH_O: + case METH_FASTCALL: + case METH_FASTCALL | METH_KEYWORDS: return SPEC_FAIL_OTHER; default: return SPEC_FAIL_CALL_BAD_CALL_FLAGS; @@ -1583,12 +1604,12 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); return 0; } - case METH_FASTCALL|METH_KEYWORDS: { + case METH_FASTCALL | METH_KEYWORDS: { _py_set_opcode(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } - SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags)); + SPECIALIZATION_FAIL(CALL, meth_descr_call_fail_kind(descr->d_method->ml_flags)); return -1; } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 81b06f9f7469..1c8d10f70277 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -224,7 +224,7 @@ def pretty(defname): return defname.replace("_", " ").lower() def kind_to_text(kind, defines, opname): - if kind <= 7: + if kind <= 8: return pretty(defines[kind][0]) if opname.endswith("ATTR"): opname = "ATTR" From webhook-mailer at python.org Thu Dec 22 08:28:31 2022 From: webhook-mailer at python.org (FFY00) Date: Thu, 22 Dec 2022 13:28:31 -0000 Subject: [Python-checkins] gh-78997: fix bad rebase of moved test file (#100424) Message-ID: <mailman.3016.1671715712.3313.python-checkins@python.org> https://github.com/python/cpython/commit/aa878f086b7ba8bdd7006d9d509c671167a5fb1e commit: aa878f086b7ba8bdd7006d9d509c671167a5fb1e branch: main author: Eli Schwartz <eschwartz93 at gmail.com> committer: FFY00 <filipe.lains at gmail.com> date: 2022-12-22T13:28:25Z summary: gh-78997: fix bad rebase of moved test file (#100424) files: D Lib/ctypes/test/test_loading.py M Lib/test/test_ctypes/test_loading.py diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py deleted file mode 100644 index b61d6fa2912a..000000000000 --- a/Lib/ctypes/test/test_loading.py +++ /dev/null @@ -1,188 +0,0 @@ -from ctypes import * -import os -import shutil -import subprocess -import sys -import unittest -import test.support -from test.support import import_helper -from test.support import os_helper -from ctypes.util import find_library - -libc_name = None - -def setUpModule(): - global libc_name - if os.name == "nt": - libc_name = find_library("c") - elif sys.platform == "cygwin": - libc_name = "cygwin1.dll" - else: - libc_name = find_library("c") - - if test.support.verbose: - print("libc_name is", libc_name) - -class LoaderTest(unittest.TestCase): - - unknowndll = "xxrandomnamexx" - - def test_load(self): - if libc_name is None: - self.skipTest('could not find libc') - CDLL(libc_name) - CDLL(os.path.basename(libc_name)) - self.assertRaises(OSError, CDLL, self.unknowndll) - - def test_load_version(self): - if libc_name is None: - self.skipTest('could not find libc') - if os.path.basename(libc_name) != 'libc.so.6': - self.skipTest('wrong libc path for test') - cdll.LoadLibrary("libc.so.6") - # linux uses version, libc 9 should not exist - self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") - self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) - - def test_find(self): - for name in ("c", "m"): - lib = find_library(name) - if lib: - cdll.LoadLibrary(lib) - CDLL(lib) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_library(self): - # CRT is no longer directly loadable. See issue23606 for the - # discussion about alternative approaches. - #self.assertIsNotNone(libc_name) - if test.support.verbose: - print(find_library("kernel32")) - print(find_library("user32")) - - if os.name == "nt": - windll.kernel32.GetModuleHandleW - windll["kernel32"].GetModuleHandleW - windll.LoadLibrary("kernel32").GetModuleHandleW - WinDLL("kernel32").GetModuleHandleW - # embedded null character - self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_ordinal_functions(self): - import _ctypes_test - dll = WinDLL(_ctypes_test.__file__) - # We load the same function both via ordinal and name - func_ord = dll[2] - func_name = dll.GetString - # addressof gets the address where the function pointer is stored - a_ord = addressof(func_ord) - a_name = addressof(func_name) - f_ord_addr = c_void_p.from_address(a_ord).value - f_name_addr = c_void_p.from_address(a_name).value - self.assertEqual(hex(f_ord_addr), hex(f_name_addr)) - - self.assertRaises(AttributeError, dll.__getitem__, 1234) - - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_A(self): - from _ctypes import LoadLibrary, FreeLibrary - # On winXP 64-bit, advapi32 loads at an address that does - # NOT fit into a 32-bit integer. FreeLibrary must be able - # to accept this address. - - # These are tests for https://www.python.org/sf/1703286 - handle = LoadLibrary("advapi32") - FreeLibrary(handle) - - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_B(self): - # Since on winXP 64-bit advapi32 loads like described - # above, the (arbitrarily selected) CloseEventLog function - # also has a high address. 'call_function' should accept - # addresses so large. - from _ctypes import call_function - advapi32 = windll.advapi32 - # Calling CloseEventLog with a NULL argument should fail, - # but the call should not segfault or so. - self.assertEqual(0, advapi32.CloseEventLog(None)) - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p - proc = windll.kernel32.GetProcAddress(advapi32._handle, - b"CloseEventLog") - self.assertTrue(proc) - # This is the real test: call the function via 'call_function' - self.assertEqual(0, call_function(proc, (None,))) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_hasattr(self): - # bpo-34816: shouldn't raise OSError - self.assertFalse(hasattr(windll, 'test')) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_dll_with_flags(self): - _sqlite3 = import_helper.import_module("_sqlite3") - src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): - ext = "_d.dll" - else: - ext = ".dll" - - with os_helper.temp_dir() as tmp: - # We copy two files and load _sqlite3.dll (formerly .pyd), - # which has a dependency on sqlite3.dll. Then we test - # loading it in subprocesses to avoid it starting in memory - # for each test. - target = os.path.join(tmp, "_sqlite3.dll") - shutil.copy(src, target) - shutil.copy(os.path.join(os.path.dirname(src), "sqlite3" + ext), - os.path.join(tmp, "sqlite3" + ext)) - - def should_pass(command): - with self.subTest(command): - subprocess.check_output( - [sys.executable, "-c", - "from ctypes import *; import nt;" + command], - cwd=tmp - ) - - def should_fail(command): - with self.subTest(command): - with self.assertRaises(subprocess.CalledProcessError): - subprocess.check_output( - [sys.executable, "-c", - "from ctypes import *; import nt;" + command], - cwd=tmp, stderr=subprocess.STDOUT, - ) - - # Default load should not find this in CWD - should_fail("WinDLL('_sqlite3.dll')") - - # Relative path (but not just filename) should succeed - should_pass("WinDLL('./_sqlite3.dll')") - - # Insecure load flags should succeed - # Clear the DLL directory to avoid safe search settings propagating - should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)") - - # Full path load without DLL_LOAD_DIR shouldn't find dependency - should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + - "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32)") - - # Full path load with DLL_LOAD_DIR should succeed - should_pass("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + - "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32|" + - "nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)") - - # User-specified directory should succeed - should_pass("import os; p = os.add_dll_directory(os.getcwd());" + - "WinDLL('_sqlite3.dll'); p.close()") - - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 8d8632a4eb64..15e365ed267d 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -116,6 +116,12 @@ def test_1703286_B(self): # This is the real test: call the function via 'call_function' self.assertEqual(0, call_function(proc, (None,))) + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_hasattr(self): + # bpo-34816: shouldn't raise OSError + self.assertFalse(hasattr(windll, 'test')) + @unittest.skipUnless(os.name == "nt", 'test specific to Windows') def test_load_dll_with_flags(self): From webhook-mailer at python.org Thu Dec 22 08:38:19 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 22 Dec 2022 13:38:19 -0000 Subject: [Python-checkins] gh-100344: Add C implementation for `asyncio.current_task` (#100345) Message-ID: <mailman.3017.1671716301.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4cc63e0d4e4cf3299dcc0ea81616ba072ae5589d commit: 4cc63e0d4e4cf3299dcc0ea81616ba072ae5589d branch: main author: Itamar Ostricher <itamarost at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-22T19:08:12+05:30 summary: gh-100344: Add C implementation for `asyncio.current_task` (#100345) Co-authored-by: pranavtbhat files: A Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst M Doc/whatsnew/3.12.rst M Lib/asyncio/tasks.py M Lib/test/test_asyncio/test_tasks.py M Modules/_asynciomodule.c M Modules/clinic/_asynciomodule.c.h diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 0cc4471364b6..d480be27542c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -225,6 +225,9 @@ asyncio a custom event loop factory. (Contributed by Kumar Aditya in :gh:`99388`.) +* Add C implementation of :func:`asyncio.current_task` for 4x-6x speedup. + (Contributed by Itamar Ostricher and Pranav Thulasiram Bhat in :gh:`100344`.) + inspect ------- diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fa853283c0c5..e78719de216f 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -964,6 +964,7 @@ def _unregister_task(task): _all_tasks.discard(task) +_py_current_task = current_task _py_register_task = _register_task _py_unregister_task = _unregister_task _py_enter_task = _enter_task @@ -973,10 +974,12 @@ def _unregister_task(task): try: from _asyncio import (_register_task, _unregister_task, _enter_task, _leave_task, - _all_tasks, _current_tasks) + _all_tasks, _current_tasks, + current_task) except ImportError: pass else: + _c_current_task = current_task _c_register_task = _register_task _c_unregister_task = _unregister_task _c_enter_task = _enter_task diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 5168b8250ef0..e533d5273e9f 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2804,6 +2804,7 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): class BaseCurrentLoopTests: + current_task = None def setUp(self): super().setUp() @@ -2814,33 +2815,39 @@ def new_task(self, coro): raise NotImplementedError def test_current_task_no_running_loop(self): - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): with self.assertRaisesRegex(RuntimeError, 'no running event loop'): - asyncio.current_task() + self.current_task() def test_current_task_with_implicit_loop(self): async def coro(): - self.assertIs(asyncio.current_task(loop=self.loop), task) + self.assertIs(self.current_task(loop=self.loop), task) - self.assertIs(asyncio.current_task(None), task) - self.assertIs(asyncio.current_task(), task) + self.assertIs(self.current_task(None), task) + self.assertIs(self.current_task(), task) task = self.new_task(coro()) self.loop.run_until_complete(task) - self.assertIsNone(asyncio.current_task(loop=self.loop)) + self.assertIsNone(self.current_task(loop=self.loop)) class PyCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + current_task = staticmethod(tasks._py_current_task) def new_task(self, coro): return tasks._PyTask(coro, loop=self.loop) - at unittest.skipUnless(hasattr(tasks, '_CTask'), + at unittest.skipUnless(hasattr(tasks, '_CTask') and + hasattr(tasks, '_c_current_task'), 'requires the C _asyncio module') class CCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + if hasattr(tasks, '_c_current_task'): + current_task = staticmethod(tasks._c_current_task) + else: + current_task = None def new_task(self, coro): return getattr(tasks, '_CTask')(coro, loop=self.loop) diff --git a/Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst b/Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst new file mode 100644 index 000000000000..d55f6888dbde --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-19-12-18-28.gh-issue-100344.lfCqpE.rst @@ -0,0 +1,2 @@ +Provide C implementation for :func:`asyncio.current_task` for a 4x-6x +speedup. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 32be537c00a5..6fe4ca469475 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3314,6 +3314,44 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) } +/*[clinic input] +_asyncio.current_task + + loop: object = None + +Return a currently executed task. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_current_task_impl(PyObject *module, PyObject *loop) +/*[clinic end generated code: output=fe15ac331a7f981a input=58910f61a5627112]*/ +{ + PyObject *ret; + asyncio_state *state = get_asyncio_state(module); + + if (loop == Py_None) { + loop = _asyncio_get_running_loop_impl(module); + if (loop == NULL) { + return NULL; + } + } else { + Py_INCREF(loop); + } + + ret = PyDict_GetItemWithError(state->current_tasks, loop); + Py_DECREF(loop); + if (ret == NULL && PyErr_Occurred()) { + return NULL; + } + else if (ret == NULL) { + Py_RETURN_NONE; + } + Py_INCREF(ret); + return ret; +} + + /*********************** Module **************************/ @@ -3494,6 +3532,7 @@ module_init(asyncio_state *state) PyDoc_STRVAR(module_doc, "Accelerator module for asyncio"); static PyMethodDef asyncio_methods[] = { + _ASYNCIO_CURRENT_TASK_METHODDEF _ASYNCIO_GET_EVENT_LOOP_METHODDEF _ASYNCIO_GET_RUNNING_LOOP_METHODDEF _ASYNCIO__GET_RUNNING_LOOP_METHODDEF diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index f2fbb352c2c6..43c5d7717986 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -1242,4 +1242,64 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=83580c190031241c input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_asyncio_current_task__doc__, +"current_task($module, /, loop=None)\n" +"--\n" +"\n" +"Return a currently executed task."); + +#define _ASYNCIO_CURRENT_TASK_METHODDEF \ + {"current_task", _PyCFunction_CAST(_asyncio_current_task), METH_FASTCALL|METH_KEYWORDS, _asyncio_current_task__doc__}, + +static PyObject * +_asyncio_current_task_impl(PyObject *module, PyObject *loop); + +static PyObject * +_asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(loop), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"loop", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "current_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *loop = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + loop = args[0]; +skip_optional_pos: + return_value = _asyncio_current_task_impl(module, loop); + +exit: + return return_value; +} +/*[clinic end generated code: output=00f494214f2fd008 input=a9049054013a1b77]*/ From webhook-mailer at python.org Thu Dec 22 15:13:57 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 20:13:57 -0000 Subject: [Python-checkins] GH-99554: Trim trailing whitespace (GH-100435) Message-ID: <mailman.3018.1671740038.3313.python-checkins@python.org> https://github.com/python/cpython/commit/09edde95f4841d5dffa584b1c963eb7ceab3f16a commit: 09edde95f4841d5dffa584b1c963eb7ceab3f16a branch: main author: Brandt Bucher <brandtbucher at microsoft.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T12:13:51-08:00 summary: GH-99554: Trim trailing whitespace (GH-100435) Automerge-Triggered-By: GH:brandtbucher files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 023c13507d6a..cbbdfb9e9467 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -153,7 +153,7 @@ location_is_after(location loc1, location loc2) { static inline bool same_location(location a, location b) { - return a.lineno == b.lineno && + return a.lineno == b.lineno && a.end_lineno == b.end_lineno && a.col_offset == b.col_offset && a.end_col_offset == b.end_col_offset; From webhook-mailer at python.org Thu Dec 22 16:57:23 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Thu, 22 Dec 2022 21:57:23 -0000 Subject: [Python-checkins] gh-85432: Harmonise parameter names between C and pure-Python implementations of `datetime.time.strftime`, `datetime.datetime.fromtimestamp` (#99993) Message-ID: <mailman.3019.1671746244.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9cdb6429971cd8b874ceaeeb04ae2ecdbba42bdb commit: 9cdb6429971cd8b874ceaeeb04ae2ecdbba42bdb branch: main author: Alex Waygood <Alex.Waygood at Gmail.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-22T15:57:18-06:00 summary: gh-85432: Harmonise parameter names between C and pure-Python implementations of `datetime.time.strftime`, `datetime.datetime.fromtimestamp` (#99993) files: A Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst M Lib/datetime.py M Lib/test/datetimetester.py diff --git a/Lib/datetime.py b/Lib/datetime.py index 1b0c5cb2d1c6..68746de1cabf 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1553,8 +1553,7 @@ def fromisoformat(cls, time_string): except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') - - def strftime(self, fmt): + def strftime(self, format): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. """ @@ -1563,7 +1562,7 @@ def strftime(self, fmt): timetuple = (1900, 1, 1, self._hour, self._minute, self._second, 0, 1, -1) - return _wrap_strftime(self, fmt, timetuple) + return _wrap_strftime(self, format, timetuple) def __format__(self, fmt): if not isinstance(fmt, str): @@ -1787,14 +1786,14 @@ def _fromtimestamp(cls, t, utc, tz): return result @classmethod - def fromtimestamp(cls, t, tz=None): + def fromtimestamp(cls, timestamp, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) - return cls._fromtimestamp(t, tz is not None, tz) + return cls._fromtimestamp(timestamp, tz is not None, tz) @classmethod def utcfromtimestamp(cls, t): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 121d973b6d5f..6a1df174a1b9 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2426,6 +2426,12 @@ def test_fromtimestamp(self): got = self.theclass.fromtimestamp(ts) self.verify_field_equality(expected, got) + def test_fromtimestamp_keyword_arg(self): + import time + + # gh-85432: The parameter was named "t" in the pure-Python impl. + self.theclass.fromtimestamp(timestamp=time.time()) + def test_utcfromtimestamp(self): import time @@ -3528,6 +3534,9 @@ def test_strftime(self): except UnicodeEncodeError: pass + # gh-85432: The parameter was named "fmt" in the pure-Python impl. + t.strftime(format="%f") + def test_format(self): t = self.theclass(1, 2, 3, 4) self.assertEqual(t.__format__(''), str(t)) diff --git a/Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst b/Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst new file mode 100644 index 000000000000..68f5d7c942f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-04-16-12-04.gh-issue-85432.l_ehmI.rst @@ -0,0 +1,5 @@ +Rename the *fmt* parameter of the pure-Python implementation of +:meth:`datetime.time.strftime` to *format*. Rename the *t* parameter of +:meth:`datetime.datetime.fromtimestamp` to *timestamp*. These changes mean +the parameter names in the pure-Python implementation now match the +parameter names in the C implementation. Patch by Alex Waygood. From webhook-mailer at python.org Thu Dec 22 17:35:41 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Thu, 22 Dec 2022 22:35:41 -0000 Subject: [Python-checkins] gh-57762: fix misleading tkinter.Tk docstring (#98837) Message-ID: <mailman.3020.1671748542.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ad23da0e77c4c4a3185df64d1c199b8c17e08188 commit: ad23da0e77c4c4a3185df64d1c199b8c17e08188 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-22T16:35:32-06:00 summary: gh-57762: fix misleading tkinter.Tk docstring (#98837) Mentioned as a desired change by terryjreedy on the corresponding issue, since Tk is not a subclass of Toplevel. files: M Lib/tkinter/__init__.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index a8e7bf490ad4..7565e0f7e460 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2305,7 +2305,7 @@ class Tk(Misc, Wm): def __init__(self, screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None): - """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will be created. BASENAME will be used for the identification of the profile file (see readprofile). It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME From webhook-mailer at python.org Thu Dec 22 18:01:44 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:01:44 -0000 Subject: [Python-checkins] gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Message-ID: <mailman.3021.1671750105.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1e0631e5ba7fcb4882a08d2abdc7614130c07660 commit: 1e0631e5ba7fcb4882a08d2abdc7614130c07660 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:01:38-08:00 summary: gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Mentioned as a desired change by terryjreedy on the corresponding issue, since Tk is not a subclass of Toplevel. (cherry picked from commit ad23da0e77c4c4a3185df64d1c199b8c17e08188) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Lib/tkinter/__init__.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index a8e7bf490ad4..7565e0f7e460 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2305,7 +2305,7 @@ class Tk(Misc, Wm): def __init__(self, screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None): - """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will be created. BASENAME will be used for the identification of the profile file (see readprofile). It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME From webhook-mailer at python.org Thu Dec 22 18:03:43 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:03:43 -0000 Subject: [Python-checkins] gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Message-ID: <mailman.3022.1671750224.3313.python-checkins@python.org> https://github.com/python/cpython/commit/de74d49b21f7671e2cbacf686e63efaee51959f5 commit: de74d49b21f7671e2cbacf686e63efaee51959f5 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:03:38-08:00 summary: gh-57762: fix misleading tkinter.Tk docstring (GH-98837) Mentioned as a desired change by terryjreedy on the corresponding issue, since Tk is not a subclass of Toplevel. (cherry picked from commit ad23da0e77c4c4a3185df64d1c199b8c17e08188) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Lib/tkinter/__init__.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index d42d9a013dd6..7b8dc7bdcd8c 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2278,7 +2278,7 @@ class Tk(Misc, Wm): def __init__(self, screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None): - """Return a new Toplevel widget on screen SCREENNAME. A new Tcl interpreter will + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will be created. BASENAME will be used for the identification of the profile file (see readprofile). It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME From webhook-mailer at python.org Thu Dec 22 18:23:30 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Thu, 22 Dec 2022 23:23:30 -0000 Subject: [Python-checkins] gh-48496: Added example and link to faq for UnboundLocalError in reference (#93068) Message-ID: <mailman.3023.1671751411.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f3db68e6e66ebb36e1b9cb30daba913ecc736169 commit: f3db68e6e66ebb36e1b9cb30daba913ecc736169 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-22T17:23:25-06:00 summary: gh-48496: Added example and link to faq for UnboundLocalError in reference (#93068) files: M Doc/faq/programming.rst M Doc/reference/executionmodel.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 584d33e9622e..c396e2b081fc 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -113,6 +113,8 @@ Yes. The coding style required for standard library modules is documented as Core Language ============= +.. _faq-unboundlocalerror: + Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 3f01180e13f7..a264015cbf40 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -128,6 +128,8 @@ lead to errors when a name is used within a block before it is bound. This rule is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. +See :ref:`the FAQ entry on UnboundLocalError <faq-unboundlocalerror>` +for examples. If the :keyword:`global` statement occurs within a block, all uses of the names specified in the statement refer to the bindings of those names in the top-level From webhook-mailer at python.org Thu Dec 22 18:30:15 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:30:15 -0000 Subject: [Python-checkins] gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) Message-ID: <mailman.3024.1671751815.3313.python-checkins@python.org> https://github.com/python/cpython/commit/86cdfaa885eedefe3a33fccc0bbccd0b6a388260 commit: 86cdfaa885eedefe3a33fccc0bbccd0b6a388260 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:30:09-08:00 summary: gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) (cherry picked from commit f3db68e6e66ebb36e1b9cb30daba913ecc736169) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: M Doc/faq/programming.rst M Doc/reference/executionmodel.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index f8140481b120..bd75801cc242 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -113,6 +113,8 @@ Yes. The coding style required for standard library modules is documented as Core Language ============= +.. _faq-unboundlocalerror: + Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index d9183561820b..081f71cf2e7d 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -128,6 +128,8 @@ lead to errors when a name is used within a block before it is bound. This rule is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. +See :ref:`the FAQ entry on UnboundLocalError <faq-unboundlocalerror>` +for examples. If the :keyword:`global` statement occurs within a block, all uses of the names specified in the statement refer to the bindings of those names in the top-level From webhook-mailer at python.org Thu Dec 22 18:30:18 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 22 Dec 2022 23:30:18 -0000 Subject: [Python-checkins] gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) Message-ID: <mailman.3025.1671751819.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1fa4c6bd1f895df1ca6205a8bd3f60fdd287405c commit: 1fa4c6bd1f895df1ca6205a8bd3f60fdd287405c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-22T15:30:12-08:00 summary: gh-48496: Added example and link to faq for UnboundLocalError in reference (GH-93068) (cherry picked from commit f3db68e6e66ebb36e1b9cb30daba913ecc736169) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: M Doc/faq/programming.rst M Doc/reference/executionmodel.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 584d33e9622e..c396e2b081fc 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -113,6 +113,8 @@ Yes. The coding style required for standard library modules is documented as Core Language ============= +.. _faq-unboundlocalerror: + Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 3f01180e13f7..a264015cbf40 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -128,6 +128,8 @@ lead to errors when a name is used within a block before it is bound. This rule is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. +See :ref:`the FAQ entry on UnboundLocalError <faq-unboundlocalerror>` +for examples. If the :keyword:`global` statement occurs within a block, all uses of the names specified in the statement refer to the bindings of those names in the top-level From webhook-mailer at python.org Fri Dec 23 01:22:37 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Fri, 23 Dec 2022 06:22:37 -0000 Subject: [Python-checkins] Fix typo in 3.12 What's New (#100449) Message-ID: <mailman.3026.1671776557.3313.python-checkins@python.org> https://github.com/python/cpython/commit/73c08eeaffadb73184808e462792b6793ce9f82d commit: 73c08eeaffadb73184808e462792b6793ce9f82d branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T00:22:31-06:00 summary: Fix typo in 3.12 What's New (#100449) files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d480be27542c..617fadd83f8f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -939,7 +939,7 @@ Removed internals. (Contributed by Victor Stinner in :gh:`92651`.) -* Leagcy Unicode APIs has been removed. See :pep:`623` for detail. +* Legacy Unicode APIs has been removed. See :pep:`623` for detail. * :c:macro:`PyUnicode_WCHAR_KIND` * :c:func:`PyUnicode_AS_UNICODE` From webhook-mailer at python.org Fri Dec 23 03:23:44 2022 From: webhook-mailer at python.org (mdickinson) Date: Fri, 23 Dec 2022 08:23:44 -0000 Subject: [Python-checkins] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) Message-ID: <mailman.3027.1671783825.3313.python-checkins@python.org> https://github.com/python/cpython/commit/84bc6a4f25fcf467813ee12b74118f7b1b54e285 commit: 84bc6a4f25fcf467813ee12b74118f7b1b54e285 branch: main author: Eric Wieser <wieser.eric at gmail.com> committer: mdickinson <dickinsm at gmail.com> date: 2022-12-23T08:23:19Z summary: gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) The itemsize returned in a memoryview of a ctypes array is now computed from the item type, instead of dividing the total size by the length and assuming that the length is not zero. files: A Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst M Lib/test/test_ctypes/test_pep3118.py M Modules/_ctypes/_ctypes.c diff --git a/Lib/test/test_ctypes/test_pep3118.py b/Lib/test/test_ctypes/test_pep3118.py index 81e8ca7638fd..efffc80a66fc 100644 --- a/Lib/test/test_ctypes/test_pep3118.py +++ b/Lib/test/test_ctypes/test_pep3118.py @@ -176,7 +176,9 @@ class Complete(Structure): ## arrays and pointers (c_double * 4, "<d", (4,), c_double), + (c_double * 0, "<d", (0,), c_double), (c_float * 4 * 3 * 2, "<f", (2,3,4), c_float), + (c_float * 4 * 0 * 2, "<f", (2,0,4), c_float), (POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)), (POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)), (POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)), diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst new file mode 100644 index 000000000000..841740130bd1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst @@ -0,0 +1,3 @@ +``ctypes`` arrays of length 0 now report a correct itemsize when a +``memoryview`` is constructed from them, rather than always giving a value +of 0. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b9092d3981f3..f69a37709963 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2731,11 +2731,33 @@ static PyMemberDef PyCData_members[] = { { NULL }, }; -static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) +/* Find the innermost type of an array type, returning a borrowed reference */ +static PyObject * +PyCData_item_type(PyObject *type) +{ + if (PyCArrayTypeObject_Check(type)) { + StgDictObject *stg_dict; + PyObject *elem_type; + + /* asserts used here as these are all guaranteed by construction */ + stg_dict = PyType_stgdict(type); + assert(stg_dict); + elem_type = stg_dict->proto; + assert(elem_type); + return PyCData_item_type(elem_type); + } + else { + return type; + } +} + +static int +PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; StgDictObject *dict = PyObject_stgdict(myself); - Py_ssize_t i; + PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); + StgDictObject *item_dict = PyType_stgdict(item_type); if (view == NULL) return 0; @@ -2747,12 +2769,7 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) view->format = dict->format ? dict->format : "B"; view->ndim = dict->ndim; view->shape = dict->shape; - view->itemsize = self->b_size; - if (view->itemsize) { - for (i = 0; i < view->ndim; ++i) { - view->itemsize /= dict->shape[i]; - } - } + view->itemsize = item_dict->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; From webhook-mailer at python.org Fri Dec 23 03:56:01 2022 From: webhook-mailer at python.org (mdickinson) Date: Fri, 23 Dec 2022 08:56:01 -0000 Subject: [Python-checkins] [3.10] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (#100451) Message-ID: <mailman.3028.1671785761.3313.python-checkins@python.org> https://github.com/python/cpython/commit/95c55a69b3935e36108059ea89efd372d0902c6e commit: 95c55a69b3935e36108059ea89efd372d0902c6e branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: mdickinson <dickinsm at gmail.com> date: 2022-12-23T08:55:55Z summary: [3.10] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (#100451) gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) The itemsize returned in a memoryview of a ctypes array is now computed from the item type, instead of dividing the total size by the length and assuming that the length is not zero. (cherry picked from commit 84bc6a4f25fcf467813ee12b74118f7b1b54e285) Co-authored-by: Eric Wieser <wieser.eric at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst M Lib/ctypes/test/test_pep3118.py M Modules/_ctypes/_ctypes.c diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py index 81e8ca7638fd..efffc80a66fc 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/ctypes/test/test_pep3118.py @@ -176,7 +176,9 @@ class Complete(Structure): ## arrays and pointers (c_double * 4, "<d", (4,), c_double), + (c_double * 0, "<d", (0,), c_double), (c_float * 4 * 3 * 2, "<f", (2,3,4), c_float), + (c_float * 4 * 0 * 2, "<f", (2,0,4), c_float), (POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)), (POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)), (POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)), diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst new file mode 100644 index 000000000000..841740130bd1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst @@ -0,0 +1,3 @@ +``ctypes`` arrays of length 0 now report a correct itemsize when a +``memoryview`` is constructed from them, rather than always giving a value +of 0. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 84378c40357b..a534a828d1ee 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2795,11 +2795,33 @@ static PyMemberDef PyCData_members[] = { { NULL }, }; -static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) +/* Find the innermost type of an array type, returning a borrowed reference */ +static PyObject * +PyCData_item_type(PyObject *type) +{ + if (PyCArrayTypeObject_Check(type)) { + StgDictObject *stg_dict; + PyObject *elem_type; + + /* asserts used here as these are all guaranteed by construction */ + stg_dict = PyType_stgdict(type); + assert(stg_dict); + elem_type = stg_dict->proto; + assert(elem_type); + return PyCData_item_type(elem_type); + } + else { + return type; + } +} + +static int +PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; StgDictObject *dict = PyObject_stgdict(myself); - Py_ssize_t i; + PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); + StgDictObject *item_dict = PyType_stgdict(item_type); if (view == NULL) return 0; @@ -2812,12 +2834,7 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) view->format = dict->format ? dict->format : "B"; view->ndim = dict->ndim; view->shape = dict->shape; - view->itemsize = self->b_size; - if (view->itemsize) { - for (i = 0; i < view->ndim; ++i) { - view->itemsize /= dict->shape[i]; - } - } + view->itemsize = item_dict->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; From webhook-mailer at python.org Fri Dec 23 03:56:26 2022 From: webhook-mailer at python.org (mdickinson) Date: Fri, 23 Dec 2022 08:56:26 -0000 Subject: [Python-checkins] [3.11] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (GH-100452) Message-ID: <mailman.3029.1671785786.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f6fe4bb75cb1c7423cfd08d0bf10f9843761c2ac commit: f6fe4bb75cb1c7423cfd08d0bf10f9843761c2ac branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: mdickinson <dickinsm at gmail.com> date: 2022-12-23T08:56:20Z summary: [3.11] gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) (GH-100452) gh-76963: PEP3118 itemsize of an empty ctypes array should not be 0 (GH-5576) The itemsize returned in a memoryview of a ctypes array is now computed from the item type, instead of dividing the total size by the length and assuming that the length is not zero. (cherry picked from commit 84bc6a4f25fcf467813ee12b74118f7b1b54e285) Co-authored-by: Eric Wieser <wieser.eric at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst M Lib/ctypes/test/test_pep3118.py M Modules/_ctypes/_ctypes.c diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py index 81e8ca7638fd..efffc80a66fc 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/ctypes/test/test_pep3118.py @@ -176,7 +176,9 @@ class Complete(Structure): ## arrays and pointers (c_double * 4, "<d", (4,), c_double), + (c_double * 0, "<d", (0,), c_double), (c_float * 4 * 3 * 2, "<f", (2,3,4), c_float), + (c_float * 4 * 0 * 2, "<f", (2,0,4), c_float), (POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)), (POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)), (POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)), diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst new file mode 100644 index 000000000000..841740130bd1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-06-23-21-13.bpo-32782.EJVSfR.rst @@ -0,0 +1,3 @@ +``ctypes`` arrays of length 0 now report a correct itemsize when a +``memoryview`` is constructed from them, rather than always giving a value +of 0. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d6fa11d34813..b5eafe49e4ee 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2775,11 +2775,33 @@ static PyMemberDef PyCData_members[] = { { NULL }, }; -static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) +/* Find the innermost type of an array type, returning a borrowed reference */ +static PyObject * +PyCData_item_type(PyObject *type) +{ + if (PyCArrayTypeObject_Check(type)) { + StgDictObject *stg_dict; + PyObject *elem_type; + + /* asserts used here as these are all guaranteed by construction */ + stg_dict = PyType_stgdict(type); + assert(stg_dict); + elem_type = stg_dict->proto; + assert(elem_type); + return PyCData_item_type(elem_type); + } + else { + return type; + } +} + +static int +PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; StgDictObject *dict = PyObject_stgdict(myself); - Py_ssize_t i; + PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); + StgDictObject *item_dict = PyType_stgdict(item_type); if (view == NULL) return 0; @@ -2792,12 +2814,7 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) view->format = dict->format ? dict->format : "B"; view->ndim = dict->ndim; view->shape = dict->shape; - view->itemsize = self->b_size; - if (view->itemsize) { - for (i = 0; i < view->ndim; ++i) { - view->itemsize /= dict->shape[i]; - } - } + view->itemsize = item_dict->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; From webhook-mailer at python.org Fri Dec 23 09:42:30 2022 From: webhook-mailer at python.org (markshannon) Date: Fri, 23 Dec 2022 14:42:30 -0000 Subject: [Python-checkins] GH-100459: fix copy-paste errors in specialization stats (GH-100460) Message-ID: <mailman.3030.1671806551.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2659036c757a11235c4abd21f02c3a548a344fe7 commit: 2659036c757a11235c4abd21f02c3a548a344fe7 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: markshannon <mark at hotpy.org> date: 2022-12-23T14:42:24Z summary: GH-100459: fix copy-paste errors in specialization stats (GH-100460) files: M Python/specialize.c diff --git a/Python/specialize.c b/Python/specialize.c index 69a7bf98cad0..d9af7b742d54 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -860,7 +860,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) // We *might* not really need this check, but we inherited it from // PyObject_GenericSetAttr and friends... and this way we still do the // right thing if someone forgets to call PyType_Ready(type): - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER); goto fail; } if (PyModule_CheckExact(owner)) { @@ -915,16 +915,16 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; case BUILTIN_CLASSMETHOD: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); goto fail; case PYTHON_CLASSMETHOD: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); goto fail; case NON_OVERRIDING: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR); goto fail; case NON_DESCRIPTOR: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); goto fail; case ABSENT: if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, From webhook-mailer at python.org Fri Dec 23 09:45:59 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 23 Dec 2022 14:45:59 -0000 Subject: [Python-checkins] gh-99110: Initialize `frame->previous` in init_frame to fix segmentation fault when accessing `frame.f_back` (#100182) Message-ID: <mailman.3031.1671806761.3313.python-checkins@python.org> https://github.com/python/cpython/commit/88d565f32a709140664444c6dea20ecd35a10e94 commit: 88d565f32a709140664444c6dea20ecd35a10e94 branch: main author: Bill Fisher <william.w.fisher at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-23T20:15:53+05:30 summary: gh-99110: Initialize `frame->previous` in init_frame to fix segmentation fault when accessing `frame.f_back` (#100182) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst M Include/internal/pycore_frame.h M Lib/test/test_frame.py M Modules/_testcapimodule.c M Objects/frameobject.c diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 7fa410d288c3..f18723b30322 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -96,7 +96,10 @@ static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); -/* Consumes reference to func and locals */ +/* Consumes reference to func and locals. + Does not initialize frame->previous, which happens + when frame is linked into the frame stack. + */ static inline void _PyFrame_InitializeSpecials( _PyInterpreterFrame *frame, PyFunctionObject *func, diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index ed413f105e5b..40c734b6e33a 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -408,6 +408,15 @@ def test_frame_get_generator(self): frame = next(gen) self.assertIs(gen, _testcapi.frame_getgenerator(frame)) + def test_frame_fback_api(self): + """Test that accessing `f_back` does not cause a segmentation fault on + a frame created with `PyFrame_New` (GH-99110).""" + def dummy(): + pass + + frame = _testcapi.frame_new(dummy.__code__, globals(), locals()) + # The following line should not cause a segmentation fault. + self.assertIsNone(frame.f_back) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst new file mode 100644 index 000000000000..175740dfca07 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst @@ -0,0 +1,2 @@ +Initialize frame->previous in frameobject.c to fix a segmentation fault when +accessing frames created by :c:func:`PyFrame_New`. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 35c895d9ceb2..c32fdb5f5fbe 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -20,6 +20,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "frameobject.h" // PyFrame_New #include "marshal.h" // PyMarshal_WriteLongToFile #include "structmember.h" // for offsetof(), T_OBJECT #include <float.h> // FLT_MAX @@ -2839,6 +2840,22 @@ frame_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(lasti); } +static PyObject * +frame_new(PyObject *self, PyObject *args) +{ + PyObject *code, *globals, *locals; + if (!PyArg_ParseTuple(args, "OOO", &code, &globals, &locals)) { + return NULL; + } + if (!PyCode_Check(code)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); + return NULL; + } + PyThreadState *tstate = PyThreadState_Get(); + + return (PyObject *)PyFrame_New(tstate, (PyCodeObject *)code, globals, locals); +} + static PyObject * test_frame_getvar(PyObject *self, PyObject *args) { @@ -3277,6 +3294,7 @@ static PyMethodDef TestMethods[] = { {"frame_getgenerator", frame_getgenerator, METH_O, NULL}, {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL}, {"frame_getlasti", frame_getlasti, METH_O, NULL}, + {"frame_new", frame_new, METH_VARARGS, NULL}, {"frame_getvar", test_frame_getvar, METH_VARARGS, NULL}, {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL}, {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 74c26d8d4d96..eab85c08fc01 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1013,6 +1013,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) PyCodeObject *code = (PyCodeObject *)func->func_code; _PyFrame_InitializeSpecials(frame, (PyFunctionObject*)Py_NewRef(func), Py_XNewRef(locals), code); + frame->previous = NULL; for (Py_ssize_t i = 0; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = NULL; } From webhook-mailer at python.org Fri Dec 23 10:00:33 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Fri, 23 Dec 2022 15:00:33 -0000 Subject: [Python-checkins] gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (#98710) Message-ID: <mailman.3032.1671807634.3313.python-checkins@python.org> https://github.com/python/cpython/commit/49f6ff719c4e0beeafd6c42edd696601acf72764 commit: 49f6ff719c4e0beeafd6c42edd696601acf72764 branch: main author: Petr Viktorin <encukou at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-23T07:00:21-08:00 summary: gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (#98710) files: M Doc/c-api/arg.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index c5be453c1533..9713431688d4 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -34,24 +34,39 @@ These formats allow accessing an object as a contiguous chunk of memory. You don't have to provide raw storage for the returned unicode or bytes area. -In general, when a format sets a pointer to a buffer, the buffer is -managed by the corresponding Python object, and the buffer shares -the lifetime of this object. You won't have to release any memory yourself. -The only exceptions are ``es``, ``es#``, ``et`` and ``et#``. - -However, when a :c:type:`Py_buffer` structure gets filled, the underlying -buffer is locked so that the caller can subsequently use the buffer even -inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data -being resized or destroyed. As a result, **you have to call** -:c:func:`PyBuffer_Release` after you have finished processing the data (or -in any early abort case). - Unless otherwise stated, buffers are not NUL-terminated. -Some formats require a read-only :term:`bytes-like object`, and set a -pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, -which disallows mutable objects such as :class:`bytearray`. +There are three ways strings and buffers can be converted to C: + +* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure. + This locks the underlying buffer so that the caller can subsequently use + the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS` + block without the risk of mutable data being resized or destroyed. + As a result, **you have to call** :c:func:`PyBuffer_Release` after you have + finished processing the data (or in any early abort case). + +* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer. + **You have to call** :c:func:`PyMem_Free` after you have finished + processing the data (or in any early abort case). + +* .. _c-arg-borrowed-buffer: + + Other formats take a :class:`str` or a read-only :term:`bytes-like object`, + such as :class:`bytes`, and provide a ``const char *`` pointer to + its buffer. + In this case the buffer is "borrowed": it is managed by the corresponding + Python object, and shares the lifetime of this object. + You won't have to release any memory yourself. + + To ensure that the underlying buffer may be safely borrowed, the object's + :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``. + This disallows common mutable objects such as :class:`bytearray`, + but also some read-only objects such as :class:`memoryview` of + :class:`bytes`. + + Besides this ``bf_releasebuffer`` requirement, there is no check to verify + whether the input object is immutable (e.g. whether it would honor a request + for a writable buffer, or whether another thread can mutate the data). .. note:: @@ -89,7 +104,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode objects are converted to C strings using ``'utf-8'`` encoding. ``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable objects. + Like ``s*``, except that it provides a :ref:`borrowed buffer <c-arg-borrowed-buffer>`. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. The string may contain embedded null bytes. Unicode objects are converted @@ -108,8 +123,9 @@ which disallows mutable objects such as :class:`bytearray`. pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a character - string; it does not accept Unicode objects. The bytes buffer must not + This format converts a bytes-like object to a C pointer to a + :ref:`borrowed <c-arg-borrowed-buffer>` character string; + it does not accept Unicode objects. The bytes buffer must not contain embedded null bytes; if it does, a :exc:`ValueError` exception is raised. From webhook-mailer at python.org Fri Dec 23 10:08:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 15:08:17 -0000 Subject: [Python-checkins] gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) Message-ID: <mailman.3033.1671808098.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4b3b6423c3667d8c9350fb920a5a69824c5250ff commit: 4b3b6423c3667d8c9350fb920a5a69824c5250ff branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T07:08:11-08:00 summary: gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) (cherry picked from commit 49f6ff719c4e0beeafd6c42edd696601acf72764) Co-authored-by: Petr Viktorin <encukou at gmail.com> files: M Doc/c-api/arg.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 85f9eda17a20..6a53c79bd3be 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -34,24 +34,39 @@ These formats allow accessing an object as a contiguous chunk of memory. You don't have to provide raw storage for the returned unicode or bytes area. -In general, when a format sets a pointer to a buffer, the buffer is -managed by the corresponding Python object, and the buffer shares -the lifetime of this object. You won't have to release any memory yourself. -The only exceptions are ``es``, ``es#``, ``et`` and ``et#``. - -However, when a :c:type:`Py_buffer` structure gets filled, the underlying -buffer is locked so that the caller can subsequently use the buffer even -inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data -being resized or destroyed. As a result, **you have to call** -:c:func:`PyBuffer_Release` after you have finished processing the data (or -in any early abort case). - Unless otherwise stated, buffers are not NUL-terminated. -Some formats require a read-only :term:`bytes-like object`, and set a -pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, -which disallows mutable objects such as :class:`bytearray`. +There are three ways strings and buffers can be converted to C: + +* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure. + This locks the underlying buffer so that the caller can subsequently use + the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS` + block without the risk of mutable data being resized or destroyed. + As a result, **you have to call** :c:func:`PyBuffer_Release` after you have + finished processing the data (or in any early abort case). + +* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer. + **You have to call** :c:func:`PyMem_Free` after you have finished + processing the data (or in any early abort case). + +* .. _c-arg-borrowed-buffer: + + Other formats take a :class:`str` or a read-only :term:`bytes-like object`, + such as :class:`bytes`, and provide a ``const char *`` pointer to + its buffer. + In this case the buffer is "borrowed": it is managed by the corresponding + Python object, and shares the lifetime of this object. + You won't have to release any memory yourself. + + To ensure that the underlying buffer may be safely borrowed, the object's + :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``. + This disallows common mutable objects such as :class:`bytearray`, + but also some read-only objects such as :class:`memoryview` of + :class:`bytes`. + + Besides this ``bf_releasebuffer`` requirement, there is no check to verify + whether the input object is immutable (e.g. whether it would honor a request + for a writable buffer, or whether another thread can mutate the data). .. note:: @@ -89,7 +104,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode objects are converted to C strings using ``'utf-8'`` encoding. ``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable objects. + Like ``s*``, except that it provides a :ref:`borrowed buffer <c-arg-borrowed-buffer>`. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. The string may contain embedded null bytes. Unicode objects are converted @@ -108,8 +123,9 @@ which disallows mutable objects such as :class:`bytearray`. pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a character - string; it does not accept Unicode objects. The bytes buffer must not + This format converts a bytes-like object to a C pointer to a + :ref:`borrowed <c-arg-borrowed-buffer>` character string; + it does not accept Unicode objects. The bytes buffer must not contain embedded null bytes; if it does, a :exc:`ValueError` exception is raised. From webhook-mailer at python.org Fri Dec 23 10:09:40 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 15:09:40 -0000 Subject: [Python-checkins] gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) Message-ID: <mailman.3034.1671808181.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bd472198c6ab7d69657aa3d70df1320584cf98b2 commit: bd472198c6ab7d69657aa3d70df1320584cf98b2 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T07:09:34-08:00 summary: gh-98712: Clarify "readonly bytes-like object" semantics in C arg-parsing docs (GH-98710) (cherry picked from commit 49f6ff719c4e0beeafd6c42edd696601acf72764) Co-authored-by: Petr Viktorin <encukou at gmail.com> files: M Doc/c-api/arg.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 85f9eda17a20..6a53c79bd3be 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -34,24 +34,39 @@ These formats allow accessing an object as a contiguous chunk of memory. You don't have to provide raw storage for the returned unicode or bytes area. -In general, when a format sets a pointer to a buffer, the buffer is -managed by the corresponding Python object, and the buffer shares -the lifetime of this object. You won't have to release any memory yourself. -The only exceptions are ``es``, ``es#``, ``et`` and ``et#``. - -However, when a :c:type:`Py_buffer` structure gets filled, the underlying -buffer is locked so that the caller can subsequently use the buffer even -inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data -being resized or destroyed. As a result, **you have to call** -:c:func:`PyBuffer_Release` after you have finished processing the data (or -in any early abort case). - Unless otherwise stated, buffers are not NUL-terminated. -Some formats require a read-only :term:`bytes-like object`, and set a -pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, -which disallows mutable objects such as :class:`bytearray`. +There are three ways strings and buffers can be converted to C: + +* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure. + This locks the underlying buffer so that the caller can subsequently use + the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS` + block without the risk of mutable data being resized or destroyed. + As a result, **you have to call** :c:func:`PyBuffer_Release` after you have + finished processing the data (or in any early abort case). + +* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer. + **You have to call** :c:func:`PyMem_Free` after you have finished + processing the data (or in any early abort case). + +* .. _c-arg-borrowed-buffer: + + Other formats take a :class:`str` or a read-only :term:`bytes-like object`, + such as :class:`bytes`, and provide a ``const char *`` pointer to + its buffer. + In this case the buffer is "borrowed": it is managed by the corresponding + Python object, and shares the lifetime of this object. + You won't have to release any memory yourself. + + To ensure that the underlying buffer may be safely borrowed, the object's + :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``. + This disallows common mutable objects such as :class:`bytearray`, + but also some read-only objects such as :class:`memoryview` of + :class:`bytes`. + + Besides this ``bf_releasebuffer`` requirement, there is no check to verify + whether the input object is immutable (e.g. whether it would honor a request + for a writable buffer, or whether another thread can mutate the data). .. note:: @@ -89,7 +104,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode objects are converted to C strings using ``'utf-8'`` encoding. ``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable objects. + Like ``s*``, except that it provides a :ref:`borrowed buffer <c-arg-borrowed-buffer>`. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. The string may contain embedded null bytes. Unicode objects are converted @@ -108,8 +123,9 @@ which disallows mutable objects such as :class:`bytearray`. pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a character - string; it does not accept Unicode objects. The bytes buffer must not + This format converts a bytes-like object to a C pointer to a + :ref:`borrowed <c-arg-borrowed-buffer>` character string; + it does not accept Unicode objects. The bytes buffer must not contain embedded null bytes; if it does, a :exc:`ValueError` exception is raised. From webhook-mailer at python.org Fri Dec 23 10:23:41 2022 From: webhook-mailer at python.org (Fidget-Spinner) Date: Fri, 23 Dec 2022 15:23:41 -0000 Subject: [Python-checkins] gh-92216: improve performance of `hasattr` for type objects (GH-99979) Message-ID: <mailman.3035.1671809022.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7fc7909677759fd0cb9d30cd7ff839afd3973325 commit: 7fc7909677759fd0cb9d30cd7ff839afd3973325 branch: main author: Pieter Eendebak <pieter.eendebak at gmail.com> committer: Fidget-Spinner <kenjin at python.org> date: 2022-12-23T23:23:36+08:00 summary: gh-92216: improve performance of `hasattr` for type objects (GH-99979) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst M Include/internal/pycore_typeobject.h M Objects/object.c M Objects/typeobject.c diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c207ce615c6f..4d705740a9a6 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -74,6 +74,10 @@ extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *); extern void _PyStaticType_ClearWeakRefs(PyTypeObject *type); extern void _PyStaticType_Dealloc(PyTypeObject *type); +PyObject * +_Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); +PyObject * +_Py_type_getattro(PyTypeObject *type, PyObject *name); PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst new file mode 100644 index 000000000000..f7ef52d97c27 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-04-00-38-33.gh-issue-92216.CJXuWB.rst @@ -0,0 +1 @@ +Improve the performance of :func:`hasattr` for type objects with a missing attribute. diff --git a/Objects/object.c b/Objects/object.c index 028b0edc9111..fae508cae3d6 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -939,7 +939,15 @@ _PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result) } return 0; } - if (tp->tp_getattro != NULL) { + if (tp->tp_getattro == (getattrofunc)_Py_type_getattro) { + int supress_missing_attribute_exception = 0; + *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); + if (supress_missing_attribute_exception) { + // return 0 without having to clear the exception + return 0; + } + } + else if (tp->tp_getattro != NULL) { *result = (*tp->tp_getattro)(v, name); } else if (tp->tp_getattr != NULL) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a96f993e99dd..16b1a3035d56 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4219,9 +4219,19 @@ _PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ -static PyObject * -type_getattro(PyTypeObject *type, PyObject *name) + but uses _PyType_Lookup() instead of just looking in type->tp_dict. + + The argument suppress_missing_attribute is used to provide a + fast path for hasattr. The possible values are: + + * NULL: do not suppress the exception + * Non-zero pointer: suppress the PyExc_AttributeError and + set *suppress_missing_attribute to 1 to signal we are returning NULL while + having suppressed the exception (other exceptions are not suppressed) + + */ +PyObject * +_Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missing_attribute) { PyTypeObject *metatype = Py_TYPE(type); PyObject *meta_attribute, *attribute; @@ -4301,12 +4311,25 @@ type_getattro(PyTypeObject *type, PyObject *name) } /* Give up */ - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - type->tp_name, name); + if (suppress_missing_attribute == NULL) { + PyErr_Format(PyExc_AttributeError, + "type object '%.50s' has no attribute '%U'", + type->tp_name, name); + } else { + // signal the caller we have not set an PyExc_AttributeError and gave up + *suppress_missing_attribute = 1; + } return NULL; } +/* This is similar to PyObject_GenericGetAttr(), + but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ +PyObject * +_Py_type_getattro(PyTypeObject *type, PyObject *name) +{ + return _Py_type_getattro_impl(type, name, NULL); +} + static int type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) { @@ -4798,7 +4821,7 @@ PyTypeObject PyType_Type = { 0, /* tp_hash */ (ternaryfunc)type_call, /* tp_call */ 0, /* tp_str */ - (getattrofunc)type_getattro, /* tp_getattro */ + (getattrofunc)_Py_type_getattro, /* tp_getattro */ (setattrofunc)type_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | From webhook-mailer at python.org Fri Dec 23 11:26:48 2022 From: webhook-mailer at python.org (Fidget-Spinner) Date: Fri, 23 Dec 2022 16:26:48 -0000 Subject: [Python-checkins] gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289) Message-ID: <mailman.3036.1671812808.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c3c7848a48b74a321632202e4bdcf2f465fb1cc6 commit: c3c7848a48b74a321632202e4bdcf2f465fb1cc6 branch: main author: Ken Jin <kenjin at python.org> committer: Fidget-Spinner <kenjin at python.org> date: 2022-12-24T00:26:42+08:00 summary: gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst M Include/internal/pycore_opcode.h M Include/opcode.h M Lib/opcode.py M Python/bytecodes.c M Python/generated_cases.c.h M Python/opcode_targets.h M Python/specialize.c diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index da8a272f2fa2..d702580d4980 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -152,6 +152,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_MANAGED_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, @@ -313,12 +314,12 @@ static const char *const _PyOpcode_OpName[263] = { [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_MANAGED_DICT] = "LOAD_ATTR_METHOD_MANAGED_DICT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -345,7 +346,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -353,7 +354,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -373,9 +374,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -385,27 +386,27 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", - [173] = "<173>", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [174] = "<174>", [175] = "<175>", [176] = "<176>", @@ -499,7 +500,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 173: \ case 174: \ case 175: \ case 176: \ diff --git a/Include/opcode.h b/Include/opcode.h index 888250ed37e8..088c1a5d8ca5 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -174,23 +174,24 @@ extern "C" { #define LOAD_ATTR_WITH_HINT 78 #define LOAD_ATTR_METHOD_LAZY_DICT 79 #define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_WITH_DICT 81 -#define LOAD_ATTR_METHOD_WITH_VALUES 86 -#define LOAD_CONST__LOAD_FAST 113 -#define LOAD_FAST__LOAD_CONST 121 -#define LOAD_FAST__LOAD_FAST 141 -#define LOAD_GLOBAL_BUILTIN 143 -#define LOAD_GLOBAL_MODULE 153 -#define STORE_ATTR_INSTANCE_VALUE 154 -#define STORE_ATTR_SLOT 158 -#define STORE_ATTR_WITH_HINT 159 -#define STORE_FAST__LOAD_FAST 160 -#define STORE_FAST__STORE_FAST 161 -#define STORE_SUBSCR_DICT 166 -#define STORE_SUBSCR_LIST_INT 167 -#define UNPACK_SEQUENCE_LIST 168 -#define UNPACK_SEQUENCE_TUPLE 169 -#define UNPACK_SEQUENCE_TWO_TUPLE 170 +#define LOAD_ATTR_METHOD_MANAGED_DICT 81 +#define LOAD_ATTR_METHOD_WITH_DICT 86 +#define LOAD_ATTR_METHOD_WITH_VALUES 113 +#define LOAD_CONST__LOAD_FAST 121 +#define LOAD_FAST__LOAD_CONST 141 +#define LOAD_FAST__LOAD_FAST 143 +#define LOAD_GLOBAL_BUILTIN 153 +#define LOAD_GLOBAL_MODULE 154 +#define STORE_ATTR_INSTANCE_VALUE 158 +#define STORE_ATTR_SLOT 159 +#define STORE_ATTR_WITH_HINT 160 +#define STORE_FAST__LOAD_FAST 161 +#define STORE_FAST__STORE_FAST 166 +#define STORE_SUBSCR_DICT 167 +#define STORE_SUBSCR_LIST_INT 168 +#define UNPACK_SEQUENCE_LIST 169 +#define UNPACK_SEQUENCE_TUPLE 170 +#define UNPACK_SEQUENCE_TWO_TUPLE 173 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index fc57affbac58..59f368c0d245 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -336,6 +336,7 @@ def pseudo_op(name, op, real_ops): # These will always push [unbound method, self] onto the stack. "LOAD_ATTR_METHOD_LAZY_DICT", "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_MANAGED_DICT", "LOAD_ATTR_METHOD_WITH_DICT", "LOAD_ATTR_METHOD_WITH_VALUES", ], diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst new file mode 100644 index 000000000000..45182108b1a4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst @@ -0,0 +1 @@ +Specialize method loading for objects with ``Py_TPFLAGS_MANAGED_DICT`` set. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c0b625bd662c..c2f6a7add53a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2717,6 +2717,31 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } + // error: LOAD_ATTR has irregular stack effect + inst(LOAD_ATTR_METHOD_MANAGED_DICT) { + assert(cframe.use_tracing == 0); + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + uint32_t type_version = read_u32(cache->type_version); + assert(type_version != 0); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) + assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyObject *dict = _PyDictOrValues_GetDict(dorv); + PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; + // Note: cache->keys_version can be 0 when dict is NULL. + DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + SET_TOP(Py_NewRef(res)); + PUSH(self); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 42b7ca086705..0012ba82e948 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2964,6 +2964,31 @@ DISPATCH(); } + TARGET(LOAD_ATTR_METHOD_MANAGED_DICT) { + assert(cframe.use_tracing == 0); + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + uint32_t type_version = read_u32(cache->type_version); + assert(type_version != 0); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) + assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyObject *dict = _PyDictOrValues_GetDict(dorv); + PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; + // Note: cache->keys_version can be 0 when dict is NULL. + DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + SET_TOP(Py_NewRef(res)); + PUSH(self); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + DISPATCH(); + } + TARGET(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ assert(cframe.use_tracing == 0); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index be3ad01c151c..67f5bacecc8c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -80,12 +80,12 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_MANAGED_DICT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,27 +152,27 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&_unknown_opcode, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index d9af7b742d54..53215748c99b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -335,7 +335,6 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 -#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -1036,11 +1035,14 @@ PyObject *descr, DescriptorClassification kind) PyDictKeysObject *keys; if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; if (_PyDictOrValues_IsValues(dorv)) { + keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; dictkind = MANAGED_VALUES; } else { + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + keys = dict != NULL ? dict->ma_keys : NULL; + // User has directly accessed __dict__. dictkind = MANAGED_DICT; } } @@ -1067,7 +1069,7 @@ PyObject *descr, DescriptorClassification kind) } } } - if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT) { + if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT || (dictkind == MANAGED_DICT && keys != NULL)) { Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != DKIX_EMPTY) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); @@ -1088,8 +1090,11 @@ PyObject *descr, DescriptorClassification kind) _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); - goto fail; + if (keys == NULL) { + write_u32(cache->keys_version, 0); + } + _py_set_opcode(instr, LOAD_ATTR_METHOD_MANAGED_DICT); + break; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_DICT); From webhook-mailer at python.org Fri Dec 23 12:48:52 2022 From: webhook-mailer at python.org (Fidget-Spinner) Date: Fri, 23 Dec 2022 17:48:52 -0000 Subject: [Python-checkins] Revert "gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289)" (#100468) Message-ID: <mailman.3037.1671817733.3313.python-checkins@python.org> https://github.com/python/cpython/commit/36d358348de8efad75ebcf55dad8ed4a4f6dcda9 commit: 36d358348de8efad75ebcf55dad8ed4a4f6dcda9 branch: main author: Ken Jin <kenjin at python.org> committer: Fidget-Spinner <kenjin at python.org> date: 2022-12-24T01:48:43+08:00 summary: Revert "gh-100288: Specialise LOAD_ATTR_METHOD for managed dictionaries (GH-100289)" (#100468) This reverts commit c3c7848a48b74a321632202e4bdcf2f465fb1cc6. files: D Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst M Include/internal/pycore_opcode.h M Include/opcode.h M Lib/opcode.py M Python/bytecodes.c M Python/generated_cases.c.h M Python/opcode_targets.h M Python/specialize.c diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index d702580d4980..da8a272f2fa2 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -152,7 +152,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_MANAGED_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_DICT] = LOAD_ATTR, [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, @@ -314,12 +313,12 @@ static const char *const _PyOpcode_OpName[263] = { [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_MANAGED_DICT] = "LOAD_ATTR_METHOD_MANAGED_DICT", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -346,7 +345,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -354,7 +353,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -374,9 +373,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -386,27 +385,27 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [173] = "<173>", [174] = "<174>", [175] = "<175>", [176] = "<176>", @@ -500,6 +499,7 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 173: \ case 174: \ case 175: \ case 176: \ diff --git a/Include/opcode.h b/Include/opcode.h index 088c1a5d8ca5..888250ed37e8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -174,24 +174,23 @@ extern "C" { #define LOAD_ATTR_WITH_HINT 78 #define LOAD_ATTR_METHOD_LAZY_DICT 79 #define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_MANAGED_DICT 81 -#define LOAD_ATTR_METHOD_WITH_DICT 86 -#define LOAD_ATTR_METHOD_WITH_VALUES 113 -#define LOAD_CONST__LOAD_FAST 121 -#define LOAD_FAST__LOAD_CONST 141 -#define LOAD_FAST__LOAD_FAST 143 -#define LOAD_GLOBAL_BUILTIN 153 -#define LOAD_GLOBAL_MODULE 154 -#define STORE_ATTR_INSTANCE_VALUE 158 -#define STORE_ATTR_SLOT 159 -#define STORE_ATTR_WITH_HINT 160 -#define STORE_FAST__LOAD_FAST 161 -#define STORE_FAST__STORE_FAST 166 -#define STORE_SUBSCR_DICT 167 -#define STORE_SUBSCR_LIST_INT 168 -#define UNPACK_SEQUENCE_LIST 169 -#define UNPACK_SEQUENCE_TUPLE 170 -#define UNPACK_SEQUENCE_TWO_TUPLE 173 +#define LOAD_ATTR_METHOD_WITH_DICT 81 +#define LOAD_ATTR_METHOD_WITH_VALUES 86 +#define LOAD_CONST__LOAD_FAST 113 +#define LOAD_FAST__LOAD_CONST 121 +#define LOAD_FAST__LOAD_FAST 141 +#define LOAD_GLOBAL_BUILTIN 143 +#define LOAD_GLOBAL_MODULE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 160 +#define STORE_FAST__STORE_FAST 161 +#define STORE_SUBSCR_DICT 166 +#define STORE_SUBSCR_LIST_INT 167 +#define UNPACK_SEQUENCE_LIST 168 +#define UNPACK_SEQUENCE_TUPLE 169 +#define UNPACK_SEQUENCE_TWO_TUPLE 170 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 59f368c0d245..fc57affbac58 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -336,7 +336,6 @@ def pseudo_op(name, op, real_ops): # These will always push [unbound method, self] onto the stack. "LOAD_ATTR_METHOD_LAZY_DICT", "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_MANAGED_DICT", "LOAD_ATTR_METHOD_WITH_DICT", "LOAD_ATTR_METHOD_WITH_VALUES", ], diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst deleted file mode 100644 index 45182108b1a4..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-12-16-08-50-16.gh-issue-100288.5XCk0G.rst +++ /dev/null @@ -1 +0,0 @@ -Specialize method loading for objects with ``Py_TPFLAGS_MANAGED_DICT`` set. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c2f6a7add53a..c0b625bd662c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2717,31 +2717,6 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_METHOD_MANAGED_DICT) { - assert(cframe.use_tracing == 0); - PyObject *self = TOP(); - PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); - assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyObject *dict = _PyDictOrValues_GetDict(dorv); - PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; - // Note: cache->keys_version can be 0 when dict is NULL. - DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0012ba82e948..42b7ca086705 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2964,31 +2964,6 @@ DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_MANAGED_DICT) { - assert(cframe.use_tracing == 0); - PyObject *self = TOP(); - PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); - assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR) - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyObject *dict = _PyDictOrValues_GetDict(dorv); - PyDictKeysObject *keys = (dict == NULL) ? NULL : ((PyDictObject *)dict)->ma_keys; - // Note: cache->keys_version can be 0 when dict is NULL. - DEOPT_IF(keys != NULL && keys->dk_version != read_u32(cache->keys_version), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - DISPATCH(); - } - TARGET(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ assert(cframe.use_tracing == 0); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 67f5bacecc8c..be3ad01c151c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -80,12 +80,12 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_MANAGED_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,27 +152,27 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 53215748c99b..d9af7b742d54 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -335,6 +335,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 +#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -1035,14 +1036,11 @@ PyObject *descr, DescriptorClassification kind) PyDictKeysObject *keys; if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; if (_PyDictOrValues_IsValues(dorv)) { - keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; dictkind = MANAGED_VALUES; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - keys = dict != NULL ? dict->ma_keys : NULL; - // User has directly accessed __dict__. dictkind = MANAGED_DICT; } } @@ -1069,7 +1067,7 @@ PyObject *descr, DescriptorClassification kind) } } } - if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT || (dictkind == MANAGED_DICT && keys != NULL)) { + if (dictkind == MANAGED_VALUES || dictkind == OFFSET_DICT) { Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != DKIX_EMPTY) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); @@ -1090,11 +1088,8 @@ PyObject *descr, DescriptorClassification kind) _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); break; case MANAGED_DICT: - if (keys == NULL) { - write_u32(cache->keys_version, 0); - } - _py_set_opcode(instr, LOAD_ATTR_METHOD_MANAGED_DICT); - break; + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); + goto fail; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_DICT); From webhook-mailer at python.org Fri Dec 23 13:15:53 2022 From: webhook-mailer at python.org (sweeneyde) Date: Fri, 23 Dec 2022 18:15:53 -0000 Subject: [Python-checkins] gh-94155: Reduce hash collisions for code objects (#100183) Message-ID: <mailman.3038.1671819354.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a98d9ea56e7b473af54438ecc487a6bf1b4d6530 commit: a98d9ea56e7b473af54438ecc487a6bf1b4d6530 branch: main author: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com> committer: sweeneyde <36520290+sweeneyde at users.noreply.github.com> date: 2022-12-23T13:15:47-05:00 summary: gh-94155: Reduce hash collisions for code objects (#100183) * Uses a better hashing algorithm to get better dispersion and remove commutativity. * Incorporates `co_firstlineno`, `Py_SIZE(co)`, and bytecode instructions. * This is now the entire set of criteria used in `code_richcompare`, except for `_PyCode_ConstantKey` (which would incorporate the types of `co_consts` rather than just their values). files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst M Lib/test/test_code.py M Objects/codeobject.c diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 02ab8fbcdb07..b13d5770abe8 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -465,6 +465,32 @@ def f(): self.assertNotEqual(code_b, code_d) self.assertNotEqual(code_c, code_d) + def test_code_hash_uses_firstlineno(self): + c1 = (lambda: 1).__code__ + c2 = (lambda: 1).__code__ + self.assertNotEqual(c1, c2) + self.assertNotEqual(hash(c1), hash(c2)) + c3 = c1.replace(co_firstlineno=17) + self.assertNotEqual(c1, c3) + self.assertNotEqual(hash(c1), hash(c3)) + + def test_code_hash_uses_order(self): + # Swapping posonlyargcount and kwonlyargcount should change the hash. + c = (lambda x, y, *, z=1, w=1: 1).__code__ + self.assertEqual(c.co_argcount, 2) + self.assertEqual(c.co_posonlyargcount, 0) + self.assertEqual(c.co_kwonlyargcount, 2) + swapped = c.replace(co_posonlyargcount=2, co_kwonlyargcount=0) + self.assertNotEqual(c, swapped) + self.assertNotEqual(hash(c), hash(swapped)) + + def test_code_hash_uses_bytecode(self): + c = (lambda x, y: x + y).__code__ + d = (lambda x, y: x * y).__code__ + c1 = c.replace(co_code=d.co_code) + self.assertNotEqual(c, c1) + self.assertNotEqual(hash(c), hash(c1)) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst new file mode 100644 index 000000000000..e7c7ed2fad0e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-00-59-11.gh-issue-94155.LWE9y_.rst @@ -0,0 +1 @@ +Improved the hashing algorithm for code objects, mitigating some hash collisions. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 1e5a92270be8..e174c6fee9cc 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1842,28 +1842,41 @@ code_richcompare(PyObject *self, PyObject *other, int op) static Py_hash_t code_hash(PyCodeObject *co) { - Py_hash_t h, h0, h1, h2, h3; - h0 = PyObject_Hash(co->co_name); - if (h0 == -1) return -1; - h1 = PyObject_Hash(co->co_consts); - if (h1 == -1) return -1; - h2 = PyObject_Hash(co->co_names); - if (h2 == -1) return -1; - h3 = PyObject_Hash(co->co_localsplusnames); - if (h3 == -1) return -1; - Py_hash_t h4 = PyObject_Hash(co->co_linetable); - if (h4 == -1) { - return -1; + Py_uhash_t uhash = 20221211; + #define SCRAMBLE_IN(H) do { \ + uhash ^= (Py_uhash_t)(H); \ + uhash *= _PyHASH_MULTIPLIER; \ + } while (0) + #define SCRAMBLE_IN_HASH(EXPR) do { \ + Py_hash_t h = PyObject_Hash(EXPR); \ + if (h == -1) { \ + return -1; \ + } \ + SCRAMBLE_IN(h); \ + } while (0) + + SCRAMBLE_IN_HASH(co->co_name); + SCRAMBLE_IN_HASH(co->co_consts); + SCRAMBLE_IN_HASH(co->co_names); + SCRAMBLE_IN_HASH(co->co_localsplusnames); + SCRAMBLE_IN_HASH(co->co_linetable); + SCRAMBLE_IN_HASH(co->co_exceptiontable); + SCRAMBLE_IN(co->co_argcount); + SCRAMBLE_IN(co->co_posonlyargcount); + SCRAMBLE_IN(co->co_kwonlyargcount); + SCRAMBLE_IN(co->co_flags); + SCRAMBLE_IN(co->co_firstlineno); + SCRAMBLE_IN(Py_SIZE(co)); + for (int i = 0; i < Py_SIZE(co); i++) { + int deop = _PyOpcode_Deopt[_Py_OPCODE(_PyCode_CODE(co)[i])]; + SCRAMBLE_IN(deop); + SCRAMBLE_IN(_Py_OPARG(_PyCode_CODE(co)[i])); + i += _PyOpcode_Caches[deop]; } - Py_hash_t h5 = PyObject_Hash(co->co_exceptiontable); - if (h5 == -1) { - return -1; + if ((Py_hash_t)uhash == -1) { + return -2; } - h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ - co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^ - co->co_flags; - if (h == -1) h = -2; - return h; + return (Py_hash_t)uhash; } From webhook-mailer at python.org Fri Dec 23 14:41:43 2022 From: webhook-mailer at python.org (cjw296) Date: Fri, 23 Dec 2022 19:41:43 -0000 Subject: [Python-checkins] gh-83076: 3.8x speed improvement in (Async)Mock instantiation (#100252) Message-ID: <mailman.3039.1671824504.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c5726b727e26b81a267933654cf26b760a90d9aa commit: c5726b727e26b81a267933654cf26b760a90d9aa branch: main author: Carl Meyer <carl at oddbird.net> committer: cjw296 <chris at withers.org> date: 2022-12-23T19:41:37Z summary: gh-83076: 3.8x speed improvement in (Async)Mock instantiation (#100252) files: A Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst M Lib/test/test_unittest/testmock/testasync.py M Lib/unittest/mock.py diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index e05a22861d47..52a3b71be1ef 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -300,6 +300,19 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_async_attributes_instance(self): + async_instance = AsyncClass() + async_instance.async_func_attr = async_func + async_instance.later_async_func_attr = normal_func + + mock_async_instance = Mock(spec_set=async_instance) + + async_instance.later_async_func_attr = async_func + + self.assertIsInstance(mock_async_instance.async_func_attr, AsyncMock) + # only the shape of the spec at the time of mock construction matters + self.assertNotIsInstance(mock_async_instance.later_async_func_attr, AsyncMock) + def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index a273753d6a0a..583ab74a8255 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -411,15 +411,18 @@ class NonCallableMock(Base): # necessary. _lock = RLock() - def __new__(cls, /, *args, **kw): + def __new__( + cls, spec=None, wraps=None, name=None, spec_set=None, + parent=None, _spec_state=None, _new_name='', _new_parent=None, + _spec_as_instance=False, _eat_self=None, unsafe=False, **kwargs + ): # every instance has its own class # so we can create magic methods on the # class without stomping on other mocks bases = (cls,) if not issubclass(cls, AsyncMockMixin): # Check if spec is an async object or function - bound_args = _MOCK_SIG.bind_partial(cls, *args, **kw).arguments - spec_arg = bound_args.get('spec_set', bound_args.get('spec')) + spec_arg = spec_set or spec if spec_arg is not None and _is_async_obj(spec_arg): bases = (AsyncMockMixin, cls) new = type(cls.__name__, bases, {'__doc__': cls.__doc__}) @@ -505,10 +508,6 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_signature = None _spec_asyncs = [] - for attr in dir(spec): - if iscoroutinefunction(getattr(spec, attr, None)): - _spec_asyncs.append(attr) - if spec is not None and not _is_list(spec): if isinstance(spec, type): _spec_class = spec @@ -518,7 +517,13 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_as_instance, _eat_self) _spec_signature = res and res[1] - spec = dir(spec) + spec_list = dir(spec) + + for attr in spec_list: + if iscoroutinefunction(getattr(spec, attr, None)): + _spec_asyncs.append(attr) + + spec = spec_list __dict__ = self.__dict__ __dict__['_spec_class'] = _spec_class @@ -1057,9 +1062,6 @@ def _calls_repr(self, prefix="Calls"): return f"\n{prefix}: {safe_repr(self.mock_calls)}." -_MOCK_SIG = inspect.signature(NonCallableMock.__init__) - - class _AnyComparer(list): """A list which checks if it contains a call which may have an argument of ANY, flipping the components of item and self from @@ -2138,10 +2140,8 @@ def mock_add_spec(self, spec, spec_set=False): class AsyncMagicMixin(MagicMixin): - def __init__(self, /, *args, **kw): - self._mock_set_magics() # make magic work for kwargs in init - _safe_super(AsyncMagicMixin, self).__init__(*args, **kw) - self._mock_set_magics() # fix magic broken by upper level init + pass + class MagicMock(MagicMixin, Mock): """ @@ -2183,6 +2183,10 @@ def __get__(self, obj, _type=None): return self.create_mock() +_CODE_ATTRS = dir(CodeType) +_CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + + class AsyncMockMixin(Base): await_count = _delegating_property('await_count') await_args = _delegating_property('await_args') @@ -2200,7 +2204,9 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=CodeType) + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG code_mock.co_flags = inspect.CO_COROUTINE self.__dict__['__code__'] = code_mock self.__dict__['__name__'] = 'AsyncMock' diff --git a/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst new file mode 100644 index 000000000000..a4984e695b43 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst @@ -0,0 +1 @@ +Instantiation of ``Mock()`` and ``AsyncMock()`` is now 3.8x faster. From webhook-mailer at python.org Fri Dec 23 15:17:30 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Fri, 23 Dec 2022 20:17:30 -0000 Subject: [Python-checkins] gh-99482: remove `jython` compatibility parts from stdlib and tests (#99484) Message-ID: <mailman.3040.1671826652.3313.python-checkins@python.org> https://github.com/python/cpython/commit/745545b5bb847023f90505bf9caa983463413780 commit: 745545b5bb847023f90505bf9caa983463413780 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T14:17:24-06:00 summary: gh-99482: remove `jython` compatibility parts from stdlib and tests (#99484) files: A Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst M Doc/whatsnew/3.12.rst M Lib/copy.py M Lib/pickle.py M Lib/platform.py M Lib/site.py M Lib/test/support/__init__.py M Lib/test/support/os_helper.py M Lib/test/test___all__.py M Lib/test/test_codeop.py M Lib/test/test_exceptions.py M Lib/test/test_import/__init__.py M Lib/test/test_platform.py M Lib/test/test_strftime.py M Lib/test/test_unicode.py M Lib/types.py M Lib/unittest/loader.py M Lib/xml/sax/__init__.py M Lib/xml/sax/_exceptions.py M Lib/xml/sax/expatreader.py diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 617fadd83f8f..69f97debd694 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -955,6 +955,9 @@ Removed ``SSTATE_INTERNED_IMMORTAL`` macro. (Contributed by Victor Stinner in :gh:`85858`.) +* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. + (Contributed by Nikita Sobolev in :gh:`99482`.) + * Remove ``_use_broken_old_ctypes_structure_semantics_`` flag from :mod:`ctypes` module. (Contributed by Nikita Sobolev in :gh:`99285`.) diff --git a/Lib/copy.py b/Lib/copy.py index 1b276afe0812..6e8c19bc652d 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -56,11 +56,6 @@ class Error(Exception): pass error = Error # backward compatibility -try: - from org.python.core import PyStringMap -except ImportError: - PyStringMap = None - __all__ = ["Error", "copy", "deepcopy"] def copy(x): @@ -120,9 +115,6 @@ def _copy_immutable(x): d[set] = set.copy d[bytearray] = bytearray.copy -if PyStringMap is not None: - d[PyStringMap] = PyStringMap.copy - del d, t def deepcopy(x, memo=None, _nil=[]): @@ -231,8 +223,6 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy): y[deepcopy(key, memo)] = deepcopy(value, memo) return y d[dict] = _deepcopy_dict -if PyStringMap is not None: - d[PyStringMap] = _deepcopy_dict def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) @@ -301,4 +291,4 @@ def _reconstruct(x, memo, func, args, y[key] = value return y -del types, weakref, PyStringMap +del types, weakref diff --git a/Lib/pickle.py b/Lib/pickle.py index f027e0432045..15fa5f6e5799 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -98,12 +98,6 @@ class _Stop(Exception): def __init__(self, value): self.value = value -# Jython has PyStringMap; it's a dict subclass with string keys -try: - from org.python.core import PyStringMap -except ImportError: - PyStringMap = None - # Pickle opcodes. See pickletools.py for extensive docs. The listing # here is in kind-of alphabetical order of 1-character pickle code. # pickletools groups them by purpose. @@ -972,8 +966,6 @@ def save_dict(self, obj): self._batch_setitems(obj.items()) dispatch[dict] = save_dict - if PyStringMap is not None: - dispatch[PyStringMap] = save_dict def _batch_setitems(self, items): # Helper to batch up SETITEMS sequences; proto >= 1 only diff --git a/Lib/platform.py b/Lib/platform.py index 6745321e31c2..b018046f5268 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1290,7 +1290,7 @@ def platform(aliased=0, terse=0): else: platform = _platform(system, release, version, csd) - elif system in ('Linux',): + elif system == 'Linux': # check for libc vs. glibc libcname, libcversion = libc_ver() platform = _platform(system, release, machine, processor, diff --git a/Lib/site.py b/Lib/site.py index 69670d9d7f22..7faf1c6f6af2 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -404,12 +404,7 @@ def setquit(): def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) - if sys.platform[:4] == 'java': - builtins.credits = _sitebuiltins._Printer( - "credits", - "Jython is maintained by the Jython developers (www.jython.org).") - else: - builtins.credits = _sitebuiltins._Printer("credits", """\ + builtins.credits = _sitebuiltins._Printer("credits", """\ Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands for supporting Python development. See www.python.org for more information.""") files, dirs = [], [] diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b7186057990a..ff736f1c2db8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -505,6 +505,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): requires_legacy_unicode_capi = unittest.skipUnless(unicode_legacy_string, 'requires legacy Unicode C API') +# Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') is_android = hasattr(sys, 'getandroidapilevel') @@ -736,8 +737,6 @@ def gc_collect(): """ import gc gc.collect() - if is_jython: - time.sleep(0.1) gc.collect() gc.collect() diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index f37a442aa0e6..2d4356a1191b 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -11,11 +11,7 @@ # Filename used for testing -if os.name == 'java': - # Jython disallows @ in module names - TESTFN_ASCII = '$test' -else: - TESTFN_ASCII = '@test' +TESTFN_ASCII = '@test' # Disambiguate TESTFN for parallel testing, while letting it remain a valid # module name. diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 1ec83cb0b144..ecf73b3ad1be 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -100,10 +100,9 @@ def test_all(self): '__future__', ]) - if not sys.platform.startswith('java'): - # In case _socket fails to build, make this test fail more gracefully - # than an AttributeError somewhere deep in CGIHTTPServer. - import _socket + # In case _socket fails to build, make this test fail more gracefully + # than an AttributeError somewhere deep in CGIHTTPServer. + import _socket ignored = [] failed_imports = [] diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index d7b51be642e4..6966c2ffd811 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -2,47 +2,18 @@ Test cases for codeop.py Nick Mathewson """ -import sys import unittest import warnings -from test import support from test.support import warnings_helper from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT -import io - -if support.is_jython: - - def unify_callables(d): - for n,v in d.items(): - if hasattr(v, '__call__'): - d[n] = True - return d class CodeopTests(unittest.TestCase): def assertValid(self, str, symbol='single'): '''succeed iff str is a valid piece of code''' - if support.is_jython: - code = compile_command(str, "<input>", symbol) - self.assertTrue(code) - if symbol == "single": - d,r = {},{} - saved_stdout = sys.stdout - sys.stdout = io.StringIO() - try: - exec(code, d) - exec(compile(str,"<input>","single"), r) - finally: - sys.stdout = saved_stdout - elif symbol == 'eval': - ctx = {'a': 2} - d = { 'value': eval(code,ctx) } - r = { 'value': eval(str,ctx) } - self.assertEqual(unify_callables(r),unify_callables(d)) - else: - expected = compile(str, "<input>", symbol, PyCF_DONT_IMPLY_DEDENT) - self.assertEqual(compile_command(str, "<input>", symbol), expected) + expected = compile(str, "<input>", symbol, PyCF_DONT_IMPLY_DEDENT) + self.assertEqual(compile_command(str, "<input>", symbol), expected) def assertIncomplete(self, str, symbol='single'): '''succeed iff str is the start of a valid piece of code''' @@ -62,16 +33,12 @@ def test_valid(self): av = self.assertValid # special case - if not support.is_jython: - self.assertEqual(compile_command(""), - compile("pass", "<input>", 'single', - PyCF_DONT_IMPLY_DEDENT)) - self.assertEqual(compile_command("\n"), - compile("pass", "<input>", 'single', - PyCF_DONT_IMPLY_DEDENT)) - else: - av("") - av("\n") + self.assertEqual(compile_command(""), + compile("pass", "<input>", 'single', + PyCF_DONT_IMPLY_DEDENT)) + self.assertEqual(compile_command("\n"), + compile("pass", "<input>", 'single', + PyCF_DONT_IMPLY_DEDENT)) av("a = 1") av("\na = 1") diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 65a3a8a48a88..f629321458d8 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -360,10 +360,9 @@ def test_capi3(): self.assertRaises(SystemError, _testcapi.raise_exception, InvalidException, 1) - if not sys.platform.startswith('java'): - test_capi1() - test_capi2() - test_capi3() + test_capi1() + test_capi2() + test_capi3() def test_WindowsError(self): try: diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 6c5b80bcee6c..1e4429ed7efe 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -20,7 +20,7 @@ from test.support import os_helper from test.support import ( - STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten, + STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, is_wasi) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) @@ -163,10 +163,7 @@ def test_import(self): def test_with_extension(ext): # The extension is normally ".py", perhaps ".pyw". source = TESTFN + ext - if is_jython: - pyc = TESTFN + "$py.class" - else: - pyc = TESTFN + ".pyc" + pyc = TESTFN + ".pyc" with open(source, "w", encoding='utf-8') as f: print("# This tests Python's ability to import a", diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 3992faf8e5cd..72942dda3424 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -329,7 +329,7 @@ def raises_oserror(*a): def test_java_ver(self): res = platform.java_ver() - if sys.platform == 'java': + if sys.platform == 'java': # Is never actually checked in CI self.assertTrue(all(res)) def test_win32_ver(self): diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index be43c49e40aa..cebfc8927862 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -54,14 +54,10 @@ def _update_variables(self, now): self.now = now def setUp(self): - try: - import java - java.util.Locale.setDefault(java.util.Locale.US) - except ImportError: - from locale import setlocale, LC_TIME - saved_locale = setlocale(LC_TIME) - setlocale(LC_TIME, 'C') - self.addCleanup(setlocale, LC_TIME, saved_locale) + from locale import setlocale, LC_TIME + saved_locale = setlocale(LC_TIME) + setlocale(LC_TIME, 'C') + self.addCleanup(setlocale, LC_TIME, saved_locale) def test_strftime(self): now = time.time() diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 5465d35534f1..79360f5549e4 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -94,88 +94,86 @@ def test_literals(self): self.assertNotEqual(r"\u0020", " ") def test_ascii(self): - if not sys.platform.startswith('java'): - # Test basic sanity of repr() - self.assertEqual(ascii('abc'), "'abc'") - self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") - self.assertEqual(ascii('ab\\'), "'ab\\\\'") - self.assertEqual(ascii('\\c'), "'\\\\c'") - self.assertEqual(ascii('\\'), "'\\\\'") - self.assertEqual(ascii('\n'), "'\\n'") - self.assertEqual(ascii('\r'), "'\\r'") - self.assertEqual(ascii('\t'), "'\\t'") - self.assertEqual(ascii('\b'), "'\\x08'") - self.assertEqual(ascii("'\""), """'\\'"'""") - self.assertEqual(ascii("'\""), """'\\'"'""") - self.assertEqual(ascii("'"), '''"'"''') - self.assertEqual(ascii('"'), """'"'""") - latin1repr = ( - "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" - "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" - "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" - "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" - "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" - "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" - "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9" - "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7" - "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5" - "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3" - "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1" - "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef" - "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd" - "\\xfe\\xff'") - testrepr = ascii(''.join(map(chr, range(256)))) - self.assertEqual(testrepr, latin1repr) - # Test ascii works on wide unicode escapes without overflow. - self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), - ascii("\U00010000" * 39 + "\uffff" * 4096)) - - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, ascii, WrongRepr()) + # Test basic sanity of repr() + self.assertEqual(ascii('abc'), "'abc'") + self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") + self.assertEqual(ascii('ab\\'), "'ab\\\\'") + self.assertEqual(ascii('\\c'), "'\\\\c'") + self.assertEqual(ascii('\\'), "'\\\\'") + self.assertEqual(ascii('\n'), "'\\n'") + self.assertEqual(ascii('\r'), "'\\r'") + self.assertEqual(ascii('\t'), "'\\t'") + self.assertEqual(ascii('\b'), "'\\x08'") + self.assertEqual(ascii("'\""), """'\\'"'""") + self.assertEqual(ascii("'\""), """'\\'"'""") + self.assertEqual(ascii("'"), '''"'"''') + self.assertEqual(ascii('"'), """'"'""") + latin1repr = ( + "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" + "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" + "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" + "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" + "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" + "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" + "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9" + "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7" + "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5" + "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3" + "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1" + "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef" + "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd" + "\\xfe\\xff'") + testrepr = ascii(''.join(map(chr, range(256)))) + self.assertEqual(testrepr, latin1repr) + # Test ascii works on wide unicode escapes without overflow. + self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), + ascii("\U00010000" * 39 + "\uffff" * 4096)) + + class WrongRepr: + def __repr__(self): + return b'byte-repr' + self.assertRaises(TypeError, ascii, WrongRepr()) def test_repr(self): - if not sys.platform.startswith('java'): - # Test basic sanity of repr() - self.assertEqual(repr('abc'), "'abc'") - self.assertEqual(repr('ab\\c'), "'ab\\\\c'") - self.assertEqual(repr('ab\\'), "'ab\\\\'") - self.assertEqual(repr('\\c'), "'\\\\c'") - self.assertEqual(repr('\\'), "'\\\\'") - self.assertEqual(repr('\n'), "'\\n'") - self.assertEqual(repr('\r'), "'\\r'") - self.assertEqual(repr('\t'), "'\\t'") - self.assertEqual(repr('\b'), "'\\x08'") - self.assertEqual(repr("'\""), """'\\'"'""") - self.assertEqual(repr("'\""), """'\\'"'""") - self.assertEqual(repr("'"), '''"'"''') - self.assertEqual(repr('"'), """'"'""") - latin1repr = ( - "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" - "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" - "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" - "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" - "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" - "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" - "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" - "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" - "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5" - "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3" - "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1" - "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" - "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd" - "\xfe\xff'") - testrepr = repr(''.join(map(chr, range(256)))) - self.assertEqual(testrepr, latin1repr) - # Test repr works on wide unicode escapes without overflow. - self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), - repr("\U00010000" * 39 + "\uffff" * 4096)) - - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, repr, WrongRepr()) + # Test basic sanity of repr() + self.assertEqual(repr('abc'), "'abc'") + self.assertEqual(repr('ab\\c'), "'ab\\\\c'") + self.assertEqual(repr('ab\\'), "'ab\\\\'") + self.assertEqual(repr('\\c'), "'\\\\c'") + self.assertEqual(repr('\\'), "'\\\\'") + self.assertEqual(repr('\n'), "'\\n'") + self.assertEqual(repr('\r'), "'\\r'") + self.assertEqual(repr('\t'), "'\\t'") + self.assertEqual(repr('\b'), "'\\x08'") + self.assertEqual(repr("'\""), """'\\'"'""") + self.assertEqual(repr("'\""), """'\\'"'""") + self.assertEqual(repr("'"), '''"'"''') + self.assertEqual(repr('"'), """'"'""") + latin1repr = ( + "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" + "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" + "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" + "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" + "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" + "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" + "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" + "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5" + "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3" + "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1" + "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" + "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd" + "\xfe\xff'") + testrepr = repr(''.join(map(chr, range(256)))) + self.assertEqual(testrepr, latin1repr) + # Test repr works on wide unicode escapes without overflow. + self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), + repr("\U00010000" * 39 + "\uffff" * 4096)) + + class WrongRepr: + def __repr__(self): + return b'byte-repr' + self.assertRaises(TypeError, repr, WrongRepr()) def test_iterators(self): # Make sure unicode objects have an __iter__ method @@ -684,8 +682,7 @@ def test_islower(self): def test_isupper(self): super().test_isupper() - if not sys.platform.startswith('java'): - self.checkequalnofix(False, '\u1FFc', 'isupper') + self.checkequalnofix(False, '\u1FFc', 'isupper') self.assertTrue('\u2167'.isupper()) self.assertFalse('\u2177'.isupper()) # non-BMP, uppercase @@ -1473,10 +1470,9 @@ def test_formatting(self): self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.5), 'abc, abc, -1, -2.000000, 3.50') self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.57), 'abc, abc, -1, -2.000000, 3.57') self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 1003.57), 'abc, abc, -1, -2.000000, 1003.57') - if not sys.platform.startswith('java'): - self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'") - self.assertEqual("%r" % ("\u1234",), "'\u1234'") - self.assertEqual("%a" % ("\u1234",), "'\\u1234'") + self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'") + self.assertEqual("%r" % ("\u1234",), "'\u1234'") + self.assertEqual("%a" % ("\u1234",), "'\\u1234'") self.assertEqual("%(x)s, %(y)s" % {'x':"abc", 'y':"def"}, 'abc, def') self.assertEqual("%(x)s, %(\xfc)s" % {'x':"abc", '\xfc':"def"}, 'abc, def') @@ -1671,29 +1667,27 @@ def __str__(self): # unicode(obj, encoding, error) tests (this maps to # PyUnicode_FromEncodedObject() at C level) - if not sys.platform.startswith('java'): - self.assertRaises( - TypeError, - str, - 'decoding unicode is not supported', - 'utf-8', - 'strict' - ) + self.assertRaises( + TypeError, + str, + 'decoding unicode is not supported', + 'utf-8', + 'strict' + ) self.assertEqual( str(b'strings are decoded to unicode', 'utf-8', 'strict'), 'strings are decoded to unicode' ) - if not sys.platform.startswith('java'): - self.assertEqual( - str( - memoryview(b'character buffers are decoded to unicode'), - 'utf-8', - 'strict' - ), - 'character buffers are decoded to unicode' - ) + self.assertEqual( + str( + memoryview(b'character buffers are decoded to unicode'), + 'utf-8', + 'strict' + ), + 'character buffers are decoded to unicode' + ) self.assertRaises(TypeError, str, 42, 42, 42) diff --git a/Lib/types.py b/Lib/types.py index f8353126cb52..aa8a1c847223 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -56,7 +56,6 @@ def _m(self): pass TracebackType = type(exc.__traceback__) FrameType = type(exc.__traceback__.tb_frame) -# For Jython, the following two types are identical GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index eb18cd0b49cd..80d4fbdd8e36 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -57,9 +57,7 @@ def testSkipped(self): TestClass = type("ModuleSkipped", (case.TestCase,), attrs) return suiteClass((TestClass(methodname),)) -def _jython_aware_splitext(path): - if path.lower().endswith('$py.class'): - return path[:-9] +def _splitext(path): return os.path.splitext(path)[0] @@ -315,7 +313,7 @@ def _get_directory_containing_module(self, module_name): def _get_name_from_path(self, path): if path == self._top_level_dir: return '.' - path = _jython_aware_splitext(os.path.normpath(path)) + path = _splitext(os.path.normpath(path)) _relpath = os.path.relpath(path, self._top_level_dir) assert not os.path.isabs(_relpath), "Path must be within the project" @@ -393,13 +391,13 @@ def _find_test_path(self, full_path, pattern): else: mod_file = os.path.abspath( getattr(module, '__file__', full_path)) - realpath = _jython_aware_splitext( + realpath = _splitext( os.path.realpath(mod_file)) - fullpath_noext = _jython_aware_splitext( + fullpath_noext = _splitext( os.path.realpath(full_path)) if realpath.lower() != fullpath_noext.lower(): module_dir = os.path.dirname(realpath) - mod_name = _jython_aware_splitext( + mod_name = _splitext( os.path.basename(full_path)) expected_dir = os.path.dirname(full_path) msg = ("%r module incorrectly imported from %r. Expected " diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index 17b75879ebaa..b657310207cf 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -60,11 +60,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: default_parser_list = os.environ["PY_SAX_PARSER"].split(",") -del os - -_key = "python.xml.sax.parser" -if sys.platform[:4] == "java" and sys.registry.containsKey(_key): - default_parser_list = sys.registry.getProperty(_key).split(",") +del os, sys def make_parser(parser_list=()): @@ -93,15 +89,6 @@ def make_parser(parser_list=()): # --- Internal utility methods used by make_parser -if sys.platform[ : 4] == "java": - def _create_parser(parser_name): - from org.python.core import imp - drv_module = imp.importName(parser_name, 0, globals()) - return drv_module.create_parser() - -else: - def _create_parser(parser_name): - drv_module = __import__(parser_name,{},{},['create_parser']) - return drv_module.create_parser() - -del sys +def _create_parser(parser_name): + drv_module = __import__(parser_name,{},{},['create_parser']) + return drv_module.create_parser() diff --git a/Lib/xml/sax/_exceptions.py b/Lib/xml/sax/_exceptions.py index a9b2ba35c6a2..f292dc3a8e50 100644 --- a/Lib/xml/sax/_exceptions.py +++ b/Lib/xml/sax/_exceptions.py @@ -1,8 +1,4 @@ """Different kinds of SAX Exceptions""" -import sys -if sys.platform[:4] == "java": - from java.lang import Exception -del sys # ===== SAXEXCEPTION ===== diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index e334ac9fea0d..b9ad52692db8 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -12,12 +12,6 @@ from xml.sax.handler import feature_string_interning from xml.sax.handler import property_xml_string, property_interning_dict -# xml.parsers.expat does not raise ImportError in Jython -import sys -if sys.platform[:4] == "java": - raise SAXReaderNotAvailable("expat not available in Java", None) -del sys - try: from xml.parsers import expat except ImportError: diff --git a/Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst b/Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst new file mode 100644 index 000000000000..dd2c925478c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-14-19-58-36.gh-issue-99482.XmZyUr.rst @@ -0,0 +1 @@ +Remove ``Jython`` partial compatibility code from several stdlib modules. From webhook-mailer at python.org Fri Dec 23 15:19:21 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Fri, 23 Dec 2022 20:19:21 -0000 Subject: [Python-checkins] bpo-40447: accept all path-like objects in compileall.compile_file (#19883) Message-ID: <mailman.3041.1671826763.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e commit: 1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e branch: main author: Filipe La?ns <lains at riseup.net> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T14:19:16-06:00 summary: bpo-40447: accept all path-like objects in compileall.compile_file (#19883) Signed-off-by: Filipe La?ns <lains at archlinux.org> Signed-off-by: Filipe La?ns <lains at riseup.net> Co-authored-by: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst M Lib/compileall.py M Lib/test/test_compileall.py diff --git a/Lib/compileall.py b/Lib/compileall.py index 330a90786efc..a388931fb5a9 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -154,8 +154,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 73c83c9bf1ef..05154c8f1c60 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -167,6 +167,20 @@ def test_compile_file_pathlike_ddir(self): quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_file_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_file_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_path(self): with test.test_importlib.util.import_state(path=[self.directory]): self.assertTrue(compileall.compile_path(quiet=2)) @@ -219,6 +233,20 @@ def test_compile_dir_pathlike(self): self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_dir_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_dir_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + @skipUnless(_have_multiprocessing, "requires multiprocessing") @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): diff --git a/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst new file mode 100644 index 000000000000..941038d095b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst @@ -0,0 +1,2 @@ +Accept :class:`os.PathLike` (such as :class:`pathlib.Path`) in the ``stripdir`` arguments of +:meth:`compileall.compile_file` and :meth:`compileall.compile_dir`. From webhook-mailer at python.org Fri Dec 23 15:40:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 20:40:00 -0000 Subject: [Python-checkins] bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) Message-ID: <mailman.3042.1671828001.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9c4b8cca09342734eb04fc1093d5952a2317dfb7 commit: 9c4b8cca09342734eb04fc1093d5952a2317dfb7 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T12:39:55-08:00 summary: bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) (cherry picked from commit 1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e) Co-authored-by: Filipe La?ns <lains at riseup.net> Signed-off-by: Filipe La?ns <lains at archlinux.org> Signed-off-by: Filipe La?ns <lains at riseup.net> Co-authored-by: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst M Lib/compileall.py M Lib/test/test_compileall.py diff --git a/Lib/compileall.py b/Lib/compileall.py index 3755e76ba813..50183ea85468 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -154,8 +154,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index a7318bf404c3..554e2b27eea2 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -167,6 +167,20 @@ def test_compile_file_pathlike_ddir(self): quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_file_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_file_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_path(self): with test.test_importlib.util.import_state(path=[self.directory]): self.assertTrue(compileall.compile_path(quiet=2)) @@ -219,6 +233,20 @@ def test_compile_dir_pathlike(self): self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_dir_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_dir_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + @skipUnless(_have_multiprocessing, "requires multiprocessing") @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): diff --git a/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst new file mode 100644 index 000000000000..941038d095b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst @@ -0,0 +1,2 @@ +Accept :class:`os.PathLike` (such as :class:`pathlib.Path`) in the ``stripdir`` arguments of +:meth:`compileall.compile_file` and :meth:`compileall.compile_dir`. From webhook-mailer at python.org Fri Dec 23 15:43:19 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 23 Dec 2022 20:43:19 -0000 Subject: [Python-checkins] bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) Message-ID: <mailman.3043.1671828199.3313.python-checkins@python.org> https://github.com/python/cpython/commit/d5eb2f4747cd9afbc7f32fd6373a5f73aa59f61c commit: d5eb2f4747cd9afbc7f32fd6373a5f73aa59f61c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T12:43:13-08:00 summary: bpo-40447: accept all path-like objects in compileall.compile_file (GH-19883) (cherry picked from commit 1ecfd1ebf1f53ef6ac82085b25ed09952b470d4e) Co-authored-by: Filipe La?ns <lains at riseup.net> Signed-off-by: Filipe La?ns <lains at archlinux.org> Signed-off-by: Filipe La?ns <lains at riseup.net> Co-authored-by: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst M Lib/compileall.py M Lib/test/test_compileall.py diff --git a/Lib/compileall.py b/Lib/compileall.py index 330a90786efc..a388931fb5a9 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -154,8 +154,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 73c83c9bf1ef..05154c8f1c60 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -167,6 +167,20 @@ def test_compile_file_pathlike_ddir(self): quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_file_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_file_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_path(self): with test.test_importlib.util.import_state(path=[self.directory]): self.assertTrue(compileall.compile_path(quiet=2)) @@ -219,6 +233,20 @@ def test_compile_dir_pathlike(self): self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) + def test_compile_dir_pathlike_stripdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + stripdir=pathlib.Path('stripdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + + def test_compile_dir_pathlike_prependdir(self): + self.assertFalse(os.path.isfile(self.bc_path)) + self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), + prependdir=pathlib.Path('prependdir_path'), + quiet=2)) + self.assertTrue(os.path.isfile(self.bc_path)) + @skipUnless(_have_multiprocessing, "requires multiprocessing") @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): diff --git a/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst new file mode 100644 index 000000000000..941038d095b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-12-55-55.bpo-40447.oKR0Lj.rst @@ -0,0 +1,2 @@ +Accept :class:`os.PathLike` (such as :class:`pathlib.Path`) in the ``stripdir`` arguments of +:meth:`compileall.compile_file` and :meth:`compileall.compile_dir`. From webhook-mailer at python.org Fri Dec 23 17:36:04 2022 From: webhook-mailer at python.org (rhettinger) Date: Fri, 23 Dec 2022 22:36:04 -0000 Subject: [Python-checkins] GH-100425: Improve accuracy of builtin sum() for float inputs (GH-100426) Message-ID: <mailman.3044.1671834965.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5d84966cce6c596da22922a07f49bde959ff5201 commit: 5d84966cce6c596da22922a07f49bde959ff5201 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-23T14:35:58-08:00 summary: GH-100425: Improve accuracy of builtin sum() for float inputs (GH-100426) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst M Doc/library/functions.rst M Doc/library/math.rst M Doc/tutorial/floatingpoint.rst M Lib/test/test_builtin.py M Python/bltinmodule.c diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2110990d1889..817c1f858aae 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1733,6 +1733,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.8 The *start* parameter can be specified as a keyword argument. + .. versionchanged:: 3.12 Summation of floats switched to an algorithm + that gives higher accuracy on most builds. + + .. class:: super() super(type, object_or_type=None) diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 559c6ec5dd9d..aeebcaf6ab08 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -108,12 +108,7 @@ Number-theoretic and representation functions .. function:: fsum(iterable) Return an accurate floating point sum of values in the iterable. Avoids - loss of precision by tracking multiple intermediate partial sums: - - >>> sum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 0.9999999999999999 - >>> fsum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 1.0 + loss of precision by tracking multiple intermediate partial sums. The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the typical case where the rounding mode is half-even. On some non-Windows diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index e1cd7f9ece75..cedade6e3366 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -192,7 +192,7 @@ added onto a running total. That can make a difference in overall accuracy so that the errors do not accumulate to the point where they affect the final total: - >>> sum([0.1] * 10) == 1.0 + >>> 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 == 1.0 False >>> math.fsum([0.1] * 10) == 1.0 True diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index eb1c389257cc..c65600483258 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -9,6 +9,7 @@ import gc import io import locale +import math import os import pickle import platform @@ -31,6 +32,7 @@ from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok from test.support.warnings_helper import check_warnings +from test.support import requires_IEEE_754 from unittest.mock import MagicMock, patch try: import pty, signal @@ -38,6 +40,12 @@ pty = signal = None +# Detect evidence of double-rounding: sum() does not always +# get improved accuracy on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + + class Squares: def __init__(self, max): @@ -1617,6 +1625,8 @@ def test_sum(self): self.assertEqual(repr(sum([-0.0])), '0.0') self.assertEqual(repr(sum([-0.0], -0.0)), '-0.0') self.assertEqual(repr(sum([], -0.0)), '-0.0') + self.assertTrue(math.isinf(sum([float("inf"), float("inf")]))) + self.assertTrue(math.isinf(sum([1e308, 1e308]))) self.assertRaises(TypeError, sum) self.assertRaises(TypeError, sum, 42) @@ -1641,6 +1651,14 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sum accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sum_accuracy(self): + self.assertEqual(sum([0.1] * 10), 1.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + def test_type(self): self.assertEqual(type(''), type('123')) self.assertNotEqual(type(''), type(())) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst new file mode 100644 index 000000000000..5559020b11d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-21-22-48-41.gh-issue-100425.U64yLu.rst @@ -0,0 +1 @@ +Improve the accuracy of ``sum()`` with compensated summation. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ff96c25da5eb..2d4822e6d468 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2532,6 +2532,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) if (PyFloat_CheckExact(result)) { double f_result = PyFloat_AS_DOUBLE(result); + double c = 0.0; Py_SETREF(result, NULL); while(result == NULL) { item = PyIter_Next(iter); @@ -2539,10 +2540,25 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) Py_DECREF(iter); if (PyErr_Occurred()) return NULL; + /* Avoid losing the sign on a negative result, + and don't let adding the compensation convert + an infinite or overflowed sum to a NaN. */ + if (c && Py_IS_FINITE(c)) { + f_result += c; + } return PyFloat_FromDouble(f_result); } if (PyFloat_CheckExact(item)) { - f_result += PyFloat_AS_DOUBLE(item); + // Improved Kahan?Babu?ka algorithm by Arnold Neumaier + // https://www.mat.univie.ac.at/~neum/scan/01.pdf + double x = PyFloat_AS_DOUBLE(item); + double t = f_result + x; + if (fabs(f_result) >= fabs(x)) { + c += (f_result - t) + x; + } else { + c += (x - t) + f_result; + } + f_result = t; _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); continue; } @@ -2556,6 +2572,9 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) continue; } } + if (c && Py_IS_FINITE(c)) { + f_result += c; + } result = PyFloat_FromDouble(f_result); if (result == NULL) { Py_DECREF(item); From webhook-mailer at python.org Fri Dec 23 17:52:37 2022 From: webhook-mailer at python.org (brettcannon) Date: Fri, 23 Dec 2022 22:52:37 -0000 Subject: [Python-checkins] gh-68320, gh-88302 - Allow for private `pathlib.Path` subclassing (GH-31691) Message-ID: <mailman.3045.1671835958.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a68e585c8b7b27323f67905868467ce0588a1dae commit: a68e585c8b7b27323f67905868467ce0588a1dae branch: main author: Barney Gale <barney.gale at gmail.com> committer: brettcannon <brett at python.org> date: 2022-12-23T14:52:23-08:00 summary: gh-68320, gh-88302 - Allow for private `pathlib.Path` subclassing (GH-31691) Users may wish to define subclasses of `pathlib.Path` to add or modify existing methods. Before this change, attempting to instantiate a subclass raised an exception like: AttributeError: type object 'PPath' has no attribute '_flavour' Previously the `_flavour` attribute was assigned as follows: PurePath._flavour = xxx not set!! xxx PurePosixPath._flavour = _PosixFlavour() PureWindowsPath._flavour = _WindowsFlavour() This change replaces it with a `_pathmod` attribute, set as follows: PurePath._pathmod = os.path PurePosixPath._pathmod = posixpath PureWindowsPath._pathmod = ntpath Functionality from `_PosixFlavour` and `_WindowsFlavour` is moved into `PurePath` as underscored-prefixed classmethods. Flavours are removed. Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> Co-authored-by: Brett Cannon <brett at python.org> Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Eryk Sun <eryksun at gmail.com> files: A Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst M Lib/pathlib.py M Lib/test/test_pathlib.py diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 7890fdade420..b959e85d1840 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -30,6 +30,14 @@ # Internals # +# Reference for Windows paths can be found at +# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . +_WIN_RESERVED_NAMES = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + _WINERROR_NOT_READY = 21 # drive exists but is not accessible _WINERROR_INVALID_NAME = 123 # fix for bpo-35306 _WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself @@ -52,150 +60,6 @@ def _is_wildcard_pattern(pat): # be looked up directly as a file. return "*" in pat or "?" in pat or "[" in pat - -class _Flavour(object): - """A flavour implements a particular (platform-specific) set of path - semantics.""" - - def __init__(self): - self.join = self.sep.join - - def parse_parts(self, parts): - if not parts: - return '', '', [] - sep = self.sep - altsep = self.altsep - path = self.pathmod.join(*parts) - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = self.splitroot(path) - unfiltered_parsed = [drv + root] + rel.split(sep) - parsed = [sys.intern(x) for x in unfiltered_parsed if x and x != '.'] - return drv, root, parsed - - def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2): - """ - Join the two paths represented by the respective - (drive, root, parts) tuples. Return a new (drive, root, parts) tuple. - """ - if root2: - if not drv2 and drv: - return drv, root2, [drv + root2] + parts2[1:] - elif drv2: - if drv2 == drv or self.casefold(drv2) == self.casefold(drv): - # Same drive => second path is relative to the first - return drv, root, parts + parts2[1:] - else: - # Second path is non-anchored (common case) - return drv, root, parts + parts2 - return drv2, root2, parts2 - - -class _WindowsFlavour(_Flavour): - # Reference for Windows paths can be found at - # http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx - - sep = '\\' - altsep = '/' - has_drv = True - pathmod = ntpath - - is_supported = (os.name == 'nt') - - reserved_names = ( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {'COM%s' % c for c in '123456789\xb9\xb2\xb3'} | - {'LPT%s' % c for c in '123456789\xb9\xb2\xb3'} - ) - - def splitroot(self, part, sep=sep): - drv, rest = self.pathmod.splitdrive(part) - if drv[:1] == sep or rest[:1] == sep: - return drv, sep, rest.lstrip(sep) - else: - return drv, '', rest - - def casefold(self, s): - return s.lower() - - def casefold_parts(self, parts): - return [p.lower() for p in parts] - - def compile_pattern(self, pattern): - return re.compile(fnmatch.translate(pattern), re.IGNORECASE).fullmatch - - def is_reserved(self, parts): - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if not parts: - return False - if parts[0].startswith('\\\\'): - # UNC paths are never reserved - return False - name = parts[-1].partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in self.reserved_names - - def make_uri(self, path): - # Under Windows, file URIs use the UTF-8 encoding. - drive = path.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - rest = path.as_posix()[2:].lstrip('/') - return 'file:///%s/%s' % ( - drive, urlquote_from_bytes(rest.encode('utf-8'))) - else: - # It's a path on a network drive => 'file://host/share/a/b' - return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8')) - - -class _PosixFlavour(_Flavour): - sep = '/' - altsep = '' - has_drv = False - pathmod = posixpath - - is_supported = (os.name != 'nt') - - def splitroot(self, part, sep=sep): - if part and part[0] == sep: - stripped_part = part.lstrip(sep) - # According to POSIX path resolution: - # http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap04.html#tag_04_11 - # "A pathname that begins with two successive slashes may be - # interpreted in an implementation-defined manner, although more - # than two leading slashes shall be treated as a single slash". - if len(part) - len(stripped_part) == 2: - return '', sep * 2, stripped_part - else: - return '', sep, stripped_part - else: - return '', '', part - - def casefold(self, s): - return s - - def casefold_parts(self, parts): - return parts - - def compile_pattern(self, pattern): - return re.compile(fnmatch.translate(pattern)).fullmatch - - def is_reserved(self, parts): - return False - - def make_uri(self, path): - # We represent the path using the local filesystem encoding, - # for portability to other applications. - bpath = bytes(path) - return 'file://' + urlquote_from_bytes(bpath) - - -_windows_flavour = _WindowsFlavour() -_posix_flavour = _PosixFlavour() - - # # Globbing helpers # @@ -237,14 +101,15 @@ def select_from(self, parent_path): is_dir = path_cls.is_dir exists = path_cls.exists scandir = path_cls._scandir + normcase = path_cls._flavour.normcase if not is_dir(parent_path): return iter([]) - return self._select_from(parent_path, is_dir, exists, scandir) + return self._select_from(parent_path, is_dir, exists, scandir, normcase) class _TerminatingSelector: - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): yield parent_path @@ -254,11 +119,11 @@ def __init__(self, name, child_parts, flavour): self.name = name _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): try: path = parent_path._make_child_relpath(self.name) if (is_dir if self.dironly else exists)(path): - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): yield p except PermissionError: return @@ -267,10 +132,10 @@ def _select_from(self, parent_path, is_dir, exists, scandir): class _WildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): - self.match = flavour.compile_pattern(pat) + self.match = re.compile(fnmatch.translate(flavour.normcase(pat))).fullmatch _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): try: # We must close the scandir() object before proceeding to # avoid exhausting file descriptors when globbing deep trees. @@ -289,9 +154,9 @@ def _select_from(self, parent_path, is_dir, exists, scandir): raise continue name = entry.name - if self.match(name): + if self.match(normcase(name)): path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): yield p except PermissionError: return @@ -323,13 +188,13 @@ def _iterate_directories(self, parent_path, is_dir, scandir): except PermissionError: return - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, is_dir, exists, scandir, normcase): try: yielded = set() try: successor_select = self.successor._select_from for starting_point in self._iterate_directories(parent_path, is_dir, scandir): - for p in successor_select(starting_point, is_dir, exists, scandir): + for p in successor_select(starting_point, is_dir, exists, scandir, normcase): if p not in yielded: yield p yielded.add(p) @@ -387,8 +252,9 @@ class PurePath(object): """ __slots__ = ( '_drv', '_root', '_parts', - '_str', '_hash', '_pparts', '_cached_cparts', + '_str', '_hash', '_parts_tuple', '_parts_normcase_cached', ) + _flavour = os.path def __new__(cls, *args): """Construct a PurePath from one or several strings and or existing @@ -405,6 +271,33 @@ def __reduce__(self): # when pickling related paths. return (self.__class__, tuple(self._parts)) + @classmethod + def _split_root(cls, part): + sep = cls._flavour.sep + rel = cls._flavour.splitdrive(part)[1].lstrip(sep) + anchor = part.removesuffix(rel) + if anchor: + anchor = cls._flavour.normpath(anchor) + drv, root = cls._flavour.splitdrive(anchor) + if drv.startswith(sep): + # UNC paths always have a root. + root = sep + return drv, root, rel + + @classmethod + def _parse_parts(cls, parts): + if not parts: + return '', '', [] + sep = cls._flavour.sep + altsep = cls._flavour.altsep + path = cls._flavour.join(*parts) + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls._split_root(path) + unfiltered_parsed = [drv + root] + rel.split(sep) + parsed = [sys.intern(x) for x in unfiltered_parsed if x and x != '.'] + return drv, root, parsed + @classmethod def _parse_args(cls, args): # This is useful when you don't want to create an instance, just @@ -423,7 +316,7 @@ def _parse_args(cls, args): "argument should be a str object or an os.PathLike " "object returning str, not %r" % type(a)) - return cls._flavour.parse_parts(parts) + return cls._parse_parts(parts) @classmethod def _from_parts(cls, args): @@ -447,15 +340,9 @@ def _from_parsed_parts(cls, drv, root, parts): @classmethod def _format_parsed_parts(cls, drv, root, parts): if drv or root: - return drv + root + cls._flavour.join(parts[1:]) + return drv + root + cls._flavour.sep.join(parts[1:]) else: - return cls._flavour.join(parts) - - def _make_child(self, args): - drv, root, parts = self._parse_args(args) - drv, root, parts = self._flavour.join_parsed_parts( - self._drv, self._root, self._parts, drv, root, parts) - return self._from_parsed_parts(drv, root, parts) + return cls._flavour.sep.join(parts) def __str__(self): """Return the string representation of the path, suitable for @@ -488,48 +375,62 @@ def as_uri(self): """Return the path as a 'file' URI.""" if not self.is_absolute(): raise ValueError("relative path can't be expressed as a file URI") - return self._flavour.make_uri(self) + + drive = self._drv + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + return prefix + urlquote_from_bytes(os.fsencode(path)) @property - def _cparts(self): - # Cached casefolded parts, for hashing and comparison + def _parts_normcase(self): + # Cached parts with normalized case, for hashing and comparison. try: - return self._cached_cparts + return self._parts_normcase_cached except AttributeError: - self._cached_cparts = self._flavour.casefold_parts(self._parts) - return self._cached_cparts + self._parts_normcase_cached = [self._flavour.normcase(p) for p in self._parts] + return self._parts_normcase_cached def __eq__(self, other): if not isinstance(other, PurePath): return NotImplemented - return self._cparts == other._cparts and self._flavour is other._flavour + return self._parts_normcase == other._parts_normcase and self._flavour is other._flavour def __hash__(self): try: return self._hash except AttributeError: - self._hash = hash(tuple(self._cparts)) + self._hash = hash(tuple(self._parts_normcase)) return self._hash def __lt__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts < other._cparts + return self._parts_normcase < other._parts_normcase def __le__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts <= other._cparts + return self._parts_normcase <= other._parts_normcase def __gt__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts > other._cparts + return self._parts_normcase > other._parts_normcase def __ge__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented - return self._cparts >= other._cparts + return self._parts_normcase >= other._parts_normcase drive = property(attrgetter('_drv'), doc="""The drive prefix (letter or UNC path), if any.""") @@ -592,7 +493,7 @@ def with_name(self, name): """Return a new path with the file name changed.""" if not self.name: raise ValueError("%r has an empty name" % (self,)) - drv, root, parts = self._flavour.parse_parts((name,)) + drv, root, parts = self._parse_parts((name,)) if (not name or name[-1] in [self._flavour.sep, self._flavour.altsep] or drv or root or len(parts) != 1): raise ValueError("Invalid name %r" % (name)) @@ -669,10 +570,10 @@ def parts(self): # We cache the tuple to avoid building a new one each time .parts # is accessed. XXX is this necessary? try: - return self._pparts + return self._parts_tuple except AttributeError: - self._pparts = tuple(self._parts) - return self._pparts + self._parts_tuple = tuple(self._parts) + return self._parts_tuple def joinpath(self, *args): """Combine this path with one or several arguments, and return a @@ -680,11 +581,26 @@ def joinpath(self, *args): paths) or a totally different path (if one of the arguments is anchored). """ - return self._make_child(args) + drv1, root1, parts1 = self._drv, self._root, self._parts + drv2, root2, parts2 = self._parse_args(args) + if root2: + if not drv2 and drv1: + return self._from_parsed_parts(drv1, root2, [drv1 + root2] + parts2[1:]) + else: + return self._from_parsed_parts(drv2, root2, parts2) + elif drv2: + if drv2 == drv1 or self._flavour.normcase(drv2) == self._flavour.normcase(drv1): + # Same drive => second path is relative to the first. + return self._from_parsed_parts(drv1, root1, parts1 + parts2[1:]) + else: + return self._from_parsed_parts(drv2, root2, parts2) + else: + # Second path is non-anchored (common case). + return self._from_parsed_parts(drv1, root1, parts1 + parts2) def __truediv__(self, key): try: - return self._make_child((key,)) + return self.joinpath(key) except TypeError: return NotImplemented @@ -712,29 +628,40 @@ def parents(self): def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - if not self._root: - return False - return not self._flavour.has_drv or bool(self._drv) + # ntpath.isabs() is defective - see GH-44626 . + if self._flavour is ntpath: + return bool(self._drv and self._root) + return self._flavour.isabs(self) def is_reserved(self): """Return True if the path contains one of the special names reserved by the system, if any.""" - return self._flavour.is_reserved(self._parts) + if self._flavour is posixpath or not self._parts: + return False + + # NOTE: the rules for reserved names seem somewhat complicated + # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not + # exist). We err on the side of caution and return True for paths + # which are not considered reserved by Windows. + if self._parts[0].startswith('\\\\'): + # UNC paths are never reserved. + return False + name = self._parts[-1].partition('.')[0].partition(':')[0].rstrip(' ') + return name.upper() in _WIN_RESERVED_NAMES def match(self, path_pattern): """ Return True if this path matches the given pattern. """ - cf = self._flavour.casefold - path_pattern = cf(path_pattern) - drv, root, pat_parts = self._flavour.parse_parts((path_pattern,)) + path_pattern = self._flavour.normcase(path_pattern) + drv, root, pat_parts = self._parse_parts((path_pattern,)) if not pat_parts: raise ValueError("empty pattern") - if drv and drv != cf(self._drv): + elif drv and drv != self._flavour.normcase(self._drv): return False - if root and root != cf(self._root): + elif root and root != self._root: return False - parts = self._cparts + parts = self._parts_normcase if drv or root: if len(pat_parts) != len(parts): return False @@ -757,7 +684,7 @@ class PurePosixPath(PurePath): On a POSIX system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - _flavour = _posix_flavour + _flavour = posixpath __slots__ = () @@ -767,7 +694,7 @@ class PureWindowsPath(PurePath): On a Windows system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - _flavour = _windows_flavour + _flavour = ntpath __slots__ = () @@ -789,7 +716,7 @@ def __new__(cls, *args, **kwargs): if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath self = cls._from_parts(args) - if not self._flavour.is_supported: + if self._flavour is not os.path: raise NotImplementedError("cannot instantiate %r on your system" % (cls.__name__,)) return self @@ -842,7 +769,7 @@ def samefile(self, other_path): other_st = other_path.stat() except AttributeError: other_st = self.__class__(other_path).stat() - return os.path.samestat(st, other_st) + return self._flavour.samestat(st, other_st) def iterdir(self): """Yield path objects of the directory contents. @@ -866,7 +793,7 @@ def glob(self, pattern): sys.audit("pathlib.Path.glob", self, pattern) if not pattern: raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) + drv, root, pattern_parts = self._parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") if pattern[-1] in (self._flavour.sep, self._flavour.altsep): @@ -881,7 +808,7 @@ def rglob(self, pattern): this subtree. """ sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) + drv, root, pattern_parts = self._parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): @@ -912,7 +839,7 @@ def check_eloop(e): raise RuntimeError("Symlink loop from %r" % e.filename) try: - s = os.path.realpath(self, strict=strict) + s = self._flavour.realpath(self, strict=strict) except OSError as e: check_eloop(e) raise @@ -1184,7 +1111,7 @@ def is_mount(self): """ Check if this path is a mount point """ - return self._flavour.pathmod.ismount(self) + return self._flavour.ismount(self) def is_symlink(self): """ @@ -1205,7 +1132,7 @@ def is_junction(self): """ Whether this path is a junction. """ - return self._flavour.pathmod.isjunction(self) + return self._flavour.isjunction(self) def is_block_device(self): """ @@ -1277,7 +1204,7 @@ def expanduser(self): """ if (not (self._drv or self._root) and self._parts and self._parts[0][:1] == '~'): - homedir = os.path.expanduser(self._parts[0]) + homedir = self._flavour.expanduser(self._parts[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") return self._from_parts([homedir] + self._parts[1:]) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index fa6ea0ac63d8..7d4d782cf5f0 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -26,7 +26,7 @@ class _BaseFlavourTest(object): def _check_parse_parts(self, arg, expected): - f = self.flavour.parse_parts + f = self.cls._parse_parts sep = self.flavour.sep altsep = self.flavour.altsep actual = f([x.replace('/', sep) for x in arg]) @@ -65,7 +65,8 @@ def test_parse_parts_common(self): class PosixFlavourTest(_BaseFlavourTest, unittest.TestCase): - flavour = pathlib._posix_flavour + cls = pathlib.PurePosixPath + flavour = pathlib.PurePosixPath._flavour def test_parse_parts(self): check = self._check_parse_parts @@ -80,7 +81,7 @@ def test_parse_parts(self): check(['\\a'], ('', '', ['\\a'])) def test_splitroot(self): - f = self.flavour.splitroot + f = self.cls._split_root self.assertEqual(f(''), ('', '', '')) self.assertEqual(f('a'), ('', '', 'a')) self.assertEqual(f('a/b'), ('', '', 'a/b')) @@ -101,7 +102,8 @@ def test_splitroot(self): class NTFlavourTest(_BaseFlavourTest, unittest.TestCase): - flavour = pathlib._windows_flavour + cls = pathlib.PureWindowsPath + flavour = pathlib.PureWindowsPath._flavour def test_parse_parts(self): check = self._check_parse_parts @@ -142,7 +144,7 @@ def test_parse_parts(self): check(['c:/a/b', 'c:/x/y'], ('c:', '\\', ['c:\\', 'x', 'y'])) def test_splitroot(self): - f = self.flavour.splitroot + f = self.cls._split_root self.assertEqual(f(''), ('', '', '')) self.assertEqual(f('a'), ('', '', 'a')) self.assertEqual(f('a\\b'), ('', '', 'a\\b')) @@ -151,19 +153,12 @@ def test_splitroot(self): self.assertEqual(f('c:a\\b'), ('c:', '', 'a\\b')) self.assertEqual(f('c:\\a\\b'), ('c:', '\\', 'a\\b')) # Redundant slashes in the root are collapsed. - self.assertEqual(f('\\\\a'), ('', '\\', 'a')) - self.assertEqual(f('\\\\\\a/b'), ('', '\\', 'a/b')) self.assertEqual(f('c:\\\\a'), ('c:', '\\', 'a')) self.assertEqual(f('c:\\\\\\a/b'), ('c:', '\\', 'a/b')) # Valid UNC paths. self.assertEqual(f('\\\\a\\b'), ('\\\\a\\b', '\\', '')) self.assertEqual(f('\\\\a\\b\\'), ('\\\\a\\b', '\\', '')) self.assertEqual(f('\\\\a\\b\\c\\d'), ('\\\\a\\b', '\\', 'c\\d')) - # These are non-UNC paths (according to ntpath.py and test_ntpath). - # However, command.com says such paths are invalid, so it's - # difficult to know what the right semantics are. - self.assertEqual(f('\\\\\\a\\b'), ('', '\\', 'a\\b')) - self.assertEqual(f('\\\\a'), ('', '\\', 'a')) # @@ -182,8 +177,7 @@ class _BasePurePathTest(object): ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), ], '/b/c/d': [ - ('a', '/b/c', 'd'), ('a', '///b//c', 'd/'), - ('/a', '/b/c', 'd'), + ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), # Empty components get removed. ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), ], @@ -291,19 +285,26 @@ def test_as_uri_common(self): def test_repr_common(self): for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - p = self.cls(pathstr) - clsname = p.__class__.__name__ - r = repr(p) - # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) - inner = r[len(clsname) + 1 : -1] - self.assertEqual(eval(inner), p.as_posix()) - # The repr() roundtrips. - q = eval(r, pathlib.__dict__) - self.assertIs(q.__class__, p.__class__) - self.assertEqual(q, p) - self.assertEqual(repr(q), r) + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + clsname = p.__class__.__name__ + r = repr(p) + # The repr() is in the form ClassName("forward-slashes path"). + self.assertTrue(r.startswith(clsname + '('), r) + self.assertTrue(r.endswith(')'), r) + inner = r[len(clsname) + 1 : -1] + self.assertEqual(eval(inner), p.as_posix()) + + def test_repr_roundtrips(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + r = repr(p) + # The repr() roundtrips. + q = eval(r, pathlib.__dict__) + self.assertIs(q.__class__, p.__class__) + self.assertEqual(q, p) + self.assertEqual(repr(q), r) def test_eq_common(self): P = self.cls @@ -2412,9 +2413,9 @@ def test_is_symlink(self): def test_is_junction(self): P = self.cls(BASE) - with mock.patch.object(P._flavour, 'pathmod'): - self.assertEqual(P.is_junction(), P._flavour.pathmod.isjunction.return_value) - P._flavour.pathmod.isjunction.assert_called_once_with(P) + with mock.patch.object(P._flavour, 'isjunction'): + self.assertEqual(P.is_junction(), P._flavour.isjunction.return_value) + P._flavour.isjunction.assert_called_once_with(P) def test_is_fifo_false(self): P = self.cls(BASE) @@ -3072,6 +3073,22 @@ def check(): check() +class PurePathSubclassTest(_BasePurePathTest, unittest.TestCase): + class cls(pathlib.PurePath): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +class PathSubclassTest(_BasePathTest, unittest.TestCase): + class cls(pathlib.Path): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + class CompatiblePathTest(unittest.TestCase): """ Test that a type can be made compatible with PurePath diff --git a/Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst b/Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst new file mode 100644 index 000000000000..8ca5213fb23a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-05-02-14-09.bpo-24132.W6iORO.rst @@ -0,0 +1,3 @@ +Make :class:`pathlib.PurePath` and :class:`~pathlib.Path` subclassable +(private to start). Previously, attempting to instantiate a subclass +resulted in an :exc:`AttributeError` being raised. Patch by Barney Gale. From webhook-mailer at python.org Fri Dec 23 18:43:35 2022 From: webhook-mailer at python.org (brettcannon) Date: Fri, 23 Dec 2022 23:43:35 -0000 Subject: [Python-checkins] gh-99947: Ensure unreported errors are chained for SystemError during import (GH-99946) Message-ID: <mailman.3046.1671839017.3313.python-checkins@python.org> https://github.com/python/cpython/commit/474220e3a58d739acc5154eb3e000461d2222d62 commit: 474220e3a58d739acc5154eb3e000461d2222d62 branch: main author: Sebastian Berg <sebastianb at nvidia.com> committer: brettcannon <brett at python.org> date: 2022-12-23T15:43:19-08:00 summary: gh-99947: Ensure unreported errors are chained for SystemError during import (GH-99946) files: A Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst M Lib/test/test_importlib/extension/test_loader.py M Objects/moduleobject.c M Python/importdl.c diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index d69192b56bac..3bf2bbdcdcc4 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -351,9 +351,14 @@ def test_bad_modules(self): ]: with self.subTest(name_base): name = self.name + '_' + name_base - with self.assertRaises(SystemError): + with self.assertRaises(SystemError) as cm: self.load_module_by_name(name) + # If there is an unreported exception, it should be chained + # with the `SystemError`. + if "unreported_exception" in name_base: + self.assertIsNotNone(cm.exception.__cause__) + def test_nonascii(self): # Test that modules with non-ASCII names can be loaded. # punycode behaves slightly differently in some-ASCII and no-ASCII diff --git a/Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst b/Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst new file mode 100644 index 000000000000..fbed192d317b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-12-02-09-31-19.gh-issue-99947.Ski7OC.rst @@ -0,0 +1 @@ +Raising SystemError on import will now have its cause be set to the original unexpected exception. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 8e03f2446f6f..24190e320ee6 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -327,9 +327,10 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } else { if (PyErr_Occurred()) { - PyErr_Format(PyExc_SystemError, - "creation of module %s raised unreported exception", - name); + _PyErr_FormatFromCause( + PyExc_SystemError, + "creation of module %s raised unreported exception", + name); goto error; } } @@ -431,7 +432,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) return -1; } if (PyErr_Occurred()) { - PyErr_Format( + _PyErr_FormatFromCause( PyExc_SystemError, "execution of module %s raised unreported exception", name); diff --git a/Python/importdl.c b/Python/importdl.c index 40227674ca47..91fa06f49c28 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -180,8 +180,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) } goto error; } else if (PyErr_Occurred()) { - PyErr_Clear(); - PyErr_Format( + _PyErr_FormatFromCause( PyExc_SystemError, "initialization of %s raised unreported exception", name_buf); From webhook-mailer at python.org Fri Dec 23 18:52:21 2022 From: webhook-mailer at python.org (rhettinger) Date: Fri, 23 Dec 2022 23:52:21 -0000 Subject: [Python-checkins] Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) Message-ID: <mailman.3047.1671839542.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f89de679ffec35e82548341cb23e675546602288 commit: f89de679ffec35e82548341cb23e675546602288 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-23T15:52:16-08:00 summary: Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9146ed1bfb62..9688aae68e6c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -834,7 +834,8 @@ which incur interpreter overhead. return chain.from_iterable(repeat(tuple(iterable), n)) def dotproduct(vec1, vec2): - return sum(map(operator.mul, vec1, vec2)) + "Compute a sum of products." + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) def convolve(signal, kernel): # See: https://betterexplained.com/articles/intuitive-convolution/ @@ -846,7 +847,7 @@ which incur interpreter overhead. window = collections.deque([0], maxlen=n) * n for x in chain(signal, repeat(0, n-1)): window.append(x) - yield sum(map(operator.mul, kernel, window)) + yield dotproduct(kernel, window) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. From webhook-mailer at python.org Fri Dec 23 19:00:27 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 00:00:27 -0000 Subject: [Python-checkins] Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) Message-ID: <mailman.3048.1671840028.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9477594374ba58bc00a95d99d0e2e22cccd02f5e commit: 9477594374ba58bc00a95d99d0e2e22cccd02f5e branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T16:00:21-08:00 summary: Add "strict" to dotproduct(). Add docstring. Factor-out common code. (GH-100480) (cherry picked from commit f89de679ffec35e82548341cb23e675546602288) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 8eb843ab0a67..371f7c1fdcd4 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -795,7 +795,8 @@ which incur interpreter overhead. return chain.from_iterable(repeat(tuple(iterable), n)) def dotproduct(vec1, vec2): - return sum(map(operator.mul, vec1, vec2)) + "Compute a sum of products." + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) def convolve(signal, kernel): # See: https://betterexplained.com/articles/intuitive-convolution/ @@ -807,7 +808,7 @@ which incur interpreter overhead. window = collections.deque([0], maxlen=n) * n for x in chain(signal, repeat(0, n-1)): window.append(x) - yield sum(map(operator.mul, kernel, window)) + yield dotproduct(kernel, window) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. From webhook-mailer at python.org Fri Dec 23 19:03:36 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 00:03:36 -0000 Subject: [Python-checkins] gh-94808: improve test coverage of number formatting (#99472) Message-ID: <mailman.3049.1671840217.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7ca45e5ddd493411e61706d07679ea54b954e41b commit: 7ca45e5ddd493411e61706d07679ea54b954e41b branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T18:03:31-06:00 summary: gh-94808: improve test coverage of number formatting (#99472) files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 79360f5549e4..41094a7e917d 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1307,6 +1307,20 @@ def __repr__(self): self.assertRaises(ValueError, ("{" + big + "}").format) self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) + # test number formatter errors: + self.assertRaises(ValueError, '{0:x}'.format, 1j) + self.assertRaises(ValueError, '{0:x}'.format, 1.0) + self.assertRaises(ValueError, '{0:X}'.format, 1j) + self.assertRaises(ValueError, '{0:X}'.format, 1.0) + self.assertRaises(ValueError, '{0:o}'.format, 1j) + self.assertRaises(ValueError, '{0:o}'.format, 1.0) + self.assertRaises(ValueError, '{0:u}'.format, 1j) + self.assertRaises(ValueError, '{0:u}'.format, 1.0) + self.assertRaises(ValueError, '{0:i}'.format, 1j) + self.assertRaises(ValueError, '{0:i}'.format, 1.0) + self.assertRaises(ValueError, '{0:d}'.format, 1j) + self.assertRaises(ValueError, '{0:d}'.format, 1.0) + # issue 6089 self.assertRaises(ValueError, "{0[0]x}".format, [None]) self.assertRaises(ValueError, "{0[0](10)}".format, [None]) @@ -1541,11 +1555,31 @@ def __int__(self): self.assertEqual('%X' % letter_m, '6D') self.assertEqual('%o' % letter_m, '155') self.assertEqual('%c' % letter_m, 'm') - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14), - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11), - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79), - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi), - self.assertRaises(TypeError, operator.mod, '%c', pi), + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) + self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) + self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) + self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) + self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + + class RaisingNumber: + def __int__(self): + raise RuntimeError('int') # should not be `TypeError` + def __index__(self): + raise RuntimeError('index') # should not be `TypeError` + + rn = RaisingNumber() + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) def test_formatting_with_enum(self): # issue18780 From webhook-mailer at python.org Fri Dec 23 21:04:26 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:04:26 -0000 Subject: [Python-checkins] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100456) Message-ID: <mailman.3050.1671847467.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a23cb72ac82372fac05ba36ce08923840ca0de06 commit: a23cb72ac82372fac05ba36ce08923840ca0de06 branch: main author: Illia Volochii <illia.volochii at gmail.com> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:04:20-08:00 summary: gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100456) files: A Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1bdfa0681e0..f798992d8af6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,7 +235,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1s, 3.0.7] + openssl_ver: [1.1.1s, 3.0.7, 3.1.0-beta1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst new file mode 100644 index 000000000000..8b08ca0dcef7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst @@ -0,0 +1 @@ +Start running SSL tests with OpenSSL 3.1.0-beta1. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 30d66964fd1d..5ad597c8347e 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -402,15 +402,15 @@ class BuildOpenSSL(AbstractBuilder): depend_target = 'depend' def _post_install(self): - if self.version.startswith("3.0"): - self._post_install_300() + if self.version.startswith("3."): + self._post_install_3xx() def _build_src(self, config_args=()): - if self.version.startswith("3.0"): + if self.version.startswith("3."): config_args += ("enable-fips",) super()._build_src(config_args) - def _post_install_300(self): + def _post_install_3xx(self): # create ssl/ subdir with example configs # Install FIPS module self._subprocess_call( From webhook-mailer at python.org Fri Dec 23 21:07:57 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:07:57 -0000 Subject: [Python-checkins] [3.11] Correct CVE-2020-10735 documentation (GH-100306). (#100476) Message-ID: <mailman.3051.1671847678.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a852c5f8ee19f996335841a78b73549d7d4117c4 commit: a852c5f8ee19f996335841a78b73549d7d4117c4 branch: 3.11 author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:07:50-08:00 summary: [3.11] Correct CVE-2020-10735 documentation (GH-100306). (#100476) (cherry picked from commit 1cf3d78c92eb07dc09d15cc2e773b0b1b9436825) Co-authored-by: Jeremy Paige <ucodery at gmail.com> files: M Doc/library/stdtypes.rst M Python/clinic/sysmodule.c.h M Python/sysmodule.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a1514f611d4f..2bc86e978387 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5476,7 +5476,7 @@ to mitigate denial of service attacks. This limit *only* applies to decimal or other non-power-of-two number bases. Hexadecimal, octal, and binary conversions are unlimited. The limit can be configured. -The :class:`int` type in CPython is an abitrary length number stored in binary +The :class:`int` type in CPython is an arbitrary length number stored in binary form (commonly known as a "bignum"). There exists no algorithm that can convert a string to a binary integer or a binary integer to a string in linear time, *unless* the base is a power of 2. Even the best known algorithms for base 10 @@ -5540,7 +5540,7 @@ and :class:`str` or :class:`bytes`: * ``int(string)`` with default base 10. * ``int(string, base)`` for all bases that are not a power of 2. * ``str(integer)``. -* ``repr(integer)`` +* ``repr(integer)``. * any other string conversion to base 10, for example ``f"{integer}"``, ``"{}".format(integer)``, or ``b"%d" % integer``. @@ -5568,7 +5568,7 @@ command line flag to configure the limit: :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. If both the env var and the ``-X`` option are set, the ``-X`` option takes precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initilization. + :data:`sys.int_info.default_max_str_digits` was used during initialization. From code, you can inspect the current limit and set a new one using these :mod:`sys` APIs: diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index b3ecab59a733..8ca3cd086059 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -673,7 +673,7 @@ PyDoc_STRVAR(sys_get_int_max_str_digits__doc__, "get_int_max_str_digits($module, /)\n" "--\n" "\n" -"Set the maximum string digits limit for non-binary int<->str conversions."); +"Return the maximum string digits limit for non-binary int<->str conversions."); #define SYS_GET_INT_MAX_STR_DIGITS_METHODDEF \ {"get_int_max_str_digits", (PyCFunction)sys_get_int_max_str_digits, METH_NOARGS, sys_get_int_max_str_digits__doc__}, @@ -1067,4 +1067,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=21a32aa71d36a98c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3cae0e0212d88bcd input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0ecfd77ba14c..68a835a30baa 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1627,12 +1627,12 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] sys.get_int_max_str_digits -Set the maximum string digits limit for non-binary int<->str conversions. +Return the maximum string digits limit for non-binary int<->str conversions. [clinic start generated code]*/ static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=8dab13e2023e60d5]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromSsize_t(interp->int_max_str_digits); From webhook-mailer at python.org Fri Dec 23 21:08:32 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:08:32 -0000 Subject: [Python-checkins] [3.10] Correct CVE-2020-10735 documentation (GH-100306). (#100477) Message-ID: <mailman.3052.1671847713.3313.python-checkins@python.org> https://github.com/python/cpython/commit/88fe8d701af3316c8869ea18ea1c7acec6f68c04 commit: 88fe8d701af3316c8869ea18ea1c7acec6f68c04 branch: 3.10 author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:08:27-08:00 summary: [3.10] Correct CVE-2020-10735 documentation (GH-100306). (#100477) (cherry picked from commit 1cf3d78c92eb07dc09d15cc2e773b0b1b9436825) Co-authored-by: Jeremy Paige <ucodery at gmail.com> files: M Doc/library/stdtypes.rst M Python/clinic/sysmodule.c.h M Python/sysmodule.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d71d6936c568..5750c7cb68c8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5427,7 +5427,7 @@ to mitigate denial of service attacks. This limit *only* applies to decimal or other non-power-of-two number bases. Hexadecimal, octal, and binary conversions are unlimited. The limit can be configured. -The :class:`int` type in CPython is an abitrary length number stored in binary +The :class:`int` type in CPython is an arbitrary length number stored in binary form (commonly known as a "bignum"). There exists no algorithm that can convert a string to a binary integer or a binary integer to a string in linear time, *unless* the base is a power of 2. Even the best known algorithms for base 10 @@ -5491,7 +5491,7 @@ and :class:`str` or :class:`bytes`: * ``int(string)`` with default base 10. * ``int(string, base)`` for all bases that are not a power of 2. * ``str(integer)``. -* ``repr(integer)`` +* ``repr(integer)``. * any other string conversion to base 10, for example ``f"{integer}"``, ``"{}".format(integer)``, or ``b"%d" % integer``. @@ -5519,7 +5519,7 @@ command line flag to configure the limit: :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. If both the env var and the ``-X`` option are set, the ``-X`` option takes precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initilization. + :data:`sys.int_info.default_max_str_digits` was used during initialization. From code, you can inspect the current limit and set a new one using these :mod:`sys` APIs: diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 2a6ad89137b0..63c88778555a 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -651,7 +651,7 @@ PyDoc_STRVAR(sys_get_int_max_str_digits__doc__, "get_int_max_str_digits($module, /)\n" "--\n" "\n" -"Set the maximum string digits limit for non-binary int<->str conversions."); +"Return the maximum string digits limit for non-binary int<->str conversions."); #define SYS_GET_INT_MAX_STR_DIGITS_METHODDEF \ {"get_int_max_str_digits", (PyCFunction)sys_get_int_max_str_digits, METH_NOARGS, sys_get_int_max_str_digits__doc__}, @@ -1036,4 +1036,4 @@ sys__deactivate_opcache(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=6230a1e3a4415744 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1d40b6af6e80cc71 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 2b5c9d3ebbe8..e740cf933d1d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1659,12 +1659,12 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] sys.get_int_max_str_digits -Set the maximum string digits limit for non-binary int<->str conversions. +Return the maximum string digits limit for non-binary int<->str conversions. [clinic start generated code]*/ static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=8dab13e2023e60d5]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromSsize_t(interp->int_max_str_digits); From webhook-mailer at python.org Fri Dec 23 21:14:20 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 02:14:20 -0000 Subject: [Python-checkins] gh-94808: improve test coverage of number formatting (GH-99472) Message-ID: <mailman.3053.1671848061.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c4aff6d273aea44265c35ff19b30cc6aea2e06d4 commit: c4aff6d273aea44265c35ff19b30cc6aea2e06d4 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T18:14:08-08:00 summary: gh-94808: improve test coverage of number formatting (GH-99472) (cherry picked from commit 7ca45e5ddd493411e61706d07679ea54b954e41b) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index b3c02c1b9984..c265438a86d5 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1306,6 +1306,20 @@ def __repr__(self): self.assertRaises(ValueError, ("{" + big + "}").format) self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) + # test number formatter errors: + self.assertRaises(ValueError, '{0:x}'.format, 1j) + self.assertRaises(ValueError, '{0:x}'.format, 1.0) + self.assertRaises(ValueError, '{0:X}'.format, 1j) + self.assertRaises(ValueError, '{0:X}'.format, 1.0) + self.assertRaises(ValueError, '{0:o}'.format, 1j) + self.assertRaises(ValueError, '{0:o}'.format, 1.0) + self.assertRaises(ValueError, '{0:u}'.format, 1j) + self.assertRaises(ValueError, '{0:u}'.format, 1.0) + self.assertRaises(ValueError, '{0:i}'.format, 1j) + self.assertRaises(ValueError, '{0:i}'.format, 1.0) + self.assertRaises(ValueError, '{0:d}'.format, 1j) + self.assertRaises(ValueError, '{0:d}'.format, 1.0) + # issue 6089 self.assertRaises(ValueError, "{0[0]x}".format, [None]) self.assertRaises(ValueError, "{0[0](10)}".format, [None]) @@ -1541,11 +1555,31 @@ def __int__(self): self.assertEqual('%X' % letter_m, '6D') self.assertEqual('%o' % letter_m, '155') self.assertEqual('%c' % letter_m, 'm') - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14), - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11), - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79), - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi), - self.assertRaises(TypeError, operator.mod, '%c', pi), + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) + self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) + self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) + self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) + self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + + class RaisingNumber: + def __int__(self): + raise RuntimeError('int') # should not be `TypeError` + def __index__(self): + raise RuntimeError('index') # should not be `TypeError` + + rn = RaisingNumber() + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) def test_formatting_with_enum(self): # issue18780 From webhook-mailer at python.org Fri Dec 23 21:14:20 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 02:14:20 -0000 Subject: [Python-checkins] gh-94808: improve test coverage of number formatting (GH-99472) Message-ID: <mailman.3054.1671848062.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b168b0e013854b9a7d896c72dfc4d220bd036461 commit: b168b0e013854b9a7d896c72dfc4d220bd036461 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T18:14:14-08:00 summary: gh-94808: improve test coverage of number formatting (GH-99472) (cherry picked from commit 7ca45e5ddd493411e61706d07679ea54b954e41b) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index f6a1651e76f7..1b31ae1e0560 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1266,6 +1266,20 @@ def __repr__(self): self.assertRaises(ValueError, ("{" + big + "}").format) self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) + # test number formatter errors: + self.assertRaises(ValueError, '{0:x}'.format, 1j) + self.assertRaises(ValueError, '{0:x}'.format, 1.0) + self.assertRaises(ValueError, '{0:X}'.format, 1j) + self.assertRaises(ValueError, '{0:X}'.format, 1.0) + self.assertRaises(ValueError, '{0:o}'.format, 1j) + self.assertRaises(ValueError, '{0:o}'.format, 1.0) + self.assertRaises(ValueError, '{0:u}'.format, 1j) + self.assertRaises(ValueError, '{0:u}'.format, 1.0) + self.assertRaises(ValueError, '{0:i}'.format, 1j) + self.assertRaises(ValueError, '{0:i}'.format, 1.0) + self.assertRaises(ValueError, '{0:d}'.format, 1j) + self.assertRaises(ValueError, '{0:d}'.format, 1.0) + # issue 6089 self.assertRaises(ValueError, "{0[0]x}".format, [None]) self.assertRaises(ValueError, "{0[0](10)}".format, [None]) @@ -1501,11 +1515,31 @@ def __int__(self): self.assertEqual('%X' % letter_m, '6D') self.assertEqual('%o' % letter_m, '155') self.assertEqual('%c' % letter_m, 'm') - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14), - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11), - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79), - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi), - self.assertRaises(TypeError, operator.mod, '%c', pi), + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) + self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) + self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) + self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) + self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + + class RaisingNumber: + def __int__(self): + raise RuntimeError('int') # should not be `TypeError` + def __index__(self): + raise RuntimeError('index') # should not be `TypeError` + + rn = RaisingNumber() + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) def test_formatting_with_enum(self): # issue18780 From webhook-mailer at python.org Fri Dec 23 21:30:33 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 02:30:33 -0000 Subject: [Python-checkins] gh-100268: Add is_integer method to int (#100439) Message-ID: <mailman.3055.1671849034.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3e46f9fe05b40ee42009878620f448d3a4b44cb5 commit: 3e46f9fe05b40ee42009878620f448d3a4b44cb5 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: gpshead <greg at krypto.org> date: 2022-12-23T18:30:27-08:00 summary: gh-100268: Add is_integer method to int (#100439) This improves the lives of type annotation users of `float` - which type checkers implicitly treat as `int|float` because that is what most code actually wants. Before this change a `.is_integer()` method could not be assumed to exist on things annotated as `: float` due to the method not existing on both types. files: A Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst M Doc/library/stdtypes.rst M Lib/test/test_long.py M Objects/clinic/longobject.c.h M Objects/longobject.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 624284f7092f..0ef03035a572 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -609,6 +609,12 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.8 +.. method:: int.is_integer() + + Returns ``True``. Exists for duck type compatibility with :meth:`float.is_integer`. + + .. versionadded:: 3.12 + Additional Methods on Float --------------------------- diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 77b37ca1fa4a..569ab15820e3 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1553,6 +1553,11 @@ def test_from_bytes_small(self): b = i.to_bytes(2, signed=True) self.assertIs(int.from_bytes(b, signed=True), i) + def test_is_integer(self): + self.assertTrue((-1).is_integer()) + self.assertTrue((0).is_integer()) + self.assertTrue((1).is_integer()) + def test_access_to_nonexistent_digit_0(self): # http://bugs.python.org/issue14630: A bug in _PyLong_Copy meant that # ob_digit[0] was being incorrectly accessed for instances of a diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst new file mode 100644 index 000000000000..73d04c19d1cc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-22-21-56-08.gh-issue-100268.xw_phB.rst @@ -0,0 +1 @@ +Add :meth:`int.is_integer` to improve duck type compatibility between :class:`int` and :class:`float`. diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index dde49099cf95..206bffdd086a 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -467,4 +467,22 @@ int_from_bytes(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyOb exit: return return_value; } -/*[clinic end generated code: output=bf6074ecf2f32cf4 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(int_is_integer__doc__, +"is_integer($self, /)\n" +"--\n" +"\n" +"Returns True. Exists for duck type compatibility with float.is_integer."); + +#define INT_IS_INTEGER_METHODDEF \ + {"is_integer", (PyCFunction)int_is_integer, METH_NOARGS, int_is_integer__doc__}, + +static PyObject * +int_is_integer_impl(PyObject *self); + +static PyObject * +int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return int_is_integer_impl(self); +} +/*[clinic end generated code: output=e518fe2b5d519322 input=a9049054013a1b77]*/ diff --git a/Objects/longobject.c b/Objects/longobject.c index 8596ce9797b5..0df3b9a9d564 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6168,6 +6168,19 @@ long_long_meth(PyObject *self, PyObject *Py_UNUSED(ignored)) return long_long(self); } +/*[clinic input] +int.is_integer + +Returns True. Exists for duck type compatibility with float.is_integer. +[clinic start generated code]*/ + +static PyObject * +int_is_integer_impl(PyObject *self) +/*[clinic end generated code: output=90f8e794ce5430ef input=7e41c4d4416e05f2]*/ +{ + Py_RETURN_TRUE; +} + static PyMethodDef long_methods[] = { {"conjugate", long_long_meth, METH_NOARGS, "Returns self, the complex conjugate of any int."}, @@ -6186,6 +6199,7 @@ static PyMethodDef long_methods[] = { INT___GETNEWARGS___METHODDEF INT___FORMAT___METHODDEF INT___SIZEOF___METHODDEF + INT_IS_INTEGER_METHODDEF {NULL, NULL} /* sentinel */ }; From webhook-mailer at python.org Sat Dec 24 00:21:59 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 05:21:59 -0000 Subject: [Python-checkins] gh-77771: Add enterabs example in sched (#92716) Message-ID: <mailman.3056.1671859321.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0f6420640c0f3462e6b76b01a392844676de1fb9 commit: 0f6420640c0f3462e6b76b01a392844676de1fb9 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-23T23:21:52-06:00 summary: gh-77771: Add enterabs example in sched (#92716) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/sched.rst diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index a4ba2848f11d..a051c65b97b0 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -44,16 +44,22 @@ Example:: ... print(time.time()) ... s.enter(10, 1, print_time) ... s.enter(5, 2, print_time, argument=('positional',)) + ... # despite having higher priority, 'keyword' runs after 'positional' as enter() is relative ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) + ... s.enterabs(1_650_000_000, 10, print_time, argument=("first enterabs",)) + ... s.enterabs(1_650_000_000, 5, print_time, argument=("second enterabs",)) ... s.run() ... print(time.time()) ... >>> print_some_times() - 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default - 930343700.276 + 1652342830.3640375 + From print_time 1652342830.3642538 second enterabs + From print_time 1652342830.3643398 first enterabs + From print_time 1652342835.3694863 positional + From print_time 1652342835.3696074 keyword + From print_time 1652342840.369612 default + 1652342840.3697174 + .. _scheduler-objects: From webhook-mailer at python.org Sat Dec 24 00:30:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 05:30:11 -0000 Subject: [Python-checkins] gh-77771: Add enterabs example in sched (GH-92716) Message-ID: <mailman.3057.1671859812.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ecfe468a6ef10e78b1f9ea8418f04bc63d6b62f7 commit: ecfe468a6ef10e78b1f9ea8418f04bc63d6b62f7 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T21:30:06-08:00 summary: gh-77771: Add enterabs example in sched (GH-92716) (cherry picked from commit 0f6420640c0f3462e6b76b01a392844676de1fb9) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/sched.rst diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index a4ba2848f11d..a051c65b97b0 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -44,16 +44,22 @@ Example:: ... print(time.time()) ... s.enter(10, 1, print_time) ... s.enter(5, 2, print_time, argument=('positional',)) + ... # despite having higher priority, 'keyword' runs after 'positional' as enter() is relative ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) + ... s.enterabs(1_650_000_000, 10, print_time, argument=("first enterabs",)) + ... s.enterabs(1_650_000_000, 5, print_time, argument=("second enterabs",)) ... s.run() ... print(time.time()) ... >>> print_some_times() - 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default - 930343700.276 + 1652342830.3640375 + From print_time 1652342830.3642538 second enterabs + From print_time 1652342830.3643398 first enterabs + From print_time 1652342835.3694863 positional + From print_time 1652342835.3696074 keyword + From print_time 1652342840.369612 default + 1652342840.3697174 + .. _scheduler-objects: From webhook-mailer at python.org Sat Dec 24 00:31:24 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 05:31:24 -0000 Subject: [Python-checkins] gh-77771: Add enterabs example in sched (GH-92716) Message-ID: <mailman.3058.1671859886.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b914054d9de574f5b3489f8601482eac5a7699e0 commit: b914054d9de574f5b3489f8601482eac5a7699e0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-23T21:31:18-08:00 summary: gh-77771: Add enterabs example in sched (GH-92716) (cherry picked from commit 0f6420640c0f3462e6b76b01a392844676de1fb9) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/sched.rst diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index a4ba2848f11d..a051c65b97b0 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -44,16 +44,22 @@ Example:: ... print(time.time()) ... s.enter(10, 1, print_time) ... s.enter(5, 2, print_time, argument=('positional',)) + ... # despite having higher priority, 'keyword' runs after 'positional' as enter() is relative ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) + ... s.enterabs(1_650_000_000, 10, print_time, argument=("first enterabs",)) + ... s.enterabs(1_650_000_000, 5, print_time, argument=("second enterabs",)) ... s.run() ... print(time.time()) ... >>> print_some_times() - 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default - 930343700.276 + 1652342830.3640375 + From print_time 1652342830.3642538 second enterabs + From print_time 1652342830.3643398 first enterabs + From print_time 1652342835.3694863 positional + From print_time 1652342835.3696074 keyword + From print_time 1652342840.369612 default + 1652342840.3697174 + .. _scheduler-objects: From webhook-mailer at python.org Sat Dec 24 00:47:16 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 24 Dec 2022 05:47:16 -0000 Subject: [Python-checkins] [3.11] gh-99110: Initialize frame->previous in init_frame to fix segmentation fault (GH-100182) (#100478) Message-ID: <mailman.3059.1671860837.3313.python-checkins@python.org> https://github.com/python/cpython/commit/57e727af3fda446dc79d65e2d17297d1194892ed commit: 57e727af3fda446dc79d65e2d17297d1194892ed branch: 3.11 author: Bill Fisher <william.w.fisher at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-24T11:17:10+05:30 summary: [3.11] gh-99110: Initialize frame->previous in init_frame to fix segmentation fault (GH-100182) (#100478) (cherry picked from commit 88d565f32a709140664444c6dea20ecd35a10e94) Co-authored-by: Bill Fisher <william.w.fisher at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst M Include/internal/pycore_frame.h M Lib/test/test_capi/test_misc.py M Modules/_testcapimodule.c M Objects/frameobject.c diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index ecc8346f5ef0..4866ea21b998 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -94,7 +94,10 @@ static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); -/* Consumes reference to func */ +/* Consumes reference to func and locals. + Does not initialize frame->previous, which happens + when frame is linked into the frame stack. + */ static inline void _PyFrame_InitializeSpecials( _PyInterpreterFrame *frame, PyFunctionObject *func, diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 6cda91677054..db7a77416219 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1335,6 +1335,16 @@ def test_frame_get_generator(self): frame = next(gen) self.assertIs(gen, _testcapi.frame_getgenerator(frame)) + def test_frame_fback_api(self): + """Test that accessing `f_back` does not cause a segmentation fault on + a frame created with `PyFrame_New` (GH-99110).""" + def dummy(): + pass + + frame = _testcapi.frame_new(dummy.__code__, globals(), locals()) + # The following line should not cause a segmentation fault. + self.assertIsNone(frame.f_back) + SUFFICIENT_TO_DEOPT_AND_SPECIALIZE = 100 diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst new file mode 100644 index 000000000000..175740dfca07 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-12-01-05-16.gh-issue-99110.1JqtIg.rst @@ -0,0 +1,2 @@ +Initialize frame->previous in frameobject.c to fix a segmentation fault when +accessing frames created by :c:func:`PyFrame_New`. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 70242d78ab8c..bb7de220a2db 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -22,6 +22,7 @@ #include "Python.h" #include "datetime.h" // PyDateTimeAPI +#include "frameobject.h" // PyFrame_New #include "marshal.h" // PyMarshal_WriteLongToFile #include "structmember.h" // PyMemberDef #include <float.h> // FLT_MAX @@ -6000,6 +6001,22 @@ frame_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(lasti); } +static PyObject * +frame_new(PyObject *self, PyObject *args) +{ + PyObject *code, *globals, *locals; + if (!PyArg_ParseTuple(args, "OOO", &code, &globals, &locals)) { + return NULL; + } + if (!PyCode_Check(code)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); + return NULL; + } + PyThreadState *tstate = PyThreadState_Get(); + + return (PyObject *)PyFrame_New(tstate, (PyCodeObject *)code, globals, locals); +} + static PyObject * eval_get_func_name(PyObject *self, PyObject *func) { @@ -6492,6 +6509,7 @@ static PyMethodDef TestMethods[] = { {"frame_getgenerator", frame_getgenerator, METH_O, NULL}, {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL}, {"frame_getlasti", frame_getlasti, METH_O, NULL}, + {"frame_new", frame_new, METH_VARARGS, NULL}, {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f77f5ed39392..1e6bdcfb592d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1010,6 +1010,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) Py_INCREF(func); PyCodeObject *code = (PyCodeObject *)func->func_code; _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus); + frame->previous = NULL; for (Py_ssize_t i = 0; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = NULL; } From webhook-mailer at python.org Sat Dec 24 00:51:53 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 24 Dec 2022 05:51:53 -0000 Subject: [Python-checkins] GH-91166: Implement zero copy writes for `SelectorSocketTransport` in asyncio (#31871) Message-ID: <mailman.3060.1671861115.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2eea9598e39c577e26461789272c93fd96db7956 commit: 2eea9598e39c577e26461789272c93fd96db7956 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-24T11:21:48+05:30 summary: GH-91166: Implement zero copy writes for `SelectorSocketTransport` in asyncio (#31871) Co-authored-by: Guido van Rossum <gvanrossum at gmail.com> files: From webhook-mailer at python.org Sat Dec 24 03:21:35 2022 From: webhook-mailer at python.org (rhettinger) Date: Sat, 24 Dec 2022 08:21:35 -0000 Subject: [Python-checkins] Misc Itertools recipe tweaks (GH-100493) Message-ID: <mailman.3061.1671870097.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0769f957514300a75be51fc6d1b963c8e359208b commit: 0769f957514300a75be51fc6d1b963c8e359208b branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-24T00:21:30-08:00 summary: Misc Itertools recipe tweaks (GH-100493) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9688aae68e6c..b3634aecd10d 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -788,6 +788,11 @@ which incur interpreter overhead. .. testcode:: + import collections + import math + import operator + import random + def take(n, iterable): "Return first n items of the iterable as a list" return list(islice(iterable, n)) @@ -892,6 +897,21 @@ which incur interpreter overhead. data[2] = 1 return iter_index(data, 1) if n > 2 else iter([]) + def factor(n): + "Prime factors of n." + # factor(97) --> 97 + # factor(98) --> 2 7 7 + # factor(99) --> 3 3 11 + for prime in sieve(n+1): + while True: + quotient, remainder = divmod(n, prime) + if remainder: + break + yield prime + n = quotient + if n == 1: + return + def flatten(list_of_lists): "Flatten one level of nesting" return chain.from_iterable(list_of_lists) @@ -1134,11 +1154,6 @@ which incur interpreter overhead. Now, we test all of the itertool recipes - >>> import operator - >>> import collections - >>> import math - >>> import random - >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] @@ -1251,6 +1266,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True + list(factor(0)) + [] + list(factor(1)) + [] + list(factor(2)) + [2] + list(factor(3)) + [3] + list(factor(4)) + [2, 2] + list(factor(5)) + [5] + list(factor(6)) + [2, 3] + list(factor(7)) + [7] + list(factor(8)) + [2, 2, 2] + list(factor(9)) + [3, 3] + list(factor(10)) + [2, 5] + all(math.prod(factor(n)) == n for n in range(1, 1000)) + True + all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + True + all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] From webhook-mailer at python.org Sat Dec 24 03:31:47 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 08:31:47 -0000 Subject: [Python-checkins] Misc Itertools recipe tweaks (GH-100493) Message-ID: <mailman.3062.1671870709.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ba87dae4536bbc55db559950e0b8fa1201086586 commit: ba87dae4536bbc55db559950e0b8fa1201086586 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T00:31:36-08:00 summary: Misc Itertools recipe tweaks (GH-100493) (cherry picked from commit 0769f957514300a75be51fc6d1b963c8e359208b) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 371f7c1fdcd4..219ad95c76a1 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -749,6 +749,11 @@ which incur interpreter overhead. .. testcode:: + import collections + import math + import operator + import random + def take(n, iterable): "Return first n items of the iterable as a list" return list(islice(iterable, n)) @@ -853,6 +858,21 @@ which incur interpreter overhead. data[2] = 1 return iter_index(data, 1) if n > 2 else iter([]) + def factor(n): + "Prime factors of n." + # factor(97) --> 97 + # factor(98) --> 2 7 7 + # factor(99) --> 3 3 11 + for prime in sieve(n+1): + while True: + quotient, remainder = divmod(n, prime) + if remainder: + break + yield prime + n = quotient + if n == 1: + return + def flatten(list_of_lists): "Flatten one level of nesting" return chain.from_iterable(list_of_lists) @@ -1104,11 +1124,6 @@ which incur interpreter overhead. Now, we test all of the itertool recipes - >>> import operator - >>> import collections - >>> import math - >>> import random - >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] @@ -1221,6 +1236,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True + list(factor(0)) + [] + list(factor(1)) + [] + list(factor(2)) + [2] + list(factor(3)) + [3] + list(factor(4)) + [2, 2] + list(factor(5)) + [5] + list(factor(6)) + [2, 3] + list(factor(7)) + [7] + list(factor(8)) + [2, 2, 2] + list(factor(9)) + [3, 3] + list(factor(10)) + [2, 5] + all(math.prod(factor(n)) == n for n in range(1, 1000)) + True + all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + True + all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] From webhook-mailer at python.org Sat Dec 24 09:46:16 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Sat, 24 Dec 2022 14:46:16 -0000 Subject: [Python-checkins] gh-100357: Convert several functions in `bltinsmodule` to AC (#100358) Message-ID: <mailman.3063.1671893177.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bdfb6943861431a79e63f0da2e6b3fe163c12bc7 commit: bdfb6943861431a79e63f0da2e6b3fe163c12bc7 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-24T06:45:47-08:00 summary: gh-100357: Convert several functions in `bltinsmodule` to AC (#100358) files: A Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst M Python/bltinmodule.c M Python/clinic/bltinmodule.c.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst new file mode 100644 index 000000000000..fb25de6c9a3c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-20-09-56-56.gh-issue-100357.hPyTwY.rst @@ -0,0 +1,2 @@ +Convert ``vars``, ``dir``, ``next``, ``getattr``, and ``iter`` to argument +clinic. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2d4822e6d468..9ebe4c8353d0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -837,31 +837,33 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, return result; } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +dir as builtin_dir + + arg: object = NULL + / + +Show attributes of an object. + +If called without an argument, return the names in the current scope. +Else, return an alphabetized list of names comprising (some of) the attributes +of the given object, and of attributes reachable from it. +If the object supplies a method named __dir__, it will be used; otherwise +the default dir() logic is used and returns: + for a module object: the module's attributes. + for a class object: its attributes, and recursively the attributes + of its bases. + for any other object: its attributes, its class's attributes, and + recursively the attributes of its class's base classes. +[clinic start generated code]*/ + static PyObject * -builtin_dir(PyObject *self, PyObject *args) +builtin_dir_impl(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=24f2c7a52c1e3b08 input=ed6d6ccb13d52251]*/ { - PyObject *arg = NULL; - - if (!PyArg_UnpackTuple(args, "dir", 0, 1, &arg)) - return NULL; return PyObject_Dir(arg); } -PyDoc_STRVAR(dir_doc, -"dir([object]) -> list of strings\n" -"\n" -"If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" -" for a module object: the module's attributes.\n" -" for a class object: its attributes, and recursively the attributes\n" -" of its bases.\n" -" for any other object: its attributes, its class's attributes, and\n" -" recursively the attributes of its class's base classes."); - /*[clinic input] divmod as builtin_divmod @@ -1109,36 +1111,39 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +getattr as builtin_getattr + + object: object + name: object + default: object = NULL + / + +Get a named attribute from an object. + +getattr(x, 'y') is equivalent to x.y +When a default argument is given, it is returned when the attribute doesn't +exist; without it, an exception is raised in that case. +[clinic start generated code]*/ + static PyObject * -builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=74ad0e225e3f701c input=d7562cd4c3556171]*/ { - PyObject *v, *name, *result; - - if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) - return NULL; + PyObject *result; - v = args[0]; - name = args[1]; - if (nargs > 2) { - if (_PyObject_LookupAttr(v, name, &result) == 0) { - PyObject *dflt = args[2]; - return Py_NewRef(dflt); + if (default_value != NULL) { + if (_PyObject_LookupAttr(object, name, &result) == 0) { + return Py_NewRef(default_value); } } else { - result = PyObject_GetAttr(v, name); + result = PyObject_GetAttr(object, name); } return result; } -PyDoc_STRVAR(getattr_doc, -"getattr(object, name[, default]) -> value\n\ -\n\ -Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ -When a default argument is given, it is returned when the attribute doesn't\n\ -exist; without it, an exception is raised in that case."); - /*[clinic input] globals as builtin_globals @@ -1450,34 +1455,43 @@ PyTypeObject PyMap_Type = { }; -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +next as builtin_next + + iterator: object + default: object = NULL + / + +Return the next item from the iterator. + +If default is given and the iterator is exhausted, +it is returned instead of raising StopIteration. +[clinic start generated code]*/ + static PyObject * -builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_next_impl(PyObject *module, PyObject *iterator, + PyObject *default_value) +/*[clinic end generated code: output=a38a94eeb447fef9 input=180f9984f182020f]*/ { - PyObject *it, *res; - - if (!_PyArg_CheckPositional("next", nargs, 1, 2)) - return NULL; + PyObject *res; - it = args[0]; - if (!PyIter_Check(it)) { + if (!PyIter_Check(iterator)) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", - Py_TYPE(it)->tp_name); + Py_TYPE(iterator)->tp_name); return NULL; } - res = (*Py_TYPE(it)->tp_iternext)(it); + res = (*Py_TYPE(iterator)->tp_iternext)(iterator); if (res != NULL) { return res; - } else if (nargs > 1) { - PyObject *def = args[1]; + } else if (default_value != NULL) { if (PyErr_Occurred()) { if(!PyErr_ExceptionMatches(PyExc_StopIteration)) return NULL; PyErr_Clear(); } - return Py_NewRef(def); + return Py_NewRef(default_value); } else if (PyErr_Occurred()) { return NULL; } else { @@ -1486,12 +1500,6 @@ builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } } -PyDoc_STRVAR(next_doc, -"next(iterator[, default])\n\ -\n\ -Return the next item from the iterator. If default is given and the iterator\n\ -is exhausted, it is returned instead of raising StopIteration."); - /*[clinic input] setattr as builtin_setattr @@ -1584,34 +1592,33 @@ builtin_hex(PyObject *module, PyObject *number) } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +iter as builtin_iter + + object: object + sentinel: object = NULL + / + +Get an iterator from an object. + +In the first form, the argument must supply its own iterator, or be a sequence. +In the second form, the callable is called until it returns the sentinel. +[clinic start generated code]*/ + static PyObject * -builtin_iter(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel) +/*[clinic end generated code: output=12cf64203c195a94 input=a5d64d9d81880ba6]*/ { - PyObject *v; - - if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) - return NULL; - v = args[0]; - if (nargs == 1) - return PyObject_GetIter(v); - if (!PyCallable_Check(v)) { + if (sentinel == NULL) + return PyObject_GetIter(object); + if (!PyCallable_Check(object)) { PyErr_SetString(PyExc_TypeError, - "iter(v, w): v must be callable"); + "iter(object, sentinel): object must be callable"); return NULL; } - PyObject *sentinel = args[1]; - return PyCallIter_New(v, sentinel); + return PyCallIter_New(object, sentinel); } -PyDoc_STRVAR(iter_doc, -"iter(iterable) -> iterator\n\ -iter(callable, sentinel) -> iterator\n\ -\n\ -Get an iterator from an object. In the first form, the argument must\n\ -supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the sentinel."); - /*[clinic input] aiter as builtin_aiter @@ -2390,20 +2397,29 @@ builtin_sorted(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +vars as builtin_vars + + object: object = NULL + / + +Show vars. + +Without arguments, equivalent to locals(). +With an argument, equivalent to object.__dict__. +[clinic start generated code]*/ + static PyObject * -builtin_vars(PyObject *self, PyObject *args) +builtin_vars_impl(PyObject *module, PyObject *object) +/*[clinic end generated code: output=840a7f64007a3e0a input=80cbdef9182c4ba3]*/ { - PyObject *v = NULL; PyObject *d; - if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) - return NULL; - if (v == NULL) { + if (object == NULL) { d = Py_XNewRef(PyEval_GetLocals()); } else { - if (_PyObject_LookupAttr(v, &_Py_ID(__dict__), &d) == 0) { + if (_PyObject_LookupAttr(object, &_Py_ID(__dict__), &d) == 0) { PyErr_SetString(PyExc_TypeError, "vars() argument must have __dict__ attribute"); } @@ -2411,12 +2427,6 @@ builtin_vars(PyObject *self, PyObject *args) return d; } -PyDoc_STRVAR(vars_doc, -"vars([object]) -> dictionary\n\ -\n\ -Without arguments, equivalent to locals().\n\ -With an argument, equivalent to object.__dict__."); - /*[clinic input] sum as builtin_sum @@ -2966,12 +2976,12 @@ static PyMethodDef builtin_methods[] = { BUILTIN_CHR_METHODDEF BUILTIN_COMPILE_METHODDEF BUILTIN_DELATTR_METHODDEF - {"dir", builtin_dir, METH_VARARGS, dir_doc}, + BUILTIN_DIR_METHODDEF BUILTIN_DIVMOD_METHODDEF BUILTIN_EVAL_METHODDEF BUILTIN_EXEC_METHODDEF BUILTIN_FORMAT_METHODDEF - {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, getattr_doc}, + BUILTIN_GETATTR_METHODDEF BUILTIN_GLOBALS_METHODDEF BUILTIN_HASATTR_METHODDEF BUILTIN_HASH_METHODDEF @@ -2980,13 +2990,13 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF - {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, iter_doc}, + BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", _PyCFunction_CAST(builtin_max), METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", _PyCFunction_CAST(builtin_min), METH_VARARGS | METH_KEYWORDS, min_doc}, - {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, + BUILTIN_NEXT_METHODDEF BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF @@ -2997,7 +3007,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_SETATTR_METHODDEF BUILTIN_SORTED_METHODDEF BUILTIN_SUM_METHODDEF - {"vars", builtin_vars, METH_VARARGS, vars_doc}, + BUILTIN_VARS_METHODDEF {NULL, NULL}, }; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 89f069dd97f6..baf955558a21 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -386,6 +386,49 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } +PyDoc_STRVAR(builtin_dir__doc__, +"dir($module, arg=<unrepresentable>, /)\n" +"--\n" +"\n" +"Show attributes of an object.\n" +"\n" +"If called without an argument, return the names in the current scope.\n" +"Else, return an alphabetized list of names comprising (some of) the attributes\n" +"of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used; otherwise\n" +"the default dir() logic is used and returns:\n" +" for a module object: the module\'s attributes.\n" +" for a class object: its attributes, and recursively the attributes\n" +" of its bases.\n" +" for any other object: its attributes, its class\'s attributes, and\n" +" recursively the attributes of its class\'s base classes."); + +#define BUILTIN_DIR_METHODDEF \ + {"dir", _PyCFunction_CAST(builtin_dir), METH_FASTCALL, builtin_dir__doc__}, + +static PyObject * +builtin_dir_impl(PyObject *module, PyObject *arg); + +static PyObject * +builtin_dir(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *arg = NULL; + + if (!_PyArg_CheckPositional("dir", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + arg = args[0]; +skip_optional: + return_value = builtin_dir_impl(module, arg); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_divmod__doc__, "divmod($module, x, y, /)\n" "--\n" @@ -546,6 +589,47 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject return return_value; } +PyDoc_STRVAR(builtin_getattr__doc__, +"getattr($module, object, name, default=<unrepresentable>, /)\n" +"--\n" +"\n" +"Get a named attribute from an object.\n" +"\n" +"getattr(x, \'y\') is equivalent to x.y\n" +"When a default argument is given, it is returned when the attribute doesn\'t\n" +"exist; without it, an exception is raised in that case."); + +#define BUILTIN_GETATTR_METHODDEF \ + {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, builtin_getattr__doc__}, + +static PyObject * +builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, + PyObject *default_value); + +static PyObject * +builtin_getattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *object; + PyObject *name; + PyObject *default_value = NULL; + + if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) { + goto exit; + } + object = args[0]; + name = args[1]; + if (nargs < 3) { + goto skip_optional; + } + default_value = args[2]; +skip_optional: + return_value = builtin_getattr_impl(module, object, name, default_value); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_globals__doc__, "globals($module, /)\n" "--\n" @@ -611,6 +695,44 @@ PyDoc_STRVAR(builtin_id__doc__, #define BUILTIN_ID_METHODDEF \ {"id", (PyCFunction)builtin_id, METH_O, builtin_id__doc__}, +PyDoc_STRVAR(builtin_next__doc__, +"next($module, iterator, default=<unrepresentable>, /)\n" +"--\n" +"\n" +"Return the next item from the iterator.\n" +"\n" +"If default is given and the iterator is exhausted,\n" +"it is returned instead of raising StopIteration."); + +#define BUILTIN_NEXT_METHODDEF \ + {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, builtin_next__doc__}, + +static PyObject * +builtin_next_impl(PyObject *module, PyObject *iterator, + PyObject *default_value); + +static PyObject * +builtin_next(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *iterator; + PyObject *default_value = NULL; + + if (!_PyArg_CheckPositional("next", nargs, 1, 2)) { + goto exit; + } + iterator = args[0]; + if (nargs < 2) { + goto skip_optional; + } + default_value = args[1]; +skip_optional: + return_value = builtin_next_impl(module, iterator, default_value); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_setattr__doc__, "setattr($module, obj, name, value, /)\n" "--\n" @@ -702,6 +824,43 @@ PyDoc_STRVAR(builtin_hex__doc__, #define BUILTIN_HEX_METHODDEF \ {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, +PyDoc_STRVAR(builtin_iter__doc__, +"iter($module, object, sentinel=<unrepresentable>, /)\n" +"--\n" +"\n" +"Get an iterator from an object.\n" +"\n" +"In the first form, the argument must supply its own iterator, or be a sequence.\n" +"In the second form, the callable is called until it returns the sentinel."); + +#define BUILTIN_ITER_METHODDEF \ + {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, builtin_iter__doc__}, + +static PyObject * +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel); + +static PyObject * +builtin_iter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *object; + PyObject *sentinel = NULL; + + if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) { + goto exit; + } + object = args[0]; + if (nargs < 2) { + goto skip_optional; + } + sentinel = args[1]; +skip_optional: + return_value = builtin_iter_impl(module, object, sentinel); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_aiter__doc__, "aiter($module, async_iterable, /)\n" "--\n" @@ -1080,6 +1239,41 @@ builtin_round(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } +PyDoc_STRVAR(builtin_vars__doc__, +"vars($module, object=<unrepresentable>, /)\n" +"--\n" +"\n" +"Show vars.\n" +"\n" +"Without arguments, equivalent to locals().\n" +"With an argument, equivalent to object.__dict__."); + +#define BUILTIN_VARS_METHODDEF \ + {"vars", _PyCFunction_CAST(builtin_vars), METH_FASTCALL, builtin_vars__doc__}, + +static PyObject * +builtin_vars_impl(PyObject *module, PyObject *object); + +static PyObject * +builtin_vars(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *object = NULL; + + if (!_PyArg_CheckPositional("vars", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + object = args[0]; +skip_optional: + return_value = builtin_vars_impl(module, object); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_sum__doc__, "sum($module, iterable, /, start=0)\n" "--\n" @@ -1215,4 +1409,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=973da43fa65aa727 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a6a8efe82cf8b81 input=a9049054013a1b77]*/ From webhook-mailer at python.org Sat Dec 24 09:48:48 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Sat, 24 Dec 2022 14:48:48 -0000 Subject: [Python-checkins] Remove wrong comment about `repr` in `test_unicode` (#100495) Message-ID: <mailman.3064.1671893329.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c6dac128612df08fbd7f5e91dcd74a367bce0ed9 commit: c6dac128612df08fbd7f5e91dcd74a367bce0ed9 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-24T06:48:43-08:00 summary: Remove wrong comment about `repr` in `test_unicode` (#100495) files: M Lib/test/test_unicode.py diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 41094a7e917d..4ebbb9d32a3d 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -94,7 +94,6 @@ def test_literals(self): self.assertNotEqual(r"\u0020", " ") def test_ascii(self): - # Test basic sanity of repr() self.assertEqual(ascii('abc'), "'abc'") self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") self.assertEqual(ascii('ab\\'), "'ab\\\\'") From webhook-mailer at python.org Sat Dec 24 10:23:30 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 24 Dec 2022 15:23:30 -0000 Subject: [Python-checkins] gh-99908: Tutorial: Modernize the 'data-record class' example (#100499) Message-ID: <mailman.3065.1671895410.3313.python-checkins@python.org> https://github.com/python/cpython/commit/00afa5066bd45348ed82a38d3442763b2ed1a068 commit: 00afa5066bd45348ed82a38d3442763b2ed1a068 branch: main author: JosephSBoyle <48555120+JosephSBoyle at users.noreply.github.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-24T15:23:24Z summary: gh-99908: Tutorial: Modernize the 'data-record class' example (#100499) Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 0e5a9402bc50..a206ba371976 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -738,18 +738,24 @@ Odds and Ends ============= Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. An empty class definition -will do nicely:: +"struct", bundling together a few named data items. The idiomatic approach +is to use :mod:`dataclasses` for this purpose:: - class Employee: - pass + from dataclasses import dataclasses - john = Employee() # Create an empty employee record + @dataclass + class Employee: + name: str + dept: str + salary: int - # Fill the fields of the record - john.name = 'John Doe' - john.dept = 'computer lab' - john.salary = 1000 +:: + + >>> john = Employee('john', 'computer lab', 1000) + >>> john.dept + 'computer lab' + >>> john.salary + 1000 A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For From webhook-mailer at python.org Sat Dec 24 10:32:05 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 15:32:05 -0000 Subject: [Python-checkins] gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) Message-ID: <mailman.3066.1671895926.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a7eee898abb6314669d700e9a22013f2210b1b7e commit: a7eee898abb6314669d700e9a22013f2210b1b7e branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T07:32:00-08:00 summary: gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) (cherry picked from commit 00afa5066bd45348ed82a38d3442763b2ed1a068) Co-authored-by: JosephSBoyle <48555120+JosephSBoyle at users.noreply.github.com> Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index d7a24b4893fb..5abb767cb0a8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -737,18 +737,24 @@ Odds and Ends ============= Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. An empty class definition -will do nicely:: +"struct", bundling together a few named data items. The idiomatic approach +is to use :mod:`dataclasses` for this purpose:: - class Employee: - pass + from dataclasses import dataclasses - john = Employee() # Create an empty employee record + @dataclass + class Employee: + name: str + dept: str + salary: int - # Fill the fields of the record - john.name = 'John Doe' - john.dept = 'computer lab' - john.salary = 1000 +:: + + >>> john = Employee('john', 'computer lab', 1000) + >>> john.dept + 'computer lab' + >>> john.salary + 1000 A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For From webhook-mailer at python.org Sat Dec 24 10:32:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 15:32:34 -0000 Subject: [Python-checkins] gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) Message-ID: <mailman.3067.1671895956.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3ea6f7fee883faf5f4cf8abfff62deb986b89d53 commit: 3ea6f7fee883faf5f4cf8abfff62deb986b89d53 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T07:32:29-08:00 summary: gh-99908: Tutorial: Modernize the 'data-record class' example (GH-100499) (cherry picked from commit 00afa5066bd45348ed82a38d3442763b2ed1a068) Co-authored-by: JosephSBoyle <48555120+JosephSBoyle at users.noreply.github.com> Co-authored-by: Alex Waygood <Alex.Waygood at Gmail.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index d7a24b4893fb..5abb767cb0a8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -737,18 +737,24 @@ Odds and Ends ============= Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. An empty class definition -will do nicely:: +"struct", bundling together a few named data items. The idiomatic approach +is to use :mod:`dataclasses` for this purpose:: - class Employee: - pass + from dataclasses import dataclasses - john = Employee() # Create an empty employee record + @dataclass + class Employee: + name: str + dept: str + salary: int - # Fill the fields of the record - john.name = 'John Doe' - john.dept = 'computer lab' - john.salary = 1000 +:: + + >>> john = Employee('john', 'computer lab', 1000) + >>> john.dept + 'computer lab' + >>> john.salary + 1000 A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For From webhook-mailer at python.org Sat Dec 24 13:29:06 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 18:29:06 -0000 Subject: [Python-checkins] gh-100474: Fix handling of dirs named index.html in http.server (GH-100475) Message-ID: <mailman.3068.1671906546.3313.python-checkins@python.org> https://github.com/python/cpython/commit/46e6a28308def2c3a71c679a6fa4ed7d520802b9 commit: 46e6a28308def2c3a71c679a6fa4ed7d520802b9 branch: main author: James Frost <git at frost.cx> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T10:28:59-08:00 summary: gh-100474: Fix handling of dirs named index.html in http.server (GH-100475) If you had a directory called index.html or index.htm within a directory, it would cause http.server to return a 404 Not Found error instead of the directory listing. This came about due to not checking that the index was a regular file. I have also added a test case for this situation. Automerge-Triggered-By: GH:merwok files: A Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 8acabff605e7..221c8be4ae4b 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -711,7 +711,7 @@ def send_head(self): return None for index in self.index_pages: index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index ca078862cca6..cbcf94136ac4 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -489,6 +489,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: diff --git a/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst new file mode 100644 index 000000000000..31abfb8b87fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst @@ -0,0 +1,2 @@ +:mod:`http.server` now checks that an index page is actually a regular file before trying +to serve it. This avoids issues with directories named ``index.html``. From webhook-mailer at python.org Sat Dec 24 14:39:44 2022 From: webhook-mailer at python.org (cjw296) Date: Sat, 24 Dec 2022 19:39:44 -0000 Subject: [Python-checkins] gh-100287: Fix unittest.mock.seal with AsyncMock (#100496) Message-ID: <mailman.3069.1671910785.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e4b43ebb3afbd231a4e5630e7e358aa3093f8677 commit: e4b43ebb3afbd231a4e5630e7e358aa3093f8677 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: cjw296 <chris at withers.org> date: 2022-12-24T19:39:39Z summary: gh-100287: Fix unittest.mock.seal with AsyncMock (#100496) files: A Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst M Lib/test/test_unittest/testmock/testasync.py M Lib/unittest/mock.py diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 52a3b71be1ef..471162dc5050 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -11,7 +11,7 @@ from asyncio import run, iscoroutinefunction from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, - create_autospec, sentinel, _CallList) + create_autospec, sentinel, _CallList, seal) def tearDownModule(): @@ -300,6 +300,14 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_normal_methods_on_class_with_mock_seal(self): + mock = Mock(AsyncClass) + seal(mock) + with self.assertRaises(AttributeError): + mock.normal_method + with self.assertRaises(AttributeError): + mock.async_method + def test_spec_async_attributes_instance(self): async_instance = AsyncClass() async_instance.async_func_attr = async_func @@ -1089,3 +1097,7 @@ async def f(x=None): pass 'Actual: [call(1)]'))) as cm: self.mock.assert_has_awaits([call(), call(1, 2)]) self.assertIsInstance(cm.exception.__cause__, TypeError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 583ab74a8255..994947cad518 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1019,15 +1019,15 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" - _new_name = kw.get("_new_name") - if _new_name in self.__dict__['_spec_asyncs']: - return AsyncMock(**kw) - if self._mock_sealed: attribute = f".{kw['name']}" if "name" in kw else "()" mock_name = self._extract_mock_name() + attribute raise AttributeError(mock_name) + _new_name = kw.get("_new_name") + if _new_name in self.__dict__['_spec_asyncs']: + return AsyncMock(**kw) + _type = type(self) if issubclass(_type, MagicMock) and _new_name in _async_method_magics: # Any asynchronous magic becomes an AsyncMock diff --git a/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst new file mode 100644 index 000000000000..b353f0810c6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst @@ -0,0 +1 @@ +Fix the interaction of :func:`unittest.mock.seal` with :class:`unittest.mock.AsyncMock`. From webhook-mailer at python.org Sat Dec 24 15:07:20 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Sat, 24 Dec 2022 20:07:20 -0000 Subject: [Python-checkins] gh-99535: Add test for inheritance of annotations and update documentation (#99990) Message-ID: <mailman.3070.1671912441.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f5b7b19bf10724d831285fb04e00f763838bd555 commit: f5b7b19bf10724d831285fb04e00f763838bd555 branch: main author: MonadChains <monadchains at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-24T12:07:14-08:00 summary: gh-99535: Add test for inheritance of annotations and update documentation (#99990) files: M Doc/howto/annotations.rst M Doc/library/typing.rst M Lib/test/test_grammar.py diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 2bc2f2d4c839..472069032d65 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -57,6 +57,12 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer newer is to call :func:`getattr` with three arguments, for example ``getattr(o, '__annotations__', None)``. + Before Python 3.10, accessing ``__annotations__`` on a class that + defines no annotations but that has a parent class with + annotations would return the parent's ``__annotations__``. + In Python 3.10 and newer, the child class's annotations + will be an empty dict instead. + Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 356f919a1897..4eed6b4ea887 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2777,6 +2777,10 @@ Introspection helpers .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. + .. versionchanged:: 3.10 + Calling ``get_type_hints()`` on a class no longer returns the annotations + of its base classes. + .. versionchanged:: 3.11 Previously, ``Optional[t]`` was added for function and method annotations if a default value equal to ``None`` was set. diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 58f907eac09d..5b946020994e 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -415,6 +415,28 @@ class Cbad2(C): x: int x.y: list = [] + def test_annotations_inheritance(self): + # Check that annotations are not inherited by derived classes + class A: + attr: int + class B(A): + pass + class C(A): + attr: str + class D: + attr2: int + class E(A, D): + pass + class F(C, A): + pass + self.assertEqual(A.__annotations__, {"attr": int}) + self.assertEqual(B.__annotations__, {}) + self.assertEqual(C.__annotations__, {"attr" : str}) + self.assertEqual(D.__annotations__, {"attr2" : int}) + self.assertEqual(E.__annotations__, {}) + self.assertEqual(F.__annotations__, {}) + + def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod From webhook-mailer at python.org Sat Dec 24 15:09:36 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:09:36 -0000 Subject: [Python-checkins] gh-100428: Make float documentation more accurate (#100437) Message-ID: <mailman.3071.1671912577.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2e1a9ce9890aba748a518a39d01d1ea6d623d0d9 commit: 2e1a9ce9890aba748a518a39d01d1ea6d623d0d9 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:09:31-06:00 summary: gh-100428: Make float documentation more accurate (#100437) Previously, the grammar did not accept `float("10")`. Also implement mdickinson's suggestion of removing the indirection. files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 817c1f858aae..2e988257d5d3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -650,20 +650,23 @@ are always available. They are listed here in alphabetical order. sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the following grammar after leading and trailing - whitespace characters are removed: + input must conform to the ``floatvalue`` production rule in the following + grammar, after leading and trailing whitespace characters are removed: .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - numeric_value: `floatnumber` | `infinity` | `nan` - numeric_string: [`sign`] `numeric_value` - - Here ``floatnumber`` is the form of a Python floating-point literal, - described in :ref:`floating`. Case is not significant, so, for example, - "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for - positive infinity. + digitpart: `digit` (["_"] `digit`)* + number: [`digitpart`] "." `digitpart` | `digitpart` ["."] + exponent: ("e" | "E") ["+" | "-"] `digitpart` + floatnumber: number [`exponent`] + floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + + Here ``digit`` is a Unicode decimal digit (character in the Unicode general + category ``Nd``). Case is not significant, so, for example, "inf", "Inf", + "INFINITY", and "iNfINity" are all acceptable spellings for positive + infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point From webhook-mailer at python.org Sat Dec 24 15:12:10 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 20:12:10 -0000 Subject: [Python-checkins] [3.11] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100486) Message-ID: <mailman.3072.1671912730.3313.python-checkins@python.org> https://github.com/python/cpython/commit/341bdd6589cf11b1d01e8928fde8c180e1a92b5a commit: 341bdd6589cf11b1d01e8928fde8c180e1a92b5a branch: 3.11 author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-24T12:12:04-08:00 summary: [3.11] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (#100486) [3.11] gh-100454: Start running SSL tests with OpenSSL 3.1.0-beta1 (GH-100456). (cherry picked from commit a23cb72ac82372fac05ba36ce08923840ca0de06) Co-authored-by: Illia Volochii <illia.volochii at gmail.com> files: A Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 328714e87714..5647d6b29b13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -268,7 +268,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1q, 3.0.5] + openssl_ver: [1.1.1s, 3.0.7, 3.1.0-beta1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst new file mode 100644 index 000000000000..8b08ca0dcef7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-23-13-29-55.gh-issue-100454.3no0cW.rst @@ -0,0 +1 @@ +Start running SSL tests with OpenSSL 3.1.0-beta1. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 91d6f558bc51..b1aacfa550f1 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -403,15 +403,15 @@ class BuildOpenSSL(AbstractBuilder): depend_target = 'depend' def _post_install(self): - if self.version.startswith("3.0"): - self._post_install_300() + if self.version.startswith("3."): + self._post_install_3xx() def _build_src(self, config_args=()): - if self.version.startswith("3.0"): + if self.version.startswith("3."): config_args += ("enable-fips",) super()._build_src(config_args) - def _post_install_300(self): + def _post_install_3xx(self): # create ssl/ subdir with example configs # Install FIPS module self._subprocess_call( From webhook-mailer at python.org Sat Dec 24 15:15:52 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:15:52 -0000 Subject: [Python-checkins] gh-100428: Make float documentation more accurate (GH-100437) Message-ID: <mailman.3073.1671912953.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0dea92409e291cd61135d509204f60c46f6dfc0b commit: 0dea92409e291cd61135d509204f60c46f6dfc0b branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:15:46-08:00 summary: gh-100428: Make float documentation more accurate (GH-100437) Previously, the grammar did not accept `float("10")`. Also implement mdickinson's suggestion of removing the indirection. (cherry picked from commit 2e1a9ce9890aba748a518a39d01d1ea6d623d0d9) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 97641d1b85cb..46e77fdb4155 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -628,20 +628,23 @@ are always available. They are listed here in alphabetical order. sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the following grammar after leading and trailing - whitespace characters are removed: + input must conform to the ``floatvalue`` production rule in the following + grammar, after leading and trailing whitespace characters are removed: .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - numeric_value: `floatnumber` | `infinity` | `nan` - numeric_string: [`sign`] `numeric_value` - - Here ``floatnumber`` is the form of a Python floating-point literal, - described in :ref:`floating`. Case is not significant, so, for example, - "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for - positive infinity. + digitpart: `digit` (["_"] `digit`)* + number: [`digitpart`] "." `digitpart` | `digitpart` ["."] + exponent: ("e" | "E") ["+" | "-"] `digitpart` + floatnumber: number [`exponent`] + floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + + Here ``digit`` is a Unicode decimal digit (character in the Unicode general + category ``Nd``). Case is not significant, so, for example, "inf", "Inf", + "INFINITY", and "iNfINity" are all acceptable spellings for positive + infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point From webhook-mailer at python.org Sat Dec 24 15:17:44 2022 From: webhook-mailer at python.org (gpshead) Date: Sat, 24 Dec 2022 20:17:44 -0000 Subject: [Python-checkins] [Minor PR] Quotes in documentation changed into code blocks (#99536) Message-ID: <mailman.3074.1671913065.3313.python-checkins@python.org> https://github.com/python/cpython/commit/efccd04b9efc1752a845b377399d2068b06d04e7 commit: efccd04b9efc1752a845b377399d2068b06d04e7 branch: main author: Bart Broere <mail at bartbroere.eu> committer: gpshead <greg at krypto.org> date: 2022-12-24T12:17:39-08:00 summary: [Minor PR] Quotes in documentation changed into code blocks (#99536) Minor formatting fix in documentation Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 14414ea7f81e..e4e38e933681 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1567,6 +1567,8 @@ If you ever encounter a presumed highly unusual situation where you need to prevent ``vfork()`` from being used by Python, you can set the :attr:`subprocess._USE_VFORK` attribute to a false value. +:: + subprocess._USE_VFORK = False # See CPython issue gh-NNNNNN. Setting this has no impact on use of ``posix_spawn()`` which could use @@ -1574,6 +1576,8 @@ Setting this has no impact on use of ``posix_spawn()`` which could use :attr:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of that. +:: + subprocess._USE_POSIX_SPAWN = False # See CPython issue gh-NNNNNN. It is safe to set these to false on any Python version. They will have no From webhook-mailer at python.org Sat Dec 24 15:18:46 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:18:46 -0000 Subject: [Python-checkins] gh-100428: Make float documentation more accurate (GH-100437) Message-ID: <mailman.3075.1671913127.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bf0f306bcd4cc6ebe7dcb32c79dd1aba0b910804 commit: bf0f306bcd4cc6ebe7dcb32c79dd1aba0b910804 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:18:40-08:00 summary: gh-100428: Make float documentation more accurate (GH-100437) Previously, the grammar did not accept `float("10")`. Also implement mdickinson's suggestion of removing the indirection. (cherry picked from commit 2e1a9ce9890aba748a518a39d01d1ea6d623d0d9) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 110e7e5d7fb9..304716194d22 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -649,20 +649,23 @@ are always available. They are listed here in alphabetical order. sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the following grammar after leading and trailing - whitespace characters are removed: + input must conform to the ``floatvalue`` production rule in the following + grammar, after leading and trailing whitespace characters are removed: .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - numeric_value: `floatnumber` | `infinity` | `nan` - numeric_string: [`sign`] `numeric_value` - - Here ``floatnumber`` is the form of a Python floating-point literal, - described in :ref:`floating`. Case is not significant, so, for example, - "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for - positive infinity. + digitpart: `digit` (["_"] `digit`)* + number: [`digitpart`] "." `digitpart` | `digitpart` ["."] + exponent: ("e" | "E") ["+" | "-"] `digitpart` + floatnumber: number [`exponent`] + floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + + Here ``digit`` is a Unicode decimal digit (character in the Unicode general + category ``Nd``). Case is not significant, so, for example, "inf", "Inf", + "INFINITY", and "iNfINity" are all acceptable spellings for positive + infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point From webhook-mailer at python.org Sat Dec 24 15:22:54 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:22:54 -0000 Subject: [Python-checkins] gh-100472: Fix docs claim that compileall parameters could be bytes (#100473) Message-ID: <mailman.3076.1671913375.3313.python-checkins@python.org> https://github.com/python/cpython/commit/046cbc2080360b0b0bbe6ea7554045a6bbbd94bd commit: 046cbc2080360b0b0bbe6ea7554045a6bbbd94bd branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:22:49-06:00 summary: gh-100472: Fix docs claim that compileall parameters could be bytes (#100473) files: A Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst M Doc/library/compileall.rst diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 7af46cf32008..180f5b81c2b6 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -199,7 +199,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. @@ -269,7 +269,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst new file mode 100644 index 000000000000..4f4162150750 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst @@ -0,0 +1 @@ +Remove claim in documentation that the ``stripdir``, ``prependdir`` and ``limit_sl_dest`` parameters of :func:`compileall.compile_dir` and :func:`compileall.compile_file` could be :class:`bytes`. From webhook-mailer at python.org Sat Dec 24 15:25:39 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:25:39 -0000 Subject: [Python-checkins] GH-93179: Document the thread safety of functools.lru_cache (GH-95970) Message-ID: <mailman.3077.1671913541.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3b704874592a3a1895c9a69a39c23ea22d1d06ca commit: 3b704874592a3a1895c9a69a39c23ea22d1d06ca branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:25:34-08:00 summary: GH-93179: Document the thread safety of functools.lru_cache (GH-95970) (cherry picked from commit ba4bb7e4649be99d5d6b4151a1bd2eac89ef97f2) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/functools.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 9beebd2a85e5..2f0a9bd8be88 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -49,6 +49,9 @@ The :mod:`functools` module defines the following functions: >>> factorial(12) # makes two new recursive calls, the other 10 are cached 479001600 + The cache is threadsafe so the wrapped function can be used in multiple + threads. + .. versionadded:: 3.9 @@ -140,6 +143,9 @@ The :mod:`functools` module defines the following functions: *maxsize* most recent calls. It can save time when an expensive or I/O bound function is periodically called with the same arguments. + The cache is threadsafe so the wrapped function can be used in multiple + threads. + Since a dictionary is used to cache results, the positional and keyword arguments to the function must be hashable. From webhook-mailer at python.org Sat Dec 24 15:26:16 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:26:16 -0000 Subject: [Python-checkins] [Minor PR] Quotes in documentation changed into code blocks (GH-99536) Message-ID: <mailman.3078.1671913576.3313.python-checkins@python.org> https://github.com/python/cpython/commit/80013d78e0efa0bbb610408227ead50a4f16588d commit: 80013d78e0efa0bbb610408227ead50a4f16588d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:26:11-08:00 summary: [Minor PR] Quotes in documentation changed into code blocks (GH-99536) Minor formatting fix in documentation (cherry picked from commit efccd04b9efc1752a845b377399d2068b06d04e7) Co-authored-by: Bart Broere <mail at bartbroere.eu> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 14414ea7f81e..e4e38e933681 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1567,6 +1567,8 @@ If you ever encounter a presumed highly unusual situation where you need to prevent ``vfork()`` from being used by Python, you can set the :attr:`subprocess._USE_VFORK` attribute to a false value. +:: + subprocess._USE_VFORK = False # See CPython issue gh-NNNNNN. Setting this has no impact on use of ``posix_spawn()`` which could use @@ -1574,6 +1576,8 @@ Setting this has no impact on use of ``posix_spawn()`` which could use :attr:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of that. +:: + subprocess._USE_POSIX_SPAWN = False # See CPython issue gh-NNNNNN. It is safe to set these to false on any Python version. They will have no From webhook-mailer at python.org Sat Dec 24 15:28:46 2022 From: webhook-mailer at python.org (merwok) Date: Sat, 24 Dec 2022 20:28:46 -0000 Subject: [Python-checkins] [3.11] gh-100474: Fix handling of dirs named index.html in http.server (GH-100505) Message-ID: <mailman.3079.1671913727.3313.python-checkins@python.org> https://github.com/python/cpython/commit/714a93f6383042c1c12d9bdf2b5c2cdd7a72c20d commit: 714a93f6383042c1c12d9bdf2b5c2cdd7a72c20d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: merwok <merwok at netwok.org> date: 2022-12-24T15:28:41-05:00 summary: [3.11] gh-100474: Fix handling of dirs named index.html in http.server (GH-100505) Co-authored-by: James Frost <git at frost.cx> files: A Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 058ee47ba10e..eef7cbb3ec75 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -708,7 +708,7 @@ def send_head(self): return None for index in "index.html", "index.htm": index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index ca078862cca6..cbcf94136ac4 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -489,6 +489,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: diff --git a/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst new file mode 100644 index 000000000000..31abfb8b87fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst @@ -0,0 +1,2 @@ +:mod:`http.server` now checks that an index page is actually a regular file before trying +to serve it. This avoids issues with directories named ``index.html``. From webhook-mailer at python.org Sat Dec 24 15:29:27 2022 From: webhook-mailer at python.org (merwok) Date: Sat, 24 Dec 2022 20:29:27 -0000 Subject: [Python-checkins] [3.10] gh-100474: Fix handling of dirs named index.html in http.server (GH-100504) Message-ID: <mailman.3080.1671913768.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ecbf136702267857b261a81b64f234965b9de913 commit: ecbf136702267857b261a81b64f234965b9de913 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: merwok <merwok at netwok.org> date: 2022-12-24T15:29:21-05:00 summary: [3.10] gh-100474: Fix handling of dirs named index.html in http.server (GH-100504) Co-authored-by: James Frost <git at frost.cx> files: A Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 03dbaa51b798..9c218d06acf5 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -709,7 +709,7 @@ def send_head(self): return None for index in "index.html", "index.htm": index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index ac8da494e9bb..a5f787ff48c9 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -488,6 +488,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: diff --git a/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst new file mode 100644 index 000000000000..31abfb8b87fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-23-21-02-43.gh-issue-100474.gppA4U.rst @@ -0,0 +1,2 @@ +:mod:`http.server` now checks that an index page is actually a regular file before trying +to serve it. This avoids issues with directories named ``index.html``. From webhook-mailer at python.org Sat Dec 24 15:29:46 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:29:46 -0000 Subject: [Python-checkins] gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) Message-ID: <mailman.3081.1671913787.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ad8d2ef54ffde39c6d59c4fc6c0e9b8c529b306d commit: ad8d2ef54ffde39c6d59c4fc6c0e9b8c529b306d branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:29:41-08:00 summary: gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) (cherry picked from commit 046cbc2080360b0b0bbe6ea7554045a6bbbd94bd) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst M Doc/library/compileall.rst diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index de34664acb84..b020b71dea5f 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -198,7 +198,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. @@ -268,7 +268,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst new file mode 100644 index 000000000000..4f4162150750 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst @@ -0,0 +1 @@ +Remove claim in documentation that the ``stripdir``, ``prependdir`` and ``limit_sl_dest`` parameters of :func:`compileall.compile_dir` and :func:`compileall.compile_file` could be :class:`bytes`. From webhook-mailer at python.org Sat Dec 24 15:29:55 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:29:55 -0000 Subject: [Python-checkins] gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) Message-ID: <mailman.3082.1671913796.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f801fa8ee5e6f67481b78cce495a9228d82c81c3 commit: f801fa8ee5e6f67481b78cce495a9228d82c81c3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:29:50-08:00 summary: gh-100472: Fix docs claim that compileall parameters could be bytes (GH-100473) (cherry picked from commit 046cbc2080360b0b0bbe6ea7554045a6bbbd94bd) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst M Doc/library/compileall.rst diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 7af46cf32008..180f5b81c2b6 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -199,7 +199,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. @@ -269,7 +269,7 @@ Public functions The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + They may be specified as ``str`` or :py:class:`os.PathLike`. If *hardlink_dupes* is true and two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. diff --git a/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst new file mode 100644 index 000000000000..4f4162150750 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-23-21-42-26.gh-issue-100472.NNixfO.rst @@ -0,0 +1 @@ +Remove claim in documentation that the ``stripdir``, ``prependdir`` and ``limit_sl_dest`` parameters of :func:`compileall.compile_dir` and :func:`compileall.compile_file` could be :class:`bytes`. From webhook-mailer at python.org Sat Dec 24 15:31:15 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 24 Dec 2022 20:31:15 -0000 Subject: [Python-checkins] gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) Message-ID: <mailman.3083.1671913876.3313.python-checkins@python.org> https://github.com/python/cpython/commit/44b664e057772d1491748809a6a356694b3d3979 commit: 44b664e057772d1491748809a6a356694b3d3979 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-24T12:31:10-08:00 summary: gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) (cherry picked from commit f5b7b19bf10724d831285fb04e00f763838bd555) Co-authored-by: MonadChains <monadchains at gmail.com> files: M Doc/howto/annotations.rst M Doc/library/typing.rst M Lib/test/test_grammar.py diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 2bc2f2d4c839..472069032d65 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -57,6 +57,12 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer newer is to call :func:`getattr` with three arguments, for example ``getattr(o, '__annotations__', None)``. + Before Python 3.10, accessing ``__annotations__`` on a class that + defines no annotations but that has a parent class with + annotations would return the parent's ``__annotations__``. + In Python 3.10 and newer, the child class's annotations + will be an empty dict instead. + Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 7fc0aa3a8e62..a3a6181eba29 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2767,6 +2767,10 @@ Introspection helpers .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. + .. versionchanged:: 3.10 + Calling ``get_type_hints()`` on a class no longer returns the annotations + of its base classes. + .. versionchanged:: 3.11 Previously, ``Optional[t]`` was added for function and method annotations if a default value equal to ``None`` was set. diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index da8851986271..bcd8d584b0f5 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -415,6 +415,28 @@ class Cbad2(C): x: int x.y: list = [] + def test_annotations_inheritance(self): + # Check that annotations are not inherited by derived classes + class A: + attr: int + class B(A): + pass + class C(A): + attr: str + class D: + attr2: int + class E(A, D): + pass + class F(C, A): + pass + self.assertEqual(A.__annotations__, {"attr": int}) + self.assertEqual(B.__annotations__, {}) + self.assertEqual(C.__annotations__, {"attr" : str}) + self.assertEqual(D.__annotations__, {"attr2" : int}) + self.assertEqual(E.__annotations__, {}) + self.assertEqual(F.__annotations__, {}) + + def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod From webhook-mailer at python.org Sat Dec 24 15:39:25 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:39:25 -0000 Subject: [Python-checkins] [3.11] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100506) Message-ID: <mailman.3084.1671914366.3313.python-checkins@python.org> https://github.com/python/cpython/commit/75b75dfdacc25a0ffc125c2e372036c3f799f5ea commit: 75b75dfdacc25a0ffc125c2e372036c3f799f5ea branch: 3.11 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:39:19-06:00 summary: [3.11] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100506) (cherry picked from commit e4b43ebb3afbd231a4e5630e7e358aa3093f8677) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst M Lib/unittest/mock.py M Lib/unittest/test/testmock/testasync.py diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 6720e5bc22dc..6a1c932081e0 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1014,15 +1014,15 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" - _new_name = kw.get("_new_name") - if _new_name in self.__dict__['_spec_asyncs']: - return AsyncMock(**kw) - if self._mock_sealed: attribute = f".{kw['name']}" if "name" in kw else "()" mock_name = self._extract_mock_name() + attribute raise AttributeError(mock_name) + _new_name = kw.get("_new_name") + if _new_name in self.__dict__['_spec_asyncs']: + return AsyncMock(**kw) + _type = type(self) if issubclass(_type, MagicMock) and _new_name in _async_method_magics: # Any asynchronous magic becomes an AsyncMock diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py index e05a22861d47..df260abde950 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/unittest/test/testmock/testasync.py @@ -11,7 +11,7 @@ from asyncio import run, iscoroutinefunction from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, - create_autospec, sentinel, _CallList) + create_autospec, sentinel, _CallList, seal) def tearDownModule(): @@ -300,6 +300,14 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_normal_methods_on_class_with_mock_seal(self): + mock = Mock(AsyncClass) + seal(mock) + with self.assertRaises(AttributeError): + mock.normal_method + with self.assertRaises(AttributeError): + mock.async_method + def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) @@ -1076,3 +1084,7 @@ async def f(x=None): pass 'Actual: [call(1)]'))) as cm: self.mock.assert_has_awaits([call(), call(1, 2)]) self.assertIsInstance(cm.exception.__cause__, TypeError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst new file mode 100644 index 000000000000..b353f0810c6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst @@ -0,0 +1 @@ +Fix the interaction of :func:`unittest.mock.seal` with :class:`unittest.mock.AsyncMock`. From webhook-mailer at python.org Sat Dec 24 15:39:32 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sat, 24 Dec 2022 20:39:32 -0000 Subject: [Python-checkins] [3.10] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100508) Message-ID: <mailman.3085.1671914374.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9975d4e7bac052c320880e026f757cddaa91b4bf commit: 9975d4e7bac052c320880e026f757cddaa91b4bf branch: 3.10 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T14:39:27-06:00 summary: [3.10] gh-100287: Fix unittest.mock.seal with AsyncMock (GH-100496) (#100508) (cherry picked from commit e4b43ebb3afbd231a4e5630e7e358aa3093f8677) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst M Lib/unittest/mock.py M Lib/unittest/test/testmock/testasync.py diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index a647e5dbc857..7832092d4b98 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1010,15 +1010,15 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" - _new_name = kw.get("_new_name") - if _new_name in self.__dict__['_spec_asyncs']: - return AsyncMock(**kw) - if self._mock_sealed: attribute = f".{kw['name']}" if "name" in kw else "()" mock_name = self._extract_mock_name() + attribute raise AttributeError(mock_name) + _new_name = kw.get("_new_name") + if _new_name in self.__dict__['_spec_asyncs']: + return AsyncMock(**kw) + _type = type(self) if issubclass(_type, MagicMock) and _new_name in _async_method_magics: # Any asynchronous magic becomes an AsyncMock diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py index 22228b47dee5..61269d637f30 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/unittest/test/testmock/testasync.py @@ -8,7 +8,7 @@ from asyncio import run, iscoroutinefunction from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, - create_autospec, sentinel, _CallList) + create_autospec, sentinel, _CallList, seal) def tearDownModule(): @@ -298,6 +298,14 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_normal_methods_on_class_with_mock_seal(self): + mock = Mock(AsyncClass) + seal(mock) + with self.assertRaises(AttributeError): + mock.normal_method + with self.assertRaises(AttributeError): + mock.async_method + def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) @@ -1074,3 +1082,7 @@ async def f(x=None): pass 'Actual: [call(1)]'))) as cm: self.mock.assert_has_awaits([call(), call(1, 2)]) self.assertIsInstance(cm.exception.__cause__, TypeError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst new file mode 100644 index 000000000000..b353f0810c6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-08-42-05.gh-issue-100287.n0oEuG.rst @@ -0,0 +1 @@ +Fix the interaction of :func:`unittest.mock.seal` with :class:`unittest.mock.AsyncMock`. From webhook-mailer at python.org Sat Dec 24 19:15:04 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sun, 25 Dec 2022 00:15:04 -0000 Subject: [Python-checkins] gh-100519: simplification to `eff_request_host` in cookiejar.py (#99588) Message-ID: <mailman.3086.1671927305.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b9aa14a484f653cb6a3a242776df9ac5fe161bfc commit: b9aa14a484f653cb6a3a242776df9ac5fe161bfc branch: main author: Glyph <code at glyph.im> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-24T18:14:51-06:00 summary: gh-100519: simplification to `eff_request_host` in cookiejar.py (#99588) `IPV4_RE` includes a `.`, and the `.find(".") == -1` included here is already testing to make sure there's no dot, so this part of the expression is tautological. Instead use more modern `in` syntax to make it clear what the check is doing here. The simplified implementation more clearly matches the wording in RFC 2965. Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: A Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst M Lib/http/cookiejar.py diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index e3df007033b3..93b10d26c845 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -640,7 +640,7 @@ def eff_request_host(request): """ erhn = req_host = request_host(request) - if req_host.find(".") == -1 and not IPV4_RE.search(req_host): + if "." not in req_host: erhn = req_host + ".local" return req_host, erhn diff --git a/Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst b/Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst new file mode 100644 index 000000000000..6b889b61c274 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-24-16-39-53.gh-issue-100519.G_dZLP.rst @@ -0,0 +1,2 @@ +Small simplification of :func:`http.cookiejar.eff_request_host` that +improves readability and better matches the RFC wording. From webhook-mailer at python.org Sun Dec 25 01:55:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 25 Dec 2022 06:55:32 -0000 Subject: [Python-checkins] gh-99308: Clarify re docs for byte pattern group names (#99311) Message-ID: <mailman.3087.1671951333.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dbc1e696ebf273bc62545d999eb185d6c9470e71 commit: dbc1e696ebf273bc62545d999eb185d6c9470e71 branch: main author: Ilya Kulakov <kulakov.ilya at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-25T12:25:27+05:30 summary: gh-99308: Clarify re docs for byte pattern group names (#99311) files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index cbee70b01d9f..d0a16b951844 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -395,9 +395,9 @@ The special characters are: ``(?P<name>...)`` Similar to regular parentheses, but the substring matched by the group is accessible via the symbolic group name *name*. Group names must be valid - Python identifiers, and in bytes patterns they must contain only characters - in the ASCII range. Each group name must be defined only once within a - regular expression. A symbolic group is also a numbered group, just as if + Python identifiers, and in :class:`bytes` patterns they can only contain + bytes in the ASCII range. Each group name must be defined only once within + a regular expression. A symbolic group is also a numbered group, just as if the group were not named. Named groups can be referenced in three contexts. If the pattern is @@ -419,8 +419,8 @@ The special characters are: +---------------------------------------+----------------------------------+ .. versionchanged:: 3.12 - In bytes patterns group names must contain only characters in - the ASCII range. + In :class:`bytes` patterns, group *name* can only contain bytes + in the ASCII range (``b'\x00'``-``b'\x7f'``). .. index:: single: (?P=; in regular expressions @@ -496,6 +496,8 @@ The special characters are: .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. + In :class:`bytes` patterns, group *name* can only contain bytes + in the ASCII range (``b'\x00'``-``b'\x7f'``). The special sequences consist of ``'\'`` and a character from the list below. @@ -1018,8 +1020,8 @@ Functions .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. - In bytes replacement strings group names must contain only characters - in the ASCII range. + In :class:`bytes` replacement strings, group *name* can only contain bytes + in the ASCII range (``b'\x00'``-``b'\x7f'``). .. function:: subn(pattern, repl, string, count=0, flags=0) From webhook-mailer at python.org Mon Dec 26 01:23:18 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Mon, 26 Dec 2022 06:23:18 -0000 Subject: [Python-checkins] gh-92446: Improve argparse choices docs; revert bad change to lzma docs (#94627) Message-ID: <mailman.3088.1672035799.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ad3c99e521151680afc65d3f8a7d2167ec1969ad commit: ad3c99e521151680afc65d3f8a7d2167ec1969ad branch: main author: Guy Yagev <yourlefthandman8 at gmail.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-26T00:22:53-06:00 summary: gh-92446: Improve argparse choices docs; revert bad change to lzma docs (#94627) Based on the definition of the collections.abc classes, it is more accurate to use "sequence" instead of "container" when describing argparse choices. A previous attempt at fixing this in #92450 was mistaken; this PR reverts that change. Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/argparse.rst M Doc/library/lzma.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index e6c964864925..475cac70291e 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -765,7 +765,7 @@ The add_argument() method * type_ - The type to which the command-line argument should be converted. - * choices_ - A container of the allowable values for the argument. + * choices_ - A sequence of the allowable values for the argument. * required_ - Whether or not the command-line option may be omitted (optionals only). @@ -1209,7 +1209,7 @@ choices ^^^^^^^ Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword +These can be handled by passing a sequence object as the *choices* keyword argument to :meth:`~ArgumentParser.add_argument`. When the command line is parsed, argument values will be checked, and an error message will be displayed if the argument was not one of the acceptable values:: @@ -1223,9 +1223,9 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* container is checked after any type_ +Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -container should match the type_ specified:: +sequence should match the type_ specified:: >>> parser = argparse.ArgumentParser(prog='doors.py') >>> parser.add_argument('door', type=int, choices=range(1, 4)) @@ -1235,8 +1235,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any container can be passed as the *choices* value, so :class:`list` objects, -:class:`set` objects, and custom containers are all supported. +Any sequence can be passed as the *choices* value, so :class:`list` objects, +:class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index a9311f2a0356..868d4dcfb6c9 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -147,7 +147,7 @@ Compressing and decompressing data in memory This format is more limited than ``.xz`` -- it does not support integrity checks or multiple filters. - * :const:`FORMAT_RAW`: A raw data stream, not using sequences format. + * :const:`FORMAT_RAW`: A raw data stream, not using any container format. This format specifier does not support integrity checks, and requires that you always specify a custom filter chain (for both compression and decompression). Additionally, data compressed in this manner cannot be From webhook-mailer at python.org Mon Dec 26 02:09:11 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Mon, 26 Dec 2022 07:09:11 -0000 Subject: [Python-checkins] [3.11] gh-92446: Improve argparse choices docs (GH-94627) (#100528) Message-ID: <mailman.3089.1672038553.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2cb4b13af6ac975b1a98ddc7d117c3b25398ccf0 commit: 2cb4b13af6ac975b1a98ddc7d117c3b25398ccf0 branch: 3.11 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-26T01:09:06-06:00 summary: [3.11] gh-92446: Improve argparse choices docs (GH-94627) (#100528) Based on the definition of the collections.abc classes, it is more accurate to use "sequence" instead of "container" when describing argparse choices. (cherry picked from commit ad3c99e521) Co-authored-by: Guy Yagev <yourlefthandman8 at gmail.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/argparse.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 67ca6e77bc00..eeb65160bbc8 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -755,7 +755,7 @@ The add_argument() method * type_ - The type to which the command-line argument should be converted. - * choices_ - A container of the allowable values for the argument. + * choices_ - A sequence of the allowable values for the argument. * required_ - Whether or not the command-line option may be omitted (optionals only). @@ -1199,7 +1199,7 @@ choices ^^^^^^^ Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword +These can be handled by passing a sequence object as the *choices* keyword argument to :meth:`~ArgumentParser.add_argument`. When the command line is parsed, argument values will be checked, and an error message will be displayed if the argument was not one of the acceptable values:: @@ -1213,9 +1213,9 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* container is checked after any type_ +Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -container should match the type_ specified:: +sequence should match the type_ specified:: >>> parser = argparse.ArgumentParser(prog='doors.py') >>> parser.add_argument('door', type=int, choices=range(1, 4)) @@ -1225,8 +1225,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any container can be passed as the *choices* value, so :class:`list` objects, -:class:`set` objects, and custom containers are all supported. +Any sequence can be passed as the *choices* value, so :class:`list` objects, +:class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. From webhook-mailer at python.org Mon Dec 26 02:10:54 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Mon, 26 Dec 2022 07:10:54 -0000 Subject: [Python-checkins] [3.10] gh-92446: Improve argparse choices docs (GH-94627) (#100529) Message-ID: <mailman.3090.1672038654.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a167365d8585c373ff3bdff0d48bffb7e74581e5 commit: a167365d8585c373ff3bdff0d48bffb7e74581e5 branch: 3.10 author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-26T01:10:48-06:00 summary: [3.10] gh-92446: Improve argparse choices docs (GH-94627) (#100529) Based on the definition of the collections.abc classes, it is more accurate to use "sequence" instead of "container" when describing argparse choices. (cherry picked from commit ad3c99e521) Co-authored-by: Guy Yagev <yourlefthandman8 at gmail.com> Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/argparse.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index bdc2ccebb7bb..3971c000b2a0 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -702,7 +702,7 @@ The add_argument() method * type_ - The type to which the command-line argument should be converted. - * choices_ - A container of the allowable values for the argument. + * choices_ - A sequence of the allowable values for the argument. * required_ - Whether or not the command-line option may be omitted (optionals only). @@ -1124,7 +1124,7 @@ choices ^^^^^^^ Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword +These can be handled by passing a sequence object as the *choices* keyword argument to :meth:`~ArgumentParser.add_argument`. When the command line is parsed, argument values will be checked, and an error message will be displayed if the argument was not one of the acceptable values:: @@ -1138,9 +1138,9 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* container is checked after any type_ +Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -container should match the type_ specified:: +sequence should match the type_ specified:: >>> parser = argparse.ArgumentParser(prog='doors.py') >>> parser.add_argument('door', type=int, choices=range(1, 4)) @@ -1150,8 +1150,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any container can be passed as the *choices* value, so :class:`list` objects, -:class:`set` objects, and custom containers are all supported. +Any sequence can be passed as the *choices* value, so :class:`list` objects, +:class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. From webhook-mailer at python.org Mon Dec 26 06:34:24 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 11:34:24 -0000 Subject: [Python-checkins] Fix name of removed `inspect.Signature.from_builtin` method in 3.11.0a2 changelog (#100525) Message-ID: <mailman.3091.1672054465.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3ccc98fc24b278e0f8195686f3651c7c9fabeb59 commit: 3ccc98fc24b278e0f8195686f3651c7c9fabeb59 branch: main author: Jakub Kuczys <me at jacken.men> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T17:04:18+05:30 summary: Fix name of removed `inspect.Signature.from_builtin` method in 3.11.0a2 changelog (#100525) files: M Misc/NEWS.d/3.11.0a2.rst diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index 8ae8847d846b..225bd61e90d4 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -618,7 +618,7 @@ Removed from the :mod:`inspect` module: use the :func:`inspect.signature` function and :class:`Signature` object directly. -* the undocumented ``Signature.from_callable`` and ``Signature.from_function`` +* the undocumented ``Signature.from_builtin`` and ``Signature.from_function`` functions, deprecated since Python 3.5; use the :meth:`Signature.from_callable() <inspect.Signature.from_callable>` method instead. From webhook-mailer at python.org Mon Dec 26 06:38:54 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 11:38:54 -0000 Subject: [Python-checkins] gh-100520: Fix `rst` markup in `configparser` docstrings (#100524) Message-ID: <mailman.3092.1672054735.3313.python-checkins@python.org> https://github.com/python/cpython/commit/199507b81a302ea19f93593965b1e5088195a6c5 commit: 199507b81a302ea19f93593965b1e5088195a6c5 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T17:08:49+05:30 summary: gh-100520: Fix `rst` markup in `configparser` docstrings (#100524) files: M Lib/configparser.py diff --git a/Lib/configparser.py b/Lib/configparser.py index f98e6fb01f97..dee5a0db7e7d 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -19,36 +19,37 @@ inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=<unset>, converters=<unset>): - Create the parser. When `defaults' is given, it is initialized into the + + Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. - When `dict_type' is given, it will be used to create the dictionary + When `dict_type` is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. - When `delimiters' is given, it will be used as the set of substrings + When `delimiters` is given, it will be used as the set of substrings that divide keys from values. - When `comment_prefixes' is given, it will be used as the set of + When `comment_prefixes` is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. - When `inline_comment_prefixes' is given, it will be used as the set of + When `inline_comment_prefixes` is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. - When `empty_lines_in_values' is False (default: True), each empty line + When `empty_lines_in_values` is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. - When `allow_no_value' is True (default: False), options without + When `allow_no_value` is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is + When `default_section` is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` @@ -87,7 +88,7 @@ read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `<???>' is used). + messages (if f has no `name` attribute, the string `<???>` is used). read_string(string) Read configuration from a given string. @@ -103,9 +104,9 @@ Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. + provided using the `vars` argument, which must be a dictionary whose + contents override any pre-existing defaults. If `option` is a key in + `vars`, the value from `vars` is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. @@ -134,7 +135,7 @@ write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters + `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. """ @@ -323,7 +324,7 @@ def __init__(self, filename, lineno, line): # Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as +# option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. _UNSET = object() @@ -357,7 +358,7 @@ class BasicInterpolation(Interpolation): would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" + is considered a user error and raises `InterpolationSyntaxError`.""" _KEYCRE = re.compile(r"%$([^)]+)$s") @@ -418,7 +419,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" + `zc.buildout`. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") @@ -691,10 +692,10 @@ def read(self, filenames, encoding=None): def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. - The `f' argument must be iterable, returning one line at a time. - Optional second argument is the `source' specifying the name of the - file being read. If not given, it is taken from f.name. If `f' has no - `name' attribute, `<???>' is used. + The `f` argument must be iterable, returning one line at a time. + Optional second argument is the `source` specifying the name of the + file being read. If not given, it is taken from f.name. If `f` has no + `name` attribute, `<???>` is used. """ if source is None: try: @@ -718,7 +719,7 @@ def read_dict(self, dictionary, source='<dict>'): All types held in the dictionary are converted to strings during reading, including section names, option names and keys. - Optional second argument is the `source' specifying the name of the + Optional second argument is the `source` specifying the name of the dictionary being read. """ elements_added = set() @@ -742,15 +743,15 @@ def read_dict(self, dictionary, source='<dict>'): def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. - If `vars' is provided, it must be a dictionary. The option is looked up - in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. - If the key is not found and `fallback' is provided, it is used as - a fallback value. `None' can be provided as a `fallback' value. + If `vars` is provided, it must be a dictionary. The option is looked up + in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order. + If the key is not found and `fallback` is provided, it is used as + a fallback value. `None` can be provided as a `fallback` value. - If interpolation is enabled and the optional argument `raw' is False, + If interpolation is enabled and the optional argument `raw` is False, all interpolations are expanded in the return values. - Arguments `raw', `vars', and `fallback' are keyword only. + Arguments `raw`, `vars`, and `fallback` are keyword only. The section DEFAULT is special. """ @@ -810,8 +811,8 @@ def items(self, section=_UNSET, raw=False, vars=None): All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument - `raw' is true. Additional substitutions may be provided using the - `vars' argument, which must be a dictionary whose contents overrides + `raw` is true. Additional substitutions may be provided using the + `vars` argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. @@ -853,8 +854,8 @@ def optionxform(self, optionstr): def has_option(self, section, option): """Check for the existence of a given option in a given section. - If the specified `section' is None or an empty string, DEFAULT is - assumed. If the specified `section' does not exist, returns False.""" + If the specified `section` is None or an empty string, DEFAULT is + assumed. If the specified `section` does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults @@ -882,7 +883,7 @@ def set(self, section, option, value=None): def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. - If `space_around_delimiters' is True (the default), delimiters + If `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. Please note that comments in the original configuration file are not @@ -900,7 +901,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified `fp`.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, @@ -974,8 +975,8 @@ def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by - a name in square brackets (`[]'), plus key/value options, indicated by - `name' and `value' delimited with a specific substring (`=' or `:' by + a name in square brackets (`[]`), plus key/value options, indicated by + `name` and `value` delimited with a specific substring (`=` or `:` by default). Values can span multiple lines, as long as they are indented deeper @@ -983,7 +984,7 @@ def _read(self, fp, fpname): lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific - characters (`#' and `;' by default). Comments may appear on their own + characters (`#` and `;` by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ From webhook-mailer at python.org Mon Dec 26 10:43:01 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 15:43:01 -0000 Subject: [Python-checkins] [3.10] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100534) Message-ID: <mailman.3093.1672069382.3313.python-checkins@python.org> https://github.com/python/cpython/commit/bb159b448170b24756db9ddfde0e0a735489fce6 commit: bb159b448170b24756db9ddfde0e0a735489fce6 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T21:12:56+05:30 summary: [3.10] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100534) gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (cherry picked from commit 199507b81a302ea19f93593965b1e5088195a6c5) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/configparser.py diff --git a/Lib/configparser.py b/Lib/configparser.py index 3470624e63f6..785527082d07 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -19,36 +19,37 @@ inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=<unset>, converters=<unset>): - Create the parser. When `defaults' is given, it is initialized into the + + Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. - When `dict_type' is given, it will be used to create the dictionary + When `dict_type` is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. - When `delimiters' is given, it will be used as the set of substrings + When `delimiters` is given, it will be used as the set of substrings that divide keys from values. - When `comment_prefixes' is given, it will be used as the set of + When `comment_prefixes` is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. - When `inline_comment_prefixes' is given, it will be used as the set of + When `inline_comment_prefixes` is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. - When `empty_lines_in_values' is False (default: True), each empty line + When `empty_lines_in_values` is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. - When `allow_no_value' is True (default: False), options without + When `allow_no_value` is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is + When `default_section` is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` @@ -87,7 +88,7 @@ read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `<???>' is used). + messages (if f has no `name` attribute, the string `<???>` is used). read_string(string) Read configuration from a given string. @@ -103,9 +104,9 @@ Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. + provided using the `vars` argument, which must be a dictionary whose + contents override any pre-existing defaults. If `option` is a key in + `vars`, the value from `vars` is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. @@ -134,7 +135,7 @@ write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters + `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. """ @@ -352,7 +353,7 @@ def __init__(self, filename, lineno, line): # Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as +# option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. _UNSET = object() @@ -386,7 +387,7 @@ class BasicInterpolation(Interpolation): would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" + is considered a user error and raises `InterpolationSyntaxError`.""" _KEYCRE = re.compile(r"%$([^)]+)$s") @@ -447,7 +448,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" + `zc.buildout`. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") @@ -706,10 +707,10 @@ def read(self, filenames, encoding=None): def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. - The `f' argument must be iterable, returning one line at a time. - Optional second argument is the `source' specifying the name of the - file being read. If not given, it is taken from f.name. If `f' has no - `name' attribute, `<???>' is used. + The `f` argument must be iterable, returning one line at a time. + Optional second argument is the `source` specifying the name of the + file being read. If not given, it is taken from f.name. If `f` has no + `name` attribute, `<???>` is used. """ if source is None: try: @@ -733,7 +734,7 @@ def read_dict(self, dictionary, source='<dict>'): All types held in the dictionary are converted to strings during reading, including section names, option names and keys. - Optional second argument is the `source' specifying the name of the + Optional second argument is the `source` specifying the name of the dictionary being read. """ elements_added = set() @@ -766,15 +767,15 @@ def readfp(self, fp, filename=None): def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. - If `vars' is provided, it must be a dictionary. The option is looked up - in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. - If the key is not found and `fallback' is provided, it is used as - a fallback value. `None' can be provided as a `fallback' value. + If `vars` is provided, it must be a dictionary. The option is looked up + in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order. + If the key is not found and `fallback` is provided, it is used as + a fallback value. `None` can be provided as a `fallback` value. - If interpolation is enabled and the optional argument `raw' is False, + If interpolation is enabled and the optional argument `raw` is False, all interpolations are expanded in the return values. - Arguments `raw', `vars', and `fallback' are keyword only. + Arguments `raw`, `vars`, and `fallback` are keyword only. The section DEFAULT is special. """ @@ -834,8 +835,8 @@ def items(self, section=_UNSET, raw=False, vars=None): All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument - `raw' is true. Additional substitutions may be provided using the - `vars' argument, which must be a dictionary whose contents overrides + `raw` is true. Additional substitutions may be provided using the + `vars` argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. @@ -877,8 +878,8 @@ def optionxform(self, optionstr): def has_option(self, section, option): """Check for the existence of a given option in a given section. - If the specified `section' is None or an empty string, DEFAULT is - assumed. If the specified `section' does not exist, returns False.""" + If the specified `section` is None or an empty string, DEFAULT is + assumed. If the specified `section` does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults @@ -906,7 +907,7 @@ def set(self, section, option, value=None): def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. - If `space_around_delimiters' is True (the default), delimiters + If `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. Please note that comments in the original configuration file are not @@ -924,7 +925,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified `fp`.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, @@ -998,8 +999,8 @@ def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by - a name in square brackets (`[]'), plus key/value options, indicated by - `name' and `value' delimited with a specific substring (`=' or `:' by + a name in square brackets (`[]`), plus key/value options, indicated by + `name` and `value` delimited with a specific substring (`=` or `:` by default). Values can span multiple lines, as long as they are indented deeper @@ -1007,7 +1008,7 @@ def _read(self, fp, fpname): lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific - characters (`#' and `;' by default). Comments may appear on their own + characters (`#` and `;` by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ From webhook-mailer at python.org Mon Dec 26 10:43:17 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Mon, 26 Dec 2022 15:43:17 -0000 Subject: [Python-checkins] [3.11] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100533) Message-ID: <mailman.3094.1672069397.3313.python-checkins@python.org> https://github.com/python/cpython/commit/18b43cf95f2455b63a6dbefb813f48fa85a07c7b commit: 18b43cf95f2455b63a6dbefb813f48fa85a07c7b branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-26T21:13:11+05:30 summary: [3.11] gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (#100533) gh-100520: Fix `rst` markup in `configparser` docstrings (GH-100524) (cherry picked from commit 199507b81a302ea19f93593965b1e5088195a6c5) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/configparser.py diff --git a/Lib/configparser.py b/Lib/configparser.py index de9ee537ac18..df2d7e335d9d 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -19,36 +19,37 @@ inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=<unset>, converters=<unset>): - Create the parser. When `defaults' is given, it is initialized into the + + Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. - When `dict_type' is given, it will be used to create the dictionary + When `dict_type` is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. - When `delimiters' is given, it will be used as the set of substrings + When `delimiters` is given, it will be used as the set of substrings that divide keys from values. - When `comment_prefixes' is given, it will be used as the set of + When `comment_prefixes` is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. - When `inline_comment_prefixes' is given, it will be used as the set of + When `inline_comment_prefixes` is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. - When `empty_lines_in_values' is False (default: True), each empty line + When `empty_lines_in_values` is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. - When `allow_no_value' is True (default: False), options without + When `allow_no_value` is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is + When `default_section` is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` @@ -87,7 +88,7 @@ read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `<???>' is used). + messages (if f has no `name` attribute, the string `<???>` is used). read_string(string) Read configuration from a given string. @@ -103,9 +104,9 @@ Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. + provided using the `vars` argument, which must be a dictionary whose + contents override any pre-existing defaults. If `option` is a key in + `vars`, the value from `vars` is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. @@ -134,7 +135,7 @@ write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters + `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. """ @@ -352,7 +353,7 @@ def __init__(self, filename, lineno, line): # Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as +# option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. _UNSET = object() @@ -386,7 +387,7 @@ class BasicInterpolation(Interpolation): would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" + is considered a user error and raises `InterpolationSyntaxError`.""" _KEYCRE = re.compile(r"%$([^)]+)$s") @@ -447,7 +448,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" + `zc.buildout`. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") @@ -720,10 +721,10 @@ def read(self, filenames, encoding=None): def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. - The `f' argument must be iterable, returning one line at a time. - Optional second argument is the `source' specifying the name of the - file being read. If not given, it is taken from f.name. If `f' has no - `name' attribute, `<???>' is used. + The `f` argument must be iterable, returning one line at a time. + Optional second argument is the `source` specifying the name of the + file being read. If not given, it is taken from f.name. If `f` has no + `name` attribute, `<???>` is used. """ if source is None: try: @@ -747,7 +748,7 @@ def read_dict(self, dictionary, source='<dict>'): All types held in the dictionary are converted to strings during reading, including section names, option names and keys. - Optional second argument is the `source' specifying the name of the + Optional second argument is the `source` specifying the name of the dictionary being read. """ elements_added = set() @@ -780,15 +781,15 @@ def readfp(self, fp, filename=None): def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. - If `vars' is provided, it must be a dictionary. The option is looked up - in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. - If the key is not found and `fallback' is provided, it is used as - a fallback value. `None' can be provided as a `fallback' value. + If `vars` is provided, it must be a dictionary. The option is looked up + in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order. + If the key is not found and `fallback` is provided, it is used as + a fallback value. `None` can be provided as a `fallback` value. - If interpolation is enabled and the optional argument `raw' is False, + If interpolation is enabled and the optional argument `raw` is False, all interpolations are expanded in the return values. - Arguments `raw', `vars', and `fallback' are keyword only. + Arguments `raw`, `vars`, and `fallback` are keyword only. The section DEFAULT is special. """ @@ -848,8 +849,8 @@ def items(self, section=_UNSET, raw=False, vars=None): All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument - `raw' is true. Additional substitutions may be provided using the - `vars' argument, which must be a dictionary whose contents overrides + `raw` is true. Additional substitutions may be provided using the + `vars` argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. @@ -891,8 +892,8 @@ def optionxform(self, optionstr): def has_option(self, section, option): """Check for the existence of a given option in a given section. - If the specified `section' is None or an empty string, DEFAULT is - assumed. If the specified `section' does not exist, returns False.""" + If the specified `section` is None or an empty string, DEFAULT is + assumed. If the specified `section` does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults @@ -920,7 +921,7 @@ def set(self, section, option, value=None): def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. - If `space_around_delimiters' is True (the default), delimiters + If `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. Please note that comments in the original configuration file are not @@ -938,7 +939,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified `fp`.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, @@ -1012,8 +1013,8 @@ def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by - a name in square brackets (`[]'), plus key/value options, indicated by - `name' and `value' delimited with a specific substring (`=' or `:' by + a name in square brackets (`[]`), plus key/value options, indicated by + `name` and `value` delimited with a specific substring (`=` or `:` by default). Values can span multiple lines, as long as they are indented deeper @@ -1021,7 +1022,7 @@ def _read(self, fp, fpname): lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific - characters (`#' and `;' by default). Comments may appear on their own + characters (`#` and `;` by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ From webhook-mailer at python.org Mon Dec 26 23:44:59 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Tue, 27 Dec 2022 04:44:59 -0000 Subject: [Python-checkins] [3.10] gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) (#100509) Message-ID: <mailman.3095.1672116301.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1ffc67265f1a622751069997e7ca8193f983e1e2 commit: 1ffc67265f1a622751069997e7ca8193f983e1e2 branch: 3.10 author: Jelle Zijlstra <jelle.zijlstra at gmail.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-26T20:44:53-08:00 summary: [3.10] gh-99535: Add test for inheritance of annotations and update documentation (GH-99990) (#100509) (cherry picked from commit f5b7b19bf10724d831285fb04e00f763838bd555) Co-authored-by: MonadChains <monadchains at gmail.com> files: M Doc/howto/annotations.rst M Doc/library/typing.rst M Lib/test/test_grammar.py diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 2bc2f2d4c839..472069032d65 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -57,6 +57,12 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer newer is to call :func:`getattr` with three arguments, for example ``getattr(o, '__annotations__', None)``. + Before Python 3.10, accessing ``__annotations__`` on a class that + defines no annotations but that has a parent class with + annotations would return the parent's ``__annotations__``. + In Python 3.10 and newer, the child class's annotations + will be an empty dict instead. + Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index d415f149027f..d3275209e938 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2233,6 +2233,10 @@ Introspection helpers .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. + .. versionchanged:: 3.10 + Calling ``get_type_hints()`` on a class no longer returns the annotations + of its base classes. + .. function:: get_args(tp) .. function:: get_origin(tp) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 2b9f7df0bd25..27b14ab836b6 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -406,6 +406,28 @@ class Cbad2(C): x: int x.y: list = [] + def test_annotations_inheritance(self): + # Check that annotations are not inherited by derived classes + class A: + attr: int + class B(A): + pass + class C(A): + attr: str + class D: + attr2: int + class E(A, D): + pass + class F(C, A): + pass + self.assertEqual(A.__annotations__, {"attr": int}) + self.assertEqual(B.__annotations__, {}) + self.assertEqual(C.__annotations__, {"attr" : str}) + self.assertEqual(D.__annotations__, {"attr2" : int}) + self.assertEqual(E.__annotations__, {}) + self.assertEqual(F.__annotations__, {}) + + def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod From webhook-mailer at python.org Mon Dec 26 23:51:00 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Tue, 27 Dec 2022 04:51:00 -0000 Subject: [Python-checkins] gh-99509: Add `__class_getitem__` to `multiprocessing.queues.Queue` (#99511) Message-ID: <mailman.3096.1672116661.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ce39aaffeef9aa8af54a8554fe7a5609a6bba471 commit: ce39aaffeef9aa8af54a8554fe7a5609a6bba471 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-26T20:50:55-08:00 summary: gh-99509: Add `__class_getitem__` to `multiprocessing.queues.Queue` (#99511) files: A Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst M Lib/multiprocessing/queues.py M Lib/test/test_genericalias.py diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index f37f114a9688..daf9ee94a194 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -280,6 +280,8 @@ def _on_queue_feeder_error(e, obj): import traceback traceback.print_exc() + __class_getitem__ = classmethod(types.GenericAlias) + _sentinel = object() diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 6d0a556b1f7f..9b59d1e3e0aa 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -31,11 +31,15 @@ from multiprocessing.managers import ValueProxy from multiprocessing.pool import ApplyResult from multiprocessing.queues import SimpleQueue as MPSimpleQueue + from multiprocessing.queues import Queue as MPQueue + from multiprocessing.queues import JoinableQueue as MPJoinableQueue except ImportError: # _multiprocessing module is optional ValueProxy = None ApplyResult = None MPSimpleQueue = None + MPQueue = None + MPJoinableQueue = None try: from multiprocessing.shared_memory import ShareableList except ImportError: @@ -130,7 +134,8 @@ class BaseTest(unittest.TestCase): if ctypes is not None: generic_types.extend((ctypes.Array, ctypes.LibraryLoader)) if ValueProxy is not None: - generic_types.extend((ValueProxy, ApplyResult, MPSimpleQueue)) + generic_types.extend((ValueProxy, ApplyResult, + MPSimpleQueue, MPQueue, MPJoinableQueue)) def test_subscriptable(self): for t in self.generic_types: diff --git a/Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst b/Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst new file mode 100644 index 000000000000..634281061cec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-15-18-45-01.gh-issue-99509.FLK0xU.rst @@ -0,0 +1 @@ +Add :pep:`585` support for :class:`multiprocessing.queues.Queue`. From webhook-mailer at python.org Tue Dec 27 05:56:00 2022 From: webhook-mailer at python.org (corona10) Date: Tue, 27 Dec 2022 10:56:00 -0000 Subject: [Python-checkins] gh-94603: micro optimize list.pop (gh-94604) Message-ID: <mailman.3097.1672138562.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b3da6989524001d707ebb386bfab998452aa57bb commit: b3da6989524001d707ebb386bfab998452aa57bb branch: main author: Pieter Eendebak <pieter.eendebak at gmail.com> committer: corona10 <donghee.na92 at gmail.com> date: 2022-12-27T19:55:54+09:00 summary: gh-94603: micro optimize list.pop (gh-94604) files: A Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst M Objects/listobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst new file mode 100644 index 000000000000..de4fe4d6df8c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-06-18-44-00.gh-issue-94603.Q_03xV.rst @@ -0,0 +1 @@ +Improve performance of ``list.pop`` for small lists. diff --git a/Objects/listobject.c b/Objects/listobject.c index 1d32915b17a1..b093f88a35fc 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1022,21 +1022,29 @@ list_pop_impl(PyListObject *self, Py_ssize_t index) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = self->ob_item[index]; - if (index == Py_SIZE(self) - 1) { - status = list_resize(self, Py_SIZE(self) - 1); - if (status >= 0) - return v; /* and v now owns the reference the list had */ - else - return NULL; + + PyObject **items = self->ob_item; + v = items[index]; + const Py_ssize_t size_after_pop = Py_SIZE(self) - 1; + if (size_after_pop == 0) { + Py_INCREF(v); + status = _list_clear(self); } - Py_INCREF(v); - status = list_ass_slice(self, index, index+1, (PyObject *)NULL); - if (status < 0) { - Py_DECREF(v); + else { + if ((size_after_pop - index) > 0) { + memmove(&items[index], &items[index+1], (size_after_pop - index) * sizeof(PyObject *)); + } + status = list_resize(self, size_after_pop); + } + if (status >= 0) { + return v; // and v now owns the reference the list had + } + else { + // list resize failed, need to restore + memmove(&items[index+1], &items[index], (size_after_pop - index)* sizeof(PyObject *)); + items[index] = v; return NULL; } - return v; } /* Reverse a slice of a list in place, from lo up to (exclusive) hi. */ From webhook-mailer at python.org Tue Dec 27 11:11:48 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Tue, 27 Dec 2022 16:11:48 -0000 Subject: [Python-checkins] Remove `NoneType` redefinition from `clinic.py` (#100551) Message-ID: <mailman.3098.1672157509.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b0ea28913e3bf684ef847a71afcdfa8224bab63d commit: b0ea28913e3bf684ef847a71afcdfa8224bab63d branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-27T21:41:39+05:30 summary: Remove `NoneType` redefinition from `clinic.py` (#100551) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index fdf8041e14bb..2fb1902a5b54 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -27,7 +27,6 @@ import types from types import * -NoneType = type(None) # TODO: # @@ -42,7 +41,6 @@ version = '1' -NoneType = type(None) NO_VARARG = "PY_SSIZE_T_MAX" CLINIC_PREFIX = "__clinic_" CLINIC_PREFIXED_ARGS = {"args"} From webhook-mailer at python.org Tue Dec 27 19:58:10 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 28 Dec 2022 00:58:10 -0000 Subject: [Python-checkins] gh-100553: Improve accuracy of sqlite3.Row iter test (#100555) Message-ID: <mailman.3099.1672189092.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3dc48dabd48864039951715816e07986a4828d80 commit: 3dc48dabd48864039951715816e07986a4828d80 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-28T01:58:05+01:00 summary: gh-100553: Improve accuracy of sqlite3.Row iter test (#100555) files: M Lib/test/test_sqlite3/test_factory.py diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 7fdc45ab6924..7c36135ecadc 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -179,8 +179,14 @@ def test_sqlite_row_iter(self): """Checks if the row object is iterable""" self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): """Checks if the row object can be converted to a tuple""" From webhook-mailer at python.org Tue Dec 27 20:10:12 2022 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 28 Dec 2022 01:10:12 -0000 Subject: [Python-checkins] [3.11] gh-64490: Fix bugs in argument clinic varargs processing (GH-32092) (#100368) Message-ID: <mailman.3100.1672189814.3313.python-checkins@python.org> https://github.com/python/cpython/commit/a3dbd4c70e93260432fc024265fdf9222926fdad commit: a3dbd4c70e93260432fc024265fdf9222926fdad branch: 3.11 author: colorfulappl <colorfulappl at qq.com> committer: erlend-aasland <erlend.aasland at protonmail.com> date: 2022-12-28T02:10:06+01:00 summary: [3.11] gh-64490: Fix bugs in argument clinic varargs processing (GH-32092) (#100368) (cherry picked from commit 0da728387c99fe6c127b070f2d250dc5bdd62ee5) files: A Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst M Lib/test/clinic.test M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic.c.h M Python/getargs.c M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 991e82d4f966..217aa4c7e97a 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3368,7 +3368,6 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject static const char * const _keywords[] = {"a", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "test_vararg", 0}; PyObject *argsbuf[2]; - Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; @@ -3387,7 +3386,7 @@ exit: static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=a2baf8c1fade41d2 input=81d33815ad1bae6e]*/ +/*[clinic end generated code: output=ce9334333757f6ea input=81d33815ad1bae6e]*/ /*[clinic input] test_vararg_with_default @@ -3418,7 +3417,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar static const char * const _keywords[] = {"a", "b", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "test_vararg_with_default", 0}; PyObject *argsbuf[3]; - Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; int b = 0; @@ -3447,7 +3446,7 @@ exit: static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=3821d282c29f8616 input=6e110b54acd9b22d]*/ +/*[clinic end generated code: output=32fb19dd6bcf9185 input=6e110b54acd9b22d]*/ /*[clinic input] test_vararg_with_only_defaults diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 3f6cc6069669..a84d24a79844 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -730,6 +730,15 @@ def test_parameters_not_permitted_after_slash_for_now(self): x: int """) + def test_parameters_no_more_than_one_vararg(self): + s = self.parse_function_should_fail(""" +module foo +foo.bar + *vararg1: object + *vararg2: object +""") + self.assertEqual(s, "Error on line 0:\nToo many var args\n") + def test_function_not_at_column_0(self): function = self.parse_function(""" module foo @@ -1222,6 +1231,13 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + def test_posonly_vararg(self): + with self.assertRaises(TypeError): + ac_tester.posonly_vararg() + self.assertEqual(ac_tester.posonly_vararg(1, 2), (1, 2, ())) + self.assertEqual(ac_tester.posonly_vararg(1, b=2), (1, 2, ())) + self.assertEqual(ac_tester.posonly_vararg(1, 2, 3, 4), (1, 2, (3, 4))) + def test_vararg_and_posonly(self): with self.assertRaises(TypeError): ac_tester.vararg_and_posonly() @@ -1229,6 +1245,33 @@ def test_vararg_and_posonly(self): ac_tester.vararg_and_posonly(1, b=2) self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) + def test_vararg(self): + with self.assertRaises(TypeError): + ac_tester.vararg() + with self.assertRaises(TypeError): + ac_tester.vararg(1, b=2) + self.assertEqual(ac_tester.vararg(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_vararg_with_default(self): + with self.assertRaises(TypeError): + ac_tester.vararg_with_default() + self.assertEqual(ac_tester.vararg_with_default(1, b=False), (1, (), False)) + self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4, b=True), (1, (2, 3, 4), True)) + + def test_vararg_with_only_defaults(self): + self.assertEqual(ac_tester.vararg_with_only_defaults(), ((), None)) + self.assertEqual(ac_tester.vararg_with_only_defaults(b=2), ((), 2)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, b=2), ((1, ), 2)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4), ((1, 2, 3, 4), None)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + + def test_gh_32092_oob(self): + ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) + + def test_gh_32092_kw_pass(self): + ac_tester.gh_32092_kw_pass(1, 2, 3) + def test_gh_99233_refcount(self): arg = '*A unique string is not referenced by anywhere else.*' arg_refcount_origin = sys.getrefcount(arg) @@ -1241,5 +1284,6 @@ def test_gh_99240_double_free(self): with self.assertRaisesRegex(TypeError, expected_error): ac_tester.gh_99240_double_free('a', '\0b') + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst new file mode 100644 index 000000000000..4a308a930605 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst @@ -0,0 +1,7 @@ +Argument Clinic varargs bugfixes + +* Fix out-of-bounds error in :c:func:`!_PyArg_UnpackKeywordsWithVararg`. +* Fix incorrect check which allowed more than one varargs in clinic.py. +* Fix miscalculation of ``noptargs`` in generated code. +* Do not generate ``noptargs`` when there is a vararg argument and no optional argument. + diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 56eddfd6fdbf..91fdee24d328 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -950,6 +950,25 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) } +/*[clinic input] +posonly_vararg + + a: object + / + b: object + *args: object + +[clinic start generated code]*/ + +static PyObject * +posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=ee6713acda6b954e input=783427fe7ec2b67a]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + /*[clinic input] vararg_and_posonly @@ -967,6 +986,100 @@ vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) } +/*[clinic input] +vararg + + a: object + *args: object + +[clinic start generated code]*/ + +static PyObject * +vararg_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=91ab7a0efc52dd5e input=02c0f772d05f591e]*/ +{ + return pack_arguments_newref(2, a, args); +} + + +/*[clinic input] +vararg_with_default + + a: object + *args: object + b: bool = False + +[clinic start generated code]*/ + +static PyObject * +vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b) +/*[clinic end generated code: output=182c01035958ce92 input=68cafa6a79f89e36]*/ +{ + PyObject *obj_b = b ? Py_True : Py_False; + return pack_arguments_newref(3, a, args, obj_b); +} + + +/*[clinic input] +vararg_with_only_defaults + + *args: object + b: object = None + +[clinic start generated code]*/ + +static PyObject * +vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b) +/*[clinic end generated code: output=c06b1826d91f2f7b input=678c069bc67550e1]*/ +{ + return pack_arguments_newref(2, args, b); +} + + + +/*[clinic input] +gh_32092_oob + + pos1: object + pos2: object + *varargs: object + kw1: object = None + kw2: object = None + +Proof-of-concept of GH-32092 OOB bug. + +[clinic start generated code]*/ + +static PyObject * +gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, + PyObject *varargs, PyObject *kw1, PyObject *kw2) +/*[clinic end generated code: output=ee259c130054653f input=46d15c881608f8ff]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +gh_32092_kw_pass + + pos: object + *args: object + kw: object = None + +Proof-of-concept of GH-32092 keyword args passing bug. + +[clinic start generated code]*/ + +static PyObject * +gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, + PyObject *kw) +/*[clinic end generated code: output=4a2bbe4f7c8604e9 input=5c0bd5b9079a0cce]*/ +{ + Py_RETURN_NONE; +} + + /*[clinic input] gh_99233_refcount @@ -1046,7 +1159,13 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF + POSONLY_VARARG_METHODDEF VARARG_AND_POSONLY_METHODDEF + VARARG_METHODDEF + VARARG_WITH_DEFAULT_METHODDEF + VARARG_WITH_ONLY_DEFAULTS_METHODDEF + GH_32092_OOB_METHODDEF + GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF {NULL, NULL} diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 222ea783486e..41555d801059 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -1952,6 +1952,43 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(posonly_vararg__doc__, +"posonly_vararg($module, a, /, b, *args)\n" +"--\n" +"\n"); + +#define POSONLY_VARARG_METHODDEF \ + {"posonly_vararg", _PyCFunction_CAST(posonly_vararg), METH_FASTCALL|METH_KEYWORDS, posonly_vararg__doc__}, + +static PyObject * +posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "posonly_vararg", 0}; + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + __clinic_args = args[2]; + return_value = posonly_vararg_impl(module, a, b, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(vararg_and_posonly__doc__, "vararg_and_posonly($module, a, /, *args)\n" "--\n" @@ -1985,6 +2022,219 @@ vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(vararg__doc__, +"vararg($module, /, a, *args)\n" +"--\n" +"\n"); + +#define VARARG_METHODDEF \ + {"vararg", _PyCFunction_CAST(vararg), METH_FASTCALL|METH_KEYWORDS, vararg__doc__}, + +static PyObject * +vararg_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "vararg", 0}; + PyObject *argsbuf[2]; + PyObject *a; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + return_value = vararg_impl(module, a, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(vararg_with_default__doc__, +"vararg_with_default($module, /, a, *args, b=False)\n" +"--\n" +"\n"); + +#define VARARG_WITH_DEFAULT_METHODDEF \ + {"vararg_with_default", _PyCFunction_CAST(vararg_with_default), METH_FASTCALL|METH_KEYWORDS, vararg_with_default__doc__}, + +static PyObject * +vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b); + +static PyObject * +vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "vararg_with_default", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *__clinic_args = NULL; + int b = 0; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = PyObject_IsTrue(args[2]); + if (b < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = vararg_with_default_impl(module, a, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(vararg_with_only_defaults__doc__, +"vararg_with_only_defaults($module, /, *args, b=None)\n" +"--\n" +"\n"); + +#define VARARG_WITH_ONLY_DEFAULTS_METHODDEF \ + {"vararg_with_only_defaults", _PyCFunction_CAST(vararg_with_only_defaults), METH_FASTCALL|METH_KEYWORDS, vararg_with_only_defaults__doc__}, + +static PyObject * +vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b); + +static PyObject * +vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"b", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "vararg_with_only_defaults", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *__clinic_args = NULL; + PyObject *b = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = args[1]; +skip_optional_kwonly: + return_value = vararg_with_only_defaults_impl(module, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(gh_32092_oob__doc__, +"gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" +"--\n" +"\n" +"Proof-of-concept of GH-32092 OOB bug."); + +#define GH_32092_OOB_METHODDEF \ + {"gh_32092_oob", _PyCFunction_CAST(gh_32092_oob), METH_FASTCALL|METH_KEYWORDS, gh_32092_oob__doc__}, + +static PyObject * +gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, + PyObject *varargs, PyObject *kw1, PyObject *kw2); + +static PyObject * +gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"pos1", "pos2", "kw1", "kw2", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "gh_32092_oob", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = Py_MIN(nargs, 2) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *pos1; + PyObject *pos2; + PyObject *varargs = NULL; + PyObject *kw1 = Py_None; + PyObject *kw2 = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); + if (!args) { + goto exit; + } + pos1 = args[0]; + pos2 = args[1]; + varargs = args[2]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + kw1 = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + kw2 = args[4]; +skip_optional_kwonly: + return_value = gh_32092_oob_impl(module, pos1, pos2, varargs, kw1, kw2); + +exit: + Py_XDECREF(varargs); + return return_value; +} + +PyDoc_STRVAR(gh_32092_kw_pass__doc__, +"gh_32092_kw_pass($module, /, pos, *args, kw=None)\n" +"--\n" +"\n" +"Proof-of-concept of GH-32092 keyword args passing bug."); + +#define GH_32092_KW_PASS_METHODDEF \ + {"gh_32092_kw_pass", _PyCFunction_CAST(gh_32092_kw_pass), METH_FASTCALL|METH_KEYWORDS, gh_32092_kw_pass__doc__}, + +static PyObject * +gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, + PyObject *kw); + +static PyObject * +gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"pos", "kw", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "gh_32092_kw_pass", 0}; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *pos; + PyObject *__clinic_args = NULL; + PyObject *kw = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + pos = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + kw = args[2]; +skip_optional_kwonly: + return_value = gh_32092_kw_pass_impl(module, pos, __clinic_args, kw); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(gh_99233_refcount__doc__, "gh_99233_refcount($module, /, *args)\n" "--\n" @@ -2049,4 +2299,4 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=4147b953eebc3c82 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fe398ac790310bc4 input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index 9d6483f4fe0f..3105bd556c1b 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2570,7 +2570,25 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, current_arg = NULL; } - buf[i + vararg + 1] = current_arg; + /* If an arguments is passed in as a keyword argument, + * it should be placed before `buf[vararg]`. + * + * For example: + * def f(a, /, b, *args): + * pass + * f(1, b=2) + * + * This `buf` array should be: [1, 2, NULL]. + * In this case, nargs < vararg. + * + * Otherwise, we leave a place at `buf[vararg]` for vararg tuple + * so the index is `i + 1`. */ + if (nargs < vararg) { + buf[i] = current_arg; + } + else { + buf[i + 1] = current_arg; + } if (current_arg) { --nkwargs; diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c49805ce1a7e..a6d8b86a83ce 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -660,7 +660,7 @@ def output_templates(self, f): vararg = NO_VARARG pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 for i, p in enumerate(parameters, 1): - if p.is_keyword_only() or vararg != NO_VARARG: + if p.is_keyword_only(): assert not p.is_positional_only() if not p.is_optional(): min_kw_only = i - max_pos @@ -957,13 +957,14 @@ def parser_body(prototype, *fields, declarations=''): parser_definition = parser_body(parser_prototype, *parser_code) else: - has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters)) + has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) if vararg == NO_VARARG: args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( min_pos, max_pos, min_kw_only ) + nargs = "nargs" else: args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( min_pos, @@ -971,6 +972,7 @@ def parser_body(prototype, *fields, declarations=''): min_kw_only, vararg ) + nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" parser_prototype = parser_prototype_fastcall_keywords @@ -981,8 +983,7 @@ def parser_body(prototype, *fields, declarations=''): PyObject *argsbuf[%s]; """ % len(converters)) if has_optional_kw: - pre_buffer = "0" if vararg != NO_VARARG else "nargs" - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (pre_buffer, min_pos + min_kw_only) + declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) parser_code = [normalize_snippet(""" args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); if (!args) {{ @@ -1002,7 +1003,7 @@ def parser_body(prototype, *fields, declarations=''): Py_ssize_t nargs = PyTuple_GET_SIZE(args); """ % len(converters)) if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (min_pos + min_kw_only) + declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) parser_code = [normalize_snippet(""" fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); if (!fastargs) {{ From webhook-mailer at python.org Tue Dec 27 20:11:09 2022 From: webhook-mailer at python.org (gvanrossum) Date: Wed, 28 Dec 2022 01:11:09 -0000 Subject: [Python-checkins] GH-98831: Modernize a ton of simpler instructions (#100545) Message-ID: <mailman.3101.1672189869.3313.python-checkins@python.org> https://github.com/python/cpython/commit/08e5594cf3d42391a48e0311f6b9393ec2e00e1e commit: 08e5594cf3d42391a48e0311f6b9393ec2e00e1e branch: main author: Guido van Rossum <guido at python.org> committer: gvanrossum <gvanrossum at gmail.com> date: 2022-12-27T17:11:03-08:00 summary: GH-98831: Modernize a ton of simpler instructions (#100545) * load_const and load_fast aren't families for now * Don't decref unmoved names * Modernize GET_ANEXT * Modernize GET_AWAITABLE * Modernize ASYNC_GEN_WRAP * Modernize YIELD_VALUE * Modernize POP_EXCEPT (in more than one way) * Modernize PREP_RERAISE_STAR * Modernize LOAD_ASSERTION_ERROR * Modernize LOAD_BUILD_CLASS * Modernize STORE_NAME * Modernize LOAD_NAME * Modernize LOAD_CLASSDEREF * Modernize LOAD_DEREF * Modernize STORE_DEREF * Modernize COPY_FREE_VARS (mark it as done) * Modernize LIST_TO_TUPLE * Modernize LIST_EXTEND * Modernize SET_UPDATE * Modernize SETUP_ANNOTATIONS * Modernize DICT_UPDATE * Modernize DICT_MERGE * Modernize MAP_ADD * Modernize IS_OP * Modernize CONTAINS_OP * Modernize CHECK_EXC_MATCH * Modernize IMPORT_NAME * Modernize IMPORT_STAR * Modernize IMPORT_FROM * Modernize JUMP_FORWARD (mark it as done) * Modernize JUMP_BACKWARD (mark it as done) files: M Python/bytecodes.c M Python/generated_cases.c.h M Tools/cases_generator/generate_cases.py diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c0b625bd662c..e1c73ab6b32f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -83,9 +83,11 @@ static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub static PyObject *container, *start, *stop, *v, *lhs, *rhs; static PyObject *list, *tuple, *dict, *owner; static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter; +static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc; +static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from; static size_t jump; // Dummy variables for cache effects -static _Py_CODEUNIT when_to_jump_mask, invert, counter, index, hint; +static uint16_t when_to_jump_mask, invert, counter, index, hint; static uint32_t type_version; // Dummy opcode names for 'op' opcodes #define _COMPARE_OP_FLOAT 1003 @@ -638,12 +640,9 @@ dummy_func( } } - // stack effect: ( -- __0) - inst(GET_ANEXT) { + inst(GET_ANEXT, (aiter -- aiter, awaitable)) { unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyObject *awaitable = NULL; - PyObject *aiter = TOP(); PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { @@ -685,20 +684,17 @@ dummy_func( } } - PUSH(awaitable); PREDICT(LOAD_CONST); } - // stack effect: ( -- ) - inst(GET_AWAITABLE) { - PyObject *iterable = TOP(); - PyObject *iter = _PyCoro_GetAwaitableIter(iterable); + inst(GET_AWAITABLE, (iterable -- iter)) { + iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); + DECREF_INPUTS(); if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -714,11 +710,7 @@ dummy_func( } } - SET_TOP(iter); /* Even if it's NULL */ - - if (iter == NULL) { - goto error; - } + ERROR_IF(iter == NULL, error); PREDICT(LOAD_CONST); } @@ -773,29 +765,22 @@ dummy_func( } } - // stack effect: ( -- ) - inst(ASYNC_GEN_WRAP) { - PyObject *v = TOP(); + inst(ASYNC_GEN_WRAP, (v -- w)) { assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR); - PyObject *w = _PyAsyncGenValueWrapperNew(v); - if (w == NULL) { - goto error; - } - SET_TOP(w); - Py_DECREF(v); + w = _PyAsyncGenValueWrapperNew(v); + DECREF_INPUTS(); + ERROR_IF(w == NULL, error); } - // stack effect: ( -- ) - inst(YIELD_VALUE) { + inst(YIELD_VALUE, (retval --)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. assert(oparg == STACK_LEVEL()); assert(frame != &entry_frame); - PyObject *retval = POP(); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_SetStackPointer(frame, stack_pointer - 1); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; @@ -809,12 +794,9 @@ dummy_func( goto resume_frame; } - // stack effect: (__0 -- ) - inst(POP_EXCEPT) { + inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - PyObject *value = exc_info->exc_value; - exc_info->exc_value = POP(); - Py_XDECREF(value); + Py_XSETREF(exc_info->exc_value, exc_value); } // stack effect: (__0 -- ) @@ -839,21 +821,13 @@ dummy_func( goto exception_unwind; } - // stack effect: (__0 -- ) - inst(PREP_RERAISE_STAR) { - PyObject *excs = POP(); + inst(PREP_RERAISE_STAR, (orig, excs -- val)) { assert(PyList_Check(excs)); - PyObject *orig = POP(); - - PyObject *val = _PyExc_PrepReraiseStar(orig, excs); - Py_DECREF(excs); - Py_DECREF(orig); - if (val == NULL) { - goto error; - } + val = _PyExc_PrepReraiseStar(orig, excs); + DECREF_INPUTS(); - PUSH(val); + ERROR_IF(val == NULL, error); } // stack effect: (__0, __1 -- ) @@ -934,16 +908,11 @@ dummy_func( } } - - // stack effect: ( -- __0) - inst(LOAD_ASSERTION_ERROR) { - PyObject *value = PyExc_AssertionError; - PUSH(Py_NewRef(value)); + inst(LOAD_ASSERTION_ERROR, ( -- value)) { + value = Py_NewRef(PyExc_AssertionError); } - // stack effect: ( -- __0) - inst(LOAD_BUILD_CLASS) { - PyObject *bc; + inst(LOAD_BUILD_CLASS, ( -- bc)) { if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -952,7 +921,7 @@ dummy_func( _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); } - goto error; + ERROR_IF(true, error); } Py_INCREF(bc); } @@ -962,31 +931,27 @@ dummy_func( if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - goto error; + ERROR_IF(true, error); } } - PUSH(bc); } - // stack effect: (__0 -- ) - inst(STORE_NAME) { + inst(STORE_NAME, (v -- )) { PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err != 0) - goto error; + DECREF_INPUTS(); + ERROR_IF(err, error); } inst(DELETE_NAME, (--)) { @@ -1139,11 +1104,9 @@ dummy_func( } } - // stack effect: ( -- __0) - inst(LOAD_NAME) { + inst(LOAD_NAME, ( -- v)) { PyObject *name = GETITEM(names, oparg); PyObject *locals = LOCALS(); - PyObject *v; if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when loading %R", name); @@ -1200,7 +1163,6 @@ dummy_func( } } } - PUSH(v); } // error: LOAD_GLOBAL has irregular stack effect @@ -1339,9 +1301,8 @@ dummy_func( Py_DECREF(oldobj); } - // stack effect: ( -- __0) - inst(LOAD_CLASSDEREF) { - PyObject *name, *value, *locals = LOCALS(); + inst(LOAD_CLASSDEREF, ( -- value)) { + PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg); @@ -1372,31 +1333,26 @@ dummy_func( } Py_INCREF(value); } - PUSH(value); } - // stack effect: ( -- __0) - inst(LOAD_DEREF) { + inst(LOAD_DEREF, ( -- value)) { PyObject *cell = GETLOCAL(oparg); - PyObject *value = PyCell_GET(cell); + value = PyCell_GET(cell); if (value == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); - goto error; + ERROR_IF(true, error); } - PUSH(Py_NewRef(value)); + Py_INCREF(value); } - // stack effect: (__0 -- ) - inst(STORE_DEREF) { - PyObject *v = POP(); + inst(STORE_DEREF, (v --)) { PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); } - // stack effect: ( -- ) - inst(COPY_FREE_VARS) { + inst(COPY_FREE_VARS, (--)) { /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1444,21 +1400,14 @@ dummy_func( PUSH(list); } - // stack effect: ( -- ) - inst(LIST_TO_TUPLE) { - PyObject *list = POP(); - PyObject *tuple = PyList_AsTuple(list); - Py_DECREF(list); - if (tuple == NULL) { - goto error; - } - PUSH(tuple); + inst(LIST_TO_TUPLE, (list -- tuple)) { + tuple = PyList_AsTuple(list); + DECREF_INPUTS(); + ERROR_IF(tuple == NULL, error); } - // stack effect: (__0 -- ) - inst(LIST_EXTEND) { - PyObject *iterable = POP(); - PyObject *list = PEEK(oparg); + inst(LIST_EXTEND, (iterable -- )) { + PyObject *list = PEEK(oparg + 1); // iterable is still on the stack PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1469,22 +1418,18 @@ dummy_func( "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } Py_DECREF(none_val); - Py_DECREF(iterable); + DECREF_INPUTS(); } - // stack effect: (__0 -- ) - inst(SET_UPDATE) { - PyObject *iterable = POP(); - PyObject *set = PEEK(oparg); + inst(SET_UPDATE, (iterable --)) { + PyObject *set = PEEK(oparg + 1); // iterable is still on the stack int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) { - goto error; - } + DECREF_INPUTS(); + ERROR_IF(err < 0, error); } // stack effect: (__array[oparg] -- __0) @@ -1524,54 +1469,41 @@ dummy_func( PUSH(map); } - // stack effect: ( -- ) - inst(SETUP_ANNOTATIONS) { + inst(SETUP_ANNOTATIONS, (--)) { int err; PyObject *ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - goto error; + ERROR_IF(true, error); } /* check if __annotations__ in locals()... */ if (PyDict_CheckExact(LOCALS())) { ann_dict = _PyDict_GetItemWithError(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) { - goto error; - } + ERROR_IF(_PyErr_Occurred(tstate), error); /* ...if not, create a new one */ ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + ERROR_IF(ann_dict == NULL, error); err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + ERROR_IF(err, error); } } else { /* do the same if locals() is not a dict */ ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } + ERROR_IF(!_PyErr_ExceptionMatches(tstate, PyExc_KeyError), error); _PyErr_Clear(tstate); ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + ERROR_IF(ann_dict == NULL, error); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + ERROR_IF(err, error); } else { Py_DECREF(ann_dict); @@ -1603,48 +1535,38 @@ dummy_func( PUSH(map); } - // stack effect: (__0 -- ) - inst(DICT_UPDATE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + inst(DICT_UPDATE, (update --)) { + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - Py_DECREF(update); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } - Py_DECREF(update); + DECREF_INPUTS(); } - // stack effect: (__0 -- ) - inst(DICT_MERGE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + inst(DICT_MERGE, (update --)) { + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(2 + oparg), update); - Py_DECREF(update); - goto error; + format_kwargs_error(tstate, PEEK(3 + oparg), update); + DECREF_INPUTS(); + ERROR_IF(true, error); } - Py_DECREF(update); + DECREF_INPUTS(); PREDICT(CALL_FUNCTION_EX); } - // stack effect: (__0, __1 -- ) - inst(MAP_ADD) { - PyObject *value = TOP(); - PyObject *key = SECOND(); - PyObject *map; - STACK_SHRINK(2); - map = PEEK(oparg); /* dict */ - assert(PyDict_CheckExact(map)); - /* map[key] = value */ - if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) { - goto error; - } + inst(MAP_ADD, (key, value --)) { + PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); PREDICT(JUMP_BACKWARD); } @@ -2073,29 +1995,17 @@ dummy_func( } super(COMPARE_OP_STR_JUMP) = _COMPARE_OP_STR + _JUMP_IF; - // stack effect: (__0 -- ) - inst(IS_OP) { - PyObject *right = POP(); - PyObject *left = TOP(); + inst(IS_OP, (left, right -- b)) { int res = Py_Is(left, right) ^ oparg; - PyObject *b = res ? Py_True : Py_False; - SET_TOP(Py_NewRef(b)); - Py_DECREF(left); - Py_DECREF(right); + DECREF_INPUTS(); + b = Py_NewRef(res ? Py_True : Py_False); } - // stack effect: (__0 -- ) - inst(CONTAINS_OP) { - PyObject *right = POP(); - PyObject *left = POP(); + inst(CONTAINS_OP, (left, right -- b)) { int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) { - goto error; - } - PyObject *b = (res^oparg) ? Py_True : Py_False; - PUSH(Py_NewRef(b)); + DECREF_INPUTS(); + ERROR_IF(res < 0, error); + b = Py_NewRef((res^oparg) ? Py_True : Py_False); } // stack effect: ( -- ) @@ -2139,76 +2049,57 @@ dummy_func( } } - // stack effect: ( -- ) - inst(CHECK_EXC_MATCH) { - PyObject *right = POP(); - PyObject *left = TOP(); + inst(CHECK_EXC_MATCH, (left, right -- left, b)) { assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - Py_DECREF(right); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - PUSH(Py_NewRef(res ? Py_True : Py_False)); + DECREF_INPUTS(); + b = Py_NewRef(res ? Py_True : Py_False); } - // stack effect: (__0 -- ) - inst(IMPORT_NAME) { + inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(names, oparg); - PyObject *fromlist = POP(); - PyObject *level = TOP(); - PyObject *res; res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - SET_TOP(res); - if (res == NULL) - goto error; + DECREF_INPUTS(); + ERROR_IF(res == NULL, error); } - // stack effect: (__0 -- ) - inst(IMPORT_STAR) { - PyObject *from = POP(), *locals; + inst(IMPORT_STAR, (from --)) { + PyObject *locals; int err; if (_PyFrame_FastToLocalsWithError(frame) < 0) { - Py_DECREF(from); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); - Py_DECREF(from); - goto error; + DECREF_INPUTS(); + ERROR_IF(true, error); } err = import_all_from(tstate, locals, from); _PyFrame_LocalsToFast(frame, 0); - Py_DECREF(from); - if (err != 0) - goto error; + DECREF_INPUTS(); + ERROR_IF(err, error); } - // stack effect: ( -- __0) - inst(IMPORT_FROM) { + inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(names, oparg); - PyObject *from = TOP(); - PyObject *res; res = import_from(tstate, from, name); - PUSH(res); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); } - // stack effect: ( -- ) - inst(JUMP_FORWARD) { + inst(JUMP_FORWARD, (--)) { JUMPBY(oparg); } - // stack effect: ( -- ) - inst(JUMP_BACKWARD) { + inst(JUMP_BACKWARD, (--)) { assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); @@ -3631,8 +3522,6 @@ family(load_attr) = { LOAD_ATTR_PROPERTY, LOAD_ATTR_SLOT, LOAD_ATTR_WITH_HINT, LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_METHOD_NO_DICT, LOAD_ATTR_METHOD_WITH_DICT, LOAD_ATTR_METHOD_WITH_VALUES }; -family(load_const) = { LOAD_CONST, LOAD_CONST__LOAD_FAST }; -family(load_fast) = { LOAD_FAST, LOAD_FAST__LOAD_CONST, LOAD_FAST__LOAD_FAST }; family(load_global) = { LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN, LOAD_GLOBAL_MODULE }; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 42b7ca086705..1179bdfc696c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -795,10 +795,10 @@ } TARGET(GET_ANEXT) { + PyObject *aiter = PEEK(1); + PyObject *awaitable; unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyObject *awaitable = NULL; - PyObject *aiter = TOP(); PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { @@ -840,15 +840,17 @@ } } - PUSH(awaitable); + STACK_GROW(1); + POKE(1, awaitable); PREDICT(LOAD_CONST); DISPATCH(); } TARGET(GET_AWAITABLE) { PREDICTED(GET_AWAITABLE); - PyObject *iterable = TOP(); - PyObject *iter = _PyCoro_GetAwaitableIter(iterable); + PyObject *iterable = PEEK(1); + PyObject *iter; + iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); @@ -870,12 +872,9 @@ } } - SET_TOP(iter); /* Even if it's NULL */ - - if (iter == NULL) { - goto error; - } + if (iter == NULL) goto pop_1_error; + POKE(1, iter); PREDICT(LOAD_CONST); DISPATCH(); } @@ -931,27 +930,26 @@ } TARGET(ASYNC_GEN_WRAP) { - PyObject *v = TOP(); + PyObject *v = PEEK(1); + PyObject *w; assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR); - PyObject *w = _PyAsyncGenValueWrapperNew(v); - if (w == NULL) { - goto error; - } - SET_TOP(w); + w = _PyAsyncGenValueWrapperNew(v); Py_DECREF(v); + if (w == NULL) goto pop_1_error; + POKE(1, w); DISPATCH(); } TARGET(YIELD_VALUE) { + PyObject *retval = PEEK(1); // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. assert(oparg == STACK_LEVEL()); assert(frame != &entry_frame); - PyObject *retval = POP(); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_SetStackPointer(frame, stack_pointer - 1); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; @@ -966,10 +964,10 @@ } TARGET(POP_EXCEPT) { + PyObject *exc_value = PEEK(1); _PyErr_StackItem *exc_info = tstate->exc_info; - PyObject *value = exc_info->exc_value; - exc_info->exc_value = POP(); - Py_XDECREF(value); + Py_XSETREF(exc_info->exc_value, exc_value); + STACK_SHRINK(1); DISPATCH(); } @@ -995,19 +993,18 @@ } TARGET(PREP_RERAISE_STAR) { - PyObject *excs = POP(); + PyObject *excs = PEEK(1); + PyObject *orig = PEEK(2); + PyObject *val; assert(PyList_Check(excs)); - PyObject *orig = POP(); - PyObject *val = _PyExc_PrepReraiseStar(orig, excs); - Py_DECREF(excs); + val = _PyExc_PrepReraiseStar(orig, excs); Py_DECREF(orig); + Py_DECREF(excs); - if (val == NULL) { - goto error; - } - - PUSH(val); + if (val == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, val); DISPATCH(); } @@ -1091,8 +1088,10 @@ } TARGET(LOAD_ASSERTION_ERROR) { - PyObject *value = PyExc_AssertionError; - PUSH(Py_NewRef(value)); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } @@ -1106,7 +1105,7 @@ _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); } - goto error; + if (true) goto error; } Py_INCREF(bc); } @@ -1116,31 +1115,32 @@ if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - goto error; + if (true) goto error; } } - PUSH(bc); + STACK_GROW(1); + POKE(1, bc); DISPATCH(); } TARGET(STORE_NAME) { + PyObject *v = PEEK(1); PyObject *name = GETITEM(names, oparg); - PyObject *v = POP(); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); Py_DECREF(v); - goto error; + if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); Py_DECREF(v); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } @@ -1304,9 +1304,9 @@ } TARGET(LOAD_NAME) { + PyObject *v; PyObject *name = GETITEM(names, oparg); PyObject *locals = LOCALS(); - PyObject *v; if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when loading %R", name); @@ -1363,7 +1363,8 @@ } } } - PUSH(v); + STACK_GROW(1); + POKE(1, v); DISPATCH(); } @@ -1508,7 +1509,8 @@ } TARGET(LOAD_CLASSDEREF) { - PyObject *name, *value, *locals = LOCALS(); + PyObject *value; + PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg); @@ -1539,27 +1541,32 @@ } Py_INCREF(value); } - PUSH(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(LOAD_DEREF) { + PyObject *value; PyObject *cell = GETLOCAL(oparg); - PyObject *value = PyCell_GET(cell); + value = PyCell_GET(cell); if (value == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); - goto error; + if (true) goto error; } - PUSH(Py_NewRef(value)); + Py_INCREF(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(STORE_DEREF) { - PyObject *v = POP(); + PyObject *v = PEEK(1); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); + STACK_SHRINK(1); DISPATCH(); } @@ -1613,19 +1620,18 @@ } TARGET(LIST_TO_TUPLE) { - PyObject *list = POP(); - PyObject *tuple = PyList_AsTuple(list); + PyObject *list = PEEK(1); + PyObject *tuple; + tuple = PyList_AsTuple(list); Py_DECREF(list); - if (tuple == NULL) { - goto error; - } - PUSH(tuple); + if (tuple == NULL) goto pop_1_error; + POKE(1, tuple); DISPATCH(); } TARGET(LIST_EXTEND) { - PyObject *iterable = POP(); - PyObject *list = PEEK(oparg); + PyObject *iterable = PEEK(1); + PyObject *list = PEEK(oparg + 1); // iterable is still on the stack PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1637,21 +1643,21 @@ Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); - goto error; + if (true) goto pop_1_error; } Py_DECREF(none_val); Py_DECREF(iterable); + STACK_SHRINK(1); DISPATCH(); } TARGET(SET_UPDATE) { - PyObject *iterable = POP(); - PyObject *set = PEEK(oparg); + PyObject *iterable = PEEK(1); + PyObject *set = PEEK(oparg + 1); // iterable is still on the stack int err = _PySet_Update(set, iterable); Py_DECREF(iterable); - if (err < 0) { - goto error; - } + if (err < 0) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } @@ -1698,47 +1704,35 @@ if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - goto error; + if (true) goto error; } /* check if __annotations__ in locals()... */ if (PyDict_CheckExact(LOCALS())) { ann_dict = _PyDict_GetItemWithError(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) { - goto error; - } + if (_PyErr_Occurred(tstate)) goto error; /* ...if not, create a new one */ ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + if (ann_dict == NULL) goto error; err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + if (err) goto error; } } else { /* do the same if locals() is not a dict */ ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) goto error; _PyErr_Clear(tstate); ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } + if (ann_dict == NULL) goto error; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } + if (err) goto error; } else { Py_DECREF(ann_dict); @@ -1772,8 +1766,8 @@ } TARGET(DICT_UPDATE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + PyObject *update = PEEK(1); + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -1781,37 +1775,37 @@ Py_TYPE(update)->tp_name); } Py_DECREF(update); - goto error; + if (true) goto pop_1_error; } Py_DECREF(update); + STACK_SHRINK(1); DISPATCH(); } TARGET(DICT_MERGE) { - PyObject *update = POP(); - PyObject *dict = PEEK(oparg); + PyObject *update = PEEK(1); + PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(2 + oparg), update); + format_kwargs_error(tstate, PEEK(3 + oparg), update); Py_DECREF(update); - goto error; + if (true) goto pop_1_error; } Py_DECREF(update); + STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); DISPATCH(); } TARGET(MAP_ADD) { - PyObject *value = TOP(); - PyObject *key = SECOND(); - PyObject *map; + PyObject *value = PEEK(1); + PyObject *key = PEEK(2); + PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; STACK_SHRINK(2); - map = PEEK(oparg); /* dict */ - assert(PyDict_CheckExact(map)); - /* map[key] = value */ - if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) { - goto error; - } PREDICT(JUMP_BACKWARD); DISPATCH(); } @@ -2312,27 +2306,29 @@ } TARGET(IS_OP) { - PyObject *right = POP(); - PyObject *left = TOP(); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *b; int res = Py_Is(left, right) ^ oparg; - PyObject *b = res ? Py_True : Py_False; - SET_TOP(Py_NewRef(b)); Py_DECREF(left); Py_DECREF(right); + b = Py_NewRef(res ? Py_True : Py_False); + STACK_SHRINK(1); + POKE(1, b); DISPATCH(); } TARGET(CONTAINS_OP) { - PyObject *right = POP(); - PyObject *left = POP(); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *b; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) { - goto error; - } - PyObject *b = (res^oparg) ? Py_True : Py_False; - PUSH(Py_NewRef(b)); + if (res < 0) goto pop_2_error; + b = Py_NewRef((res^oparg) ? Py_True : Py_False); + STACK_SHRINK(1); + POKE(1, b); DISPATCH(); } @@ -2378,40 +2374,43 @@ } TARGET(CHECK_EXC_MATCH) { - PyObject *right = POP(); - PyObject *left = TOP(); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *b; assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { Py_DECREF(right); - goto error; + if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); - PUSH(Py_NewRef(res ? Py_True : Py_False)); + b = Py_NewRef(res ? Py_True : Py_False); + POKE(1, b); DISPATCH(); } TARGET(IMPORT_NAME) { - PyObject *name = GETITEM(names, oparg); - PyObject *fromlist = POP(); - PyObject *level = TOP(); + PyObject *fromlist = PEEK(1); + PyObject *level = PEEK(2); PyObject *res; + PyObject *name = GETITEM(names, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); Py_DECREF(fromlist); - SET_TOP(res); - if (res == NULL) - goto error; + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, res); DISPATCH(); } TARGET(IMPORT_STAR) { - PyObject *from = POP(), *locals; + PyObject *from = PEEK(1); + PyObject *locals; int err; if (_PyFrame_FastToLocalsWithError(frame) < 0) { Py_DECREF(from); - goto error; + if (true) goto pop_1_error; } locals = LOCALS(); @@ -2419,24 +2418,24 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); Py_DECREF(from); - goto error; + if (true) goto pop_1_error; } err = import_all_from(tstate, locals, from); _PyFrame_LocalsToFast(frame, 0); Py_DECREF(from); - if (err != 0) - goto error; + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } TARGET(IMPORT_FROM) { - PyObject *name = GETITEM(names, oparg); - PyObject *from = TOP(); + PyObject *from = PEEK(1); PyObject *res; + PyObject *name = GETITEM(names, oparg); res = import_from(tstate, from, name); - PUSH(res); - if (res == NULL) - goto error; + if (res == NULL) goto error; + STACK_GROW(1); + POKE(1, res); DISPATCH(); } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 85a7c6098e0b..5eed74c5e147 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -125,6 +125,7 @@ class Instruction: # Set later family: parser.Family | None = None predicted: bool = False + unmoved_names: frozenset[str] = frozenset() def __init__(self, inst: parser.InstDef): self.inst = inst @@ -141,6 +142,13 @@ def __init__(self, inst: parser.InstDef): effect for effect in inst.inputs if isinstance(effect, StackEffect) ] self.output_effects = inst.outputs # For consistency/completeness + unmoved_names: set[str] = set() + for ieffect, oeffect in zip(self.input_effects, self.output_effects): + if ieffect.name == oeffect.name: + unmoved_names.add(ieffect.name) + else: + break + self.unmoved_names = frozenset(unmoved_names) def write(self, out: Formatter) -> None: """Write one instruction, sans prologue and epilogue.""" @@ -175,12 +183,8 @@ def write(self, out: Formatter) -> None: out.stack_adjust(diff) # Write output stack effect assignments - unmoved_names: set[str] = set() - for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect.name == oeffect.name: - unmoved_names.add(ieffect.name) for i, oeffect in enumerate(reversed(self.output_effects), 1): - if oeffect.name not in unmoved_names: + if oeffect.name not in self.unmoved_names: dst = StackEffect(f"PEEK({i})", "") out.assign(dst, oeffect) @@ -235,7 +239,8 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None elif m := re.match(r"(\s*)DECREF_INPUTS;\s*$", line): space = m.group(1) for ieff in self.input_effects: - out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") + if ieff.name not in self.unmoved_names: + out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") else: out.write_raw(extra + line) @@ -533,7 +538,7 @@ def stack_analysis( ) -> tuple[list[StackEffect], int]: """Analyze a super-instruction or macro. - Print an error if there's a cache effect (which we don't support yet). + Ignore cache effects. Return the list of variable names and the initial stack pointer. """ From webhook-mailer at python.org Tue Dec 27 20:16:58 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 01:16:58 -0000 Subject: [Python-checkins] gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) Message-ID: <mailman.3102.1672190219.3313.python-checkins@python.org> https://github.com/python/cpython/commit/04285502ba7d90e41138b0282434372be18ac042 commit: 04285502ba7d90e41138b0282434372be18ac042 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T17:16:53-08:00 summary: gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) (cherry picked from commit 3dc48dabd48864039951715816e07986a4828d80) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/sqlite3/test/factory.py diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py index 876428497542..40a290f0c981 100644 --- a/Lib/sqlite3/test/factory.py +++ b/Lib/sqlite3/test/factory.py @@ -155,8 +155,14 @@ def test_sqlite_row_iter(self): """Checks if the row object is iterable""" self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): """Checks if the row object can be converted to a tuple""" From webhook-mailer at python.org Tue Dec 27 20:25:34 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 01:25:34 -0000 Subject: [Python-checkins] gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) Message-ID: <mailman.3103.1672190734.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fba8c7cf740433cc71280ea6dce6b44ee477b2d3 commit: fba8c7cf740433cc71280ea6dce6b44ee477b2d3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T17:25:28-08:00 summary: gh-100553: Improve accuracy of sqlite3.Row iter test (GH-100555) (cherry picked from commit 3dc48dabd48864039951715816e07986a4828d80) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_sqlite3/test_factory.py diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index cdaf12588a7f..2b70093bd9b6 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -179,8 +179,14 @@ def test_sqlite_row_iter(self): """Checks if the row object is iterable""" self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): """Checks if the row object can be converted to a tuple""" From webhook-mailer at python.org Tue Dec 27 23:12:22 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:12:22 -0000 Subject: [Python-checkins] GH-100192: add more `asyncio` subprocess tests (#100194) Message-ID: <mailman.3104.1672200743.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e97afefda54b818b140b3cc905642b69d9d65f0c commit: e97afefda54b818b140b3cc905642b69d9d65f0c branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T09:42:16+05:30 summary: GH-100192: add more `asyncio` subprocess tests (#100194) files: M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3830dea7d9ba..1ae290a003f2 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -686,6 +686,88 @@ async def execute(): self.assertIsNone(self.loop.run_until_complete(execute())) + async def check_stdout_output(self, coro, output): + proc = await coro + stdout, _ = await proc.communicate() + self.assertEqual(stdout, output) + self.assertEqual(proc.returncode, 0) + task = asyncio.create_task(proc.wait()) + await asyncio.sleep(0) + self.assertEqual(task.result(), proc.returncode) + + def test_create_subprocess_env_shell(self) -> None: + async def main() -> None: + cmd = f'''{sys.executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' + env = {"FOO": 'bar'} + proc = await asyncio.create_subprocess_shell( + cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'bar')) + + def test_create_subprocess_env_exec(self) -> None: + async def main() -> None: + cmd = [sys.executable, "-c", + "import os, sys; sys.stdout.write(os.getenv('FOO'))"] + env = {"FOO": 'baz'} + proc = await asyncio.create_subprocess_exec( + *cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'baz')) + + + def test_subprocess_concurrent_wait(self) -> None: + async def main() -> None: + proc = await asyncio.create_subprocess_exec( + *PROGRAM_CAT, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, _ = await proc.communicate(b'some data') + self.assertEqual(stdout, b"some data") + self.assertEqual(proc.returncode, 0) + self.assertEqual(await asyncio.gather(*[proc.wait() for _ in range(10)]), + [proc.returncode] * 10) + + self.loop.run_until_complete(main()) + + def test_subprocess_consistent_callbacks(self): + events = [] + class MyProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future: asyncio.Future) -> None: + self.exit_future = exit_future + + def pipe_data_received(self, fd, data) -> None: + events.append(('pipe_data_received', fd, data)) + + def pipe_connection_lost(self, fd, exc) -> None: + events.append('pipe_connection_lost') + + def process_exited(self) -> None: + events.append('process_exited') + self.exit_future.set_result(True) + + async def main() -> None: + loop = asyncio.get_running_loop() + exit_future = asyncio.Future() + code = 'import sys; sys.stdout.write("stdout"); sys.stderr.write("stderr")' + transport, _ = await loop.subprocess_exec(lambda: MyProtocol(exit_future), + sys.executable, '-c', code, stdin=None) + await exit_future + transport.close() + self.assertEqual(events, [ + ('pipe_data_received', 1, b'stdout'), + ('pipe_data_received', 2, b'stderr'), + 'pipe_connection_lost', + 'pipe_connection_lost', + 'process_exited', + ]) + + self.loop.run_until_complete(main()) + def test_subprocess_communicate_stdout(self): # See https://github.com/python/cpython/issues/100133 async def get_command_stdout(cmd, *args): From webhook-mailer at python.org Tue Dec 27 23:16:34 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:16:34 -0000 Subject: [Python-checkins] GH-100342: check for allocation failure in AC `*args` parsing (#100343) Message-ID: <mailman.3105.1672200995.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7cf164ad5e3c8c6af5ae8813ad6a784448605418 commit: 7cf164ad5e3c8c6af5ae8813ad6a784448605418 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T09:46:28+05:30 summary: GH-100342: check for allocation failure in AC `*args` parsing (#100343) files: A Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst M Lib/test/clinic.test M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 0d844234d9d1..53e5df5ba872 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3781,6 +3781,9 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -3793,7 +3796,7 @@ exit: static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=081a953b8cbe7617 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=79b75dc07decc8d6 input=08dc2bf7afbf1613]*/ /*[clinic input] test_vararg diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst new file mode 100644 index 000000000000..28f736337526 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst @@ -0,0 +1 @@ +Add missing ``NULL`` check for possible allocation failure in ``*args`` parsing in Argument Clinic. diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 21bde5294702..831f58ca650a 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2409,6 +2409,9 @@ vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -2769,6 +2772,9 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 0; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); } @@ -2811,4 +2817,4 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=9a5ca5909c087102 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e8211606b03d733a input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2fb1902a5b54..552ed2c889a7 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -960,12 +960,16 @@ def parser_body(prototype, *fields, declarations=''): if not new_or_init: parser_code.append(normalize_snippet(""" %s = PyTuple_New(%s); + if (!%s) {{ + goto exit; + }} for (Py_ssize_t i = 0; i < %s; ++i) {{ PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); }} """ % ( p.converter.parser_name, left_args, + p.converter.parser_name, left_args, p.converter.parser_name, max_pos From webhook-mailer at python.org Tue Dec 27 23:28:03 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:28:03 -0000 Subject: [Python-checkins] Fix copy-paste error in `Tools/clinic.py` (#100560) Message-ID: <mailman.3106.1672201684.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3da71ff0f26516c433538b2bfc4db196e20e0d44 commit: 3da71ff0f26516c433538b2bfc4db196e20e0d44 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T09:57:57+05:30 summary: Fix copy-paste error in `Tools/clinic.py` (#100560) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 552ed2c889a7..abff4d2583e1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2832,7 +2832,7 @@ def modify(self): """ The C statements required to modify this variable after parsing. Returns a string containing this code indented at column 0. - If no initialization is necessary, returns an empty string. + If no modification is necessary, returns an empty string. """ return "" From webhook-mailer at python.org Tue Dec 27 23:38:01 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:38:01 -0000 Subject: [Python-checkins] add tests for `asyncio` transport sockets (#100263) Message-ID: <mailman.3107.1672202281.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5369bba8c594de7a643408550e19e1ff6df5178a commit: 5369bba8c594de7a643408550e19e1ff6df5178a branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T10:07:55+05:30 summary: add tests for `asyncio` transport sockets (#100263) files: M Lib/test/test_asyncio/test_events.py diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 153b2de81722..e7771edd2e4a 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -823,6 +823,29 @@ def test_create_server(self): # close server server.close() + def test_create_server_trsock(self): + proto = MyProto(self.loop) + f = self.loop.create_server(lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertIsInstance(sock, asyncio.trsock.TransportSocket) + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + dup = sock.dup() + self.addCleanup(dup.close) + self.assertIsInstance(dup, socket.socket) + self.assertFalse(sock.get_inheritable()) + with self.assertRaises(ValueError): + sock.settimeout(1) + sock.settimeout(0) + self.assertEqual(sock.gettimeout(), 0) + with self.assertRaises(ValueError): + sock.setblocking(True) + sock.setblocking(False) + server.close() + + @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT') def test_create_server_reuse_port(self): proto = MyProto(self.loop) From webhook-mailer at python.org Tue Dec 27 23:41:32 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 04:41:32 -0000 Subject: [Python-checkins] [3.11] GH-100342: check for allocation failure in AC `*args` parsing (GH-100343). (#100568) Message-ID: <mailman.3108.1672202493.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ebe428824d9c749577256cbcfb35945d8b19c479 commit: ebe428824d9c749577256cbcfb35945d8b19c479 branch: 3.11 author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T10:11:27+05:30 summary: [3.11] GH-100342: check for allocation failure in AC `*args` parsing (GH-100343). (#100568) (cherry picked from commit 7cf164ad5e3c8c6af5ae8813ad6a784448605418) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst M Lib/test/clinic.test M Modules/clinic/_testclinic.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 217aa4c7e97a..0abadbe7f347 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3327,6 +3327,9 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -3339,7 +3342,7 @@ exit: static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=081a953b8cbe7617 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=79b75dc07decc8d6 input=08dc2bf7afbf1613]*/ /*[clinic input] test_vararg diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst new file mode 100644 index 000000000000..28f736337526 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-12-19-10-08-53.gh-issue-100342.qDFlQG.rst @@ -0,0 +1 @@ +Add missing ``NULL`` check for possible allocation failure in ``*args`` parsing in Argument Clinic. diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 41555d801059..c97ac1c66296 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2012,6 +2012,9 @@ vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } a = args[0]; __clinic_args = PyTuple_New(nargs - 1); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 1; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); } @@ -2257,6 +2260,9 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } for (Py_ssize_t i = 0; i < nargs - 0; ++i) { PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); } @@ -2299,4 +2305,4 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=fe398ac790310bc4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b719efc1b8bd2c8 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a6d8b86a83ce..82e4919242c7 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -903,12 +903,16 @@ def parser_body(prototype, *fields, declarations=''): if not new_or_init: parser_code.append(normalize_snippet(""" %s = PyTuple_New(%s); + if (!%s) {{ + goto exit; + }} for (Py_ssize_t i = 0; i < %s; ++i) {{ PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); }} """ % ( p.converter.parser_name, left_args, + p.converter.parser_name, left_args, p.converter.parser_name, max_pos From webhook-mailer at python.org Wed Dec 28 00:30:47 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Wed, 28 Dec 2022 05:30:47 -0000 Subject: [Python-checkins] gh-55688: Add note about ending backslashes for raw strings (#94768) Message-ID: <mailman.3109.1672205448.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8 commit: b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8 branch: main author: Stanley <46876382+slateny at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-28T00:30:42-05:00 summary: gh-55688: Add note about ending backslashes for raw strings (#94768) Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: M Doc/faq/programming.rst M Doc/tutorial/introduction.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index c396e2b081fc..ba42289f3466 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1026,6 +1026,46 @@ What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? See the :ref:`unicode-howto`. +.. _faq-programming-raw-string-backslash: + +Can I end a raw string with an odd number of backslashes? +--------------------------------------------------------- + +A raw string ending with an odd number of backslashes will escape the string's quote:: + + >>> r'C:\this\will\not\work\' + File "<stdin>", line 1 + r'C:\this\will\not\work\' + ^ + SyntaxError: unterminated string literal (detected at line 1) + +There are several workarounds for this. One is to use regular strings and double +the backslashes:: + + >>> 'C:\\this\\will\\work\\' + 'C:\\this\\will\\work\\' + +Another is to concatenate a regular string containing an escaped backslash to the +raw string:: + + >>> r'C:\this\will\work' '\\' + 'C:\\this\\will\\work\\' + +It is also possible to use :func:`os.path.join` to append a backslash on Windows:: + + >>> os.path.join(r'C:\this\will\work', '') + 'C:\\this\\will\\work\\' + +Note that while a backslash will "escape" a quote for the purposes of +determining where the raw string ends, no escaping occurs when interpreting the +value of the raw string. That is, the backslash remains present in the value of +the raw string:: + + >>> r'backslash\'preserved' + "backslash\\'preserved" + +Also see the specification in the :ref:`language reference <strings>`. + Performance =========== diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 6532a4e2cc83..ebc2e9187534 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -189,6 +189,11 @@ the first quote:: >>> print(r'C:\some\name') # note the r before the quote C:\some\name +There is one subtle aspect to raw strings: a raw string may not end in +an odd number of ``\`` characters; see +:ref:`the FAQ entry <faq-programming-raw-string-backslash>` for more information +and workarounds. + String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at From webhook-mailer at python.org Wed Dec 28 00:39:58 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 05:39:58 -0000 Subject: [Python-checkins] gh-55688: Add note about ending backslashes for raw strings (GH-94768) Message-ID: <mailman.3110.1672205999.3313.python-checkins@python.org> https://github.com/python/cpython/commit/97a3e18c8873228a7ab7f720954798c2c5c7486d commit: 97a3e18c8873228a7ab7f720954798c2c5c7486d branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T21:39:53-08:00 summary: gh-55688: Add note about ending backslashes for raw strings (GH-94768) (cherry picked from commit b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: M Doc/faq/programming.rst M Doc/tutorial/introduction.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index bd75801cc242..1bcca4c62d27 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1026,6 +1026,46 @@ What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? See the :ref:`unicode-howto`. +.. _faq-programming-raw-string-backslash: + +Can I end a raw string with an odd number of backslashes? +--------------------------------------------------------- + +A raw string ending with an odd number of backslashes will escape the string's quote:: + + >>> r'C:\this\will\not\work\' + File "<stdin>", line 1 + r'C:\this\will\not\work\' + ^ + SyntaxError: unterminated string literal (detected at line 1) + +There are several workarounds for this. One is to use regular strings and double +the backslashes:: + + >>> 'C:\\this\\will\\work\\' + 'C:\\this\\will\\work\\' + +Another is to concatenate a regular string containing an escaped backslash to the +raw string:: + + >>> r'C:\this\will\work' '\\' + 'C:\\this\\will\\work\\' + +It is also possible to use :func:`os.path.join` to append a backslash on Windows:: + + >>> os.path.join(r'C:\this\will\work', '') + 'C:\\this\\will\\work\\' + +Note that while a backslash will "escape" a quote for the purposes of +determining where the raw string ends, no escaping occurs when interpreting the +value of the raw string. That is, the backslash remains present in the value of +the raw string:: + + >>> r'backslash\'preserved' + "backslash\\'preserved" + +Also see the specification in the :ref:`language reference <strings>`. + Performance =========== diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 558b1c3eec60..b7a89905e4f3 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -189,6 +189,11 @@ the first quote:: >>> print(r'C:\some\name') # note the r before the quote C:\some\name +There is one subtle aspect to raw strings: a raw string may not end in +an odd number of ``\`` characters; see +:ref:`the FAQ entry <faq-programming-raw-string-backslash>` for more information +and workarounds. + String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at From webhook-mailer at python.org Wed Dec 28 00:45:51 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 05:45:51 -0000 Subject: [Python-checkins] gh-55688: Add note about ending backslashes for raw strings (GH-94768) Message-ID: <mailman.3111.1672206352.3313.python-checkins@python.org> https://github.com/python/cpython/commit/de621281ce704546b87334fd7da7a7ec534719b7 commit: de621281ce704546b87334fd7da7a7ec534719b7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-27T21:45:45-08:00 summary: gh-55688: Add note about ending backslashes for raw strings (GH-94768) (cherry picked from commit b95b1b3b25b0a93a22c7d58ac5bd5870e62070a8) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> Co-authored-by: hauntsaninja <hauntsaninja at gmail.com> files: M Doc/faq/programming.rst M Doc/tutorial/introduction.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index c396e2b081fc..ba42289f3466 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1026,6 +1026,46 @@ What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? See the :ref:`unicode-howto`. +.. _faq-programming-raw-string-backslash: + +Can I end a raw string with an odd number of backslashes? +--------------------------------------------------------- + +A raw string ending with an odd number of backslashes will escape the string's quote:: + + >>> r'C:\this\will\not\work\' + File "<stdin>", line 1 + r'C:\this\will\not\work\' + ^ + SyntaxError: unterminated string literal (detected at line 1) + +There are several workarounds for this. One is to use regular strings and double +the backslashes:: + + >>> 'C:\\this\\will\\work\\' + 'C:\\this\\will\\work\\' + +Another is to concatenate a regular string containing an escaped backslash to the +raw string:: + + >>> r'C:\this\will\work' '\\' + 'C:\\this\\will\\work\\' + +It is also possible to use :func:`os.path.join` to append a backslash on Windows:: + + >>> os.path.join(r'C:\this\will\work', '') + 'C:\\this\\will\\work\\' + +Note that while a backslash will "escape" a quote for the purposes of +determining where the raw string ends, no escaping occurs when interpreting the +value of the raw string. That is, the backslash remains present in the value of +the raw string:: + + >>> r'backslash\'preserved' + "backslash\\'preserved" + +Also see the specification in the :ref:`language reference <strings>`. + Performance =========== diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 558b1c3eec60..b7a89905e4f3 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -189,6 +189,11 @@ the first quote:: >>> print(r'C:\some\name') # note the r before the quote C:\some\name +There is one subtle aspect to raw strings: a raw string may not end in +an odd number of ``\`` characters; see +:ref:`the FAQ entry <faq-programming-raw-string-backslash>` for more information +and workarounds. + String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at From webhook-mailer at python.org Wed Dec 28 00:46:09 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 05:46:09 -0000 Subject: [Python-checkins] GH-100192: fix `asyncio` subprocess tests to pass env vars to subprocess (#100569) Message-ID: <mailman.3112.1672206370.3313.python-checkins@python.org> https://github.com/python/cpython/commit/6835184a44389747dd151023b31a20ff75d6a402 commit: 6835184a44389747dd151023b31a20ff75d6a402 branch: main author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T11:16:04+05:30 summary: GH-100192: fix `asyncio` subprocess tests to pass env vars to subprocess (#100569) files: M Lib/test/test_asyncio/test_subprocess.py diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 1ae290a003f2..f1ad10a9903f 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -698,7 +698,8 @@ async def check_stdout_output(self, coro, output): def test_create_subprocess_env_shell(self) -> None: async def main() -> None: cmd = f'''{sys.executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' - env = {"FOO": 'bar'} + env = os.environ.copy() + env["FOO"] = "bar" proc = await asyncio.create_subprocess_shell( cmd, env=env, stdout=subprocess.PIPE ) @@ -710,7 +711,8 @@ def test_create_subprocess_env_exec(self) -> None: async def main() -> None: cmd = [sys.executable, "-c", "import os, sys; sys.stdout.write(os.getenv('FOO'))"] - env = {"FOO": 'baz'} + env = os.environ.copy() + env["FOO"] = "baz" proc = await asyncio.create_subprocess_exec( *cmd, env=env, stdout=subprocess.PIPE ) From webhook-mailer at python.org Wed Dec 28 05:56:18 2022 From: webhook-mailer at python.org (cjw296) Date: Wed, 28 Dec 2022 10:56:18 -0000 Subject: [Python-checkins] Remove skipped test in test for async mocks. (#100559) Message-ID: <mailman.3113.1672224979.3313.python-checkins@python.org> https://github.com/python/cpython/commit/984894a9a25c0f8298565b0c0c2e1f41917e4f88 commit: 984894a9a25c0f8298565b0c0c2e1f41917e4f88 branch: main author: Chris Withers <chris at withers.org> committer: cjw296 <chris at withers.org> date: 2022-12-28T10:55:50Z summary: Remove skipped test in test for async mocks. (#100559) Remove skipped test. See discussion on https://github.com/python/cpython/pull/25326. Fix is apparently here, but no-one is confident to review and land: https://github.com/python/cpython/pull/25347. files: M Lib/test/test_unittest/testmock/testasync.py diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 471162dc5050..5f12f9f95667 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -218,10 +218,6 @@ def test_create_autospec_instance(self): with self.assertRaises(RuntimeError): create_autospec(async_func, instance=True) - @unittest.skip('Broken test from https://bugs.python.org/issue37251') - def test_create_autospec_awaitable_class(self): - self.assertIsInstance(create_autospec(AwaitableClass), AsyncMock) - def test_create_autospec(self): spec = create_autospec(async_func_args) awaitable = spec(1, 2, c=3) From webhook-mailer at python.org Wed Dec 28 06:14:05 2022 From: webhook-mailer at python.org (rhettinger) Date: Wed, 28 Dec 2022 11:14:05 -0000 Subject: [Python-checkins] Improve factor() recipe and fix its tests (GH-100576) Message-ID: <mailman.3114.1672226046.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2d524068351a33feafa905becc148f3447697e92 commit: 2d524068351a33feafa905becc148f3447697e92 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-28T03:13:58-08:00 summary: Improve factor() recipe and fix its tests (GH-100576) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index b3634aecd10d..f1277dfdbdc6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -899,18 +899,16 @@ which incur interpreter overhead. def factor(n): "Prime factors of n." - # factor(97) --> 97 - # factor(98) --> 2 7 7 # factor(99) --> 3 3 11 - for prime in sieve(n+1): - while True: + for prime in sieve(math.isqrt(n) + 1): + while n >= prime: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient - if n == 1: - return + if n >= 2: + yield n def flatten(list_of_lists): "Flatten one level of nesting" @@ -1266,33 +1264,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True - list(factor(0)) + >>> list(factor(0)) [] - list(factor(1)) + >>> list(factor(1)) [] - list(factor(2)) + >>> list(factor(2)) [2] - list(factor(3)) + >>> list(factor(3)) [3] - list(factor(4)) + >>> list(factor(4)) [2, 2] - list(factor(5)) + >>> list(factor(5)) [5] - list(factor(6)) + >>> list(factor(6)) [2, 3] - list(factor(7)) + >>> list(factor(7)) [7] - list(factor(8)) + >>> list(factor(8)) [2, 2, 2] - list(factor(9)) + >>> list(factor(9)) [3, 3] - list(factor(10)) + >>> list(factor(10)) [2, 5] - all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(999953*999983)) + [999953, 999983] + >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) True - all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) True - all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 06:22:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 11:22:17 -0000 Subject: [Python-checkins] Improve factor() recipe and fix its tests (GH-100576) Message-ID: <mailman.3115.1672226537.3313.python-checkins@python.org> https://github.com/python/cpython/commit/95fa61cbce17a13d45aa0d0d85f0ad8e48f3edbd commit: 95fa61cbce17a13d45aa0d0d85f0ad8e48f3edbd branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T03:22:11-08:00 summary: Improve factor() recipe and fix its tests (GH-100576) (cherry picked from commit 2d524068351a33feafa905becc148f3447697e92) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 219ad95c76a1..221863cbbcc5 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -860,18 +860,16 @@ which incur interpreter overhead. def factor(n): "Prime factors of n." - # factor(97) --> 97 - # factor(98) --> 2 7 7 # factor(99) --> 3 3 11 - for prime in sieve(n+1): - while True: + for prime in sieve(math.isqrt(n) + 1): + while n >= prime: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient - if n == 1: - return + if n >= 2: + yield n def flatten(list_of_lists): "Flatten one level of nesting" @@ -1236,33 +1234,35 @@ which incur interpreter overhead. >>> set(sieve(10_000)).isdisjoint(carmichael) True - list(factor(0)) + >>> list(factor(0)) [] - list(factor(1)) + >>> list(factor(1)) [] - list(factor(2)) + >>> list(factor(2)) [2] - list(factor(3)) + >>> list(factor(3)) [3] - list(factor(4)) + >>> list(factor(4)) [2, 2] - list(factor(5)) + >>> list(factor(5)) [5] - list(factor(6)) + >>> list(factor(6)) [2, 3] - list(factor(7)) + >>> list(factor(7)) [7] - list(factor(8)) + >>> list(factor(8)) [2, 2, 2] - list(factor(9)) + >>> list(factor(9)) [3, 3] - list(factor(10)) + >>> list(factor(10)) [2, 5] - all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(999953*999983)) + [999953, 999983] + >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) True - all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) True - all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 07:36:40 2022 From: webhook-mailer at python.org (cjw296) Date: Wed, 28 Dec 2022 12:36:40 -0000 Subject: [Python-checkins] Fix mock code coverage. (#100580) Message-ID: <mailman.3116.1672231001.3313.python-checkins@python.org> https://github.com/python/cpython/commit/457c1f4a19a096a52d6553687c7c4cee415818dc commit: 457c1f4a19a096a52d6553687c7c4cee415818dc branch: main author: Chris Withers <chris at withers.org> committer: cjw296 <chris at withers.org> date: 2022-12-28T12:36:26Z summary: Fix mock code coverage. (#100580) files: M Lib/test/test_unittest/testmock/testsealable.py diff --git a/Lib/test/test_unittest/testmock/testsealable.py b/Lib/test/test_unittest/testmock/testsealable.py index e0c38293cffd..8bf98cfa562b 100644 --- a/Lib/test/test_unittest/testmock/testsealable.py +++ b/Lib/test/test_unittest/testmock/testsealable.py @@ -175,15 +175,12 @@ def test_seal_with_autospec(self): # https://bugs.python.org/issue45156 class Foo: foo = 0 - def bar1(self): - return 1 - def bar2(self): - return 2 + def bar1(self): pass + def bar2(self): pass class Baz: baz = 3 - def ban(self): - return 4 + def ban(self): pass for spec_set in (True, False): with self.subTest(spec_set=spec_set): From webhook-mailer at python.org Wed Dec 28 10:32:23 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 28 Dec 2022 15:32:23 -0000 Subject: [Python-checkins] gh-100577: Replace `assert(0)` with `Py_UNREACHABLE` in `symtable.c` (#100579) Message-ID: <mailman.3117.1672241544.3313.python-checkins@python.org> https://github.com/python/cpython/commit/5e1adb4f8861f2a5969952d24c8ad0ce8ec0a8ec commit: 5e1adb4f8861f2a5969952d24c8ad0ce8ec0a8ec branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-28T21:01:53+05:30 summary: gh-100577: Replace `assert(0)` with `Py_UNREACHABLE` in `symtable.c` (#100579) files: M Python/symtable.c diff --git a/Python/symtable.c b/Python/symtable.c index fb2bb7d83835..3c130186d0ad 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1539,7 +1539,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* We should always find either a FunctionBlock, ModuleBlock or ClassBlock and should never fall to this case */ - assert(0); + Py_UNREACHABLE(); return 0; } From webhook-mailer at python.org Wed Dec 28 13:07:12 2022 From: webhook-mailer at python.org (zware) Date: Wed, 28 Dec 2022 18:07:12 -0000 Subject: [Python-checkins] gh-100540: Remove unnecessary '-DMACOSX' for ctypes on macOS (GH-100542) Message-ID: <mailman.3118.1672250833.3313.python-checkins@python.org> https://github.com/python/cpython/commit/fd4b55ff17b1525d36f1e81762fca6f1a7bc59c5 commit: fd4b55ff17b1525d36f1e81762fca6f1a7bc59c5 branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-28T12:06:58-06:00 summary: gh-100540: Remove unnecessary '-DMACOSX' for ctypes on macOS (GH-100542) The define was only used to protect a `#pragma clang diagnostic` setting, which is already better guarded by `__clang__` anwyay. files: M Modules/_ctypes/callbacks.c M configure M configure.ac diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 459632b90907..f6880889dc0f 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -426,7 +426,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); goto error; #else -#if defined(__clang__) || defined(MACOSX) +#if defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif @@ -436,7 +436,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, #endif result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); -#if defined(__clang__) || defined(MACOSX) +#if defined(__clang__) #pragma clang diagnostic pop #endif #if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5))) diff --git a/configure b/configure index 3f8daf9dad5f..72299939892d 100755 --- a/configure +++ b/configure @@ -12846,7 +12846,7 @@ if test "x$have_libffi" = xyes; then : case $ac_sys_system in #( Darwin) : - as_fn_append LIBFFI_CFLAGS " -I\$(srcdir)/Modules/_ctypes/darwin -DMACOSX" + as_fn_append LIBFFI_CFLAGS " -I\$(srcdir)/Modules/_ctypes/darwin" ctypes_malloc_closure=yes ;; #( sunos5) : diff --git a/configure.ac b/configure.ac index 734a4db83899..d6499be4c007 100644 --- a/configure.ac +++ b/configure.ac @@ -3769,7 +3769,7 @@ AS_VAR_IF([have_libffi], [yes], [ AS_CASE([$ac_sys_system], [Darwin], [ dnl when do we need USING_APPLE_OS_LIBFFI? - AS_VAR_APPEND([LIBFFI_CFLAGS], [" -I\$(srcdir)/Modules/_ctypes/darwin -DMACOSX"]) + AS_VAR_APPEND([LIBFFI_CFLAGS], [" -I\$(srcdir)/Modules/_ctypes/darwin"]) ctypes_malloc_closure=yes ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] From webhook-mailer at python.org Wed Dec 28 14:36:15 2022 From: webhook-mailer at python.org (zware) Date: Wed, 28 Dec 2022 19:36:15 -0000 Subject: [Python-checkins] gh-100540: Remove obsolete '--with-system-ffi' configure option (GH-100544) Message-ID: <mailman.3119.1672256176.3313.python-checkins@python.org> https://github.com/python/cpython/commit/25590eb5dee5176f3ac60916b19450f8198e7ffc commit: 25590eb5dee5176f3ac60916b19450f8198e7ffc branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-28T13:36:03-06:00 summary: gh-100540: Remove obsolete '--with-system-ffi' configure option (GH-100544) It has had no effect on non-macOS platforms for a long time, and has had the non-obvious effect of invoking `pkg_config` and not setting `-DUSING_APPLE_OS_LIBFFI` on macOS since GH-22855. files: A Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst M Doc/using/configure.rst M configure M configure.ac diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index db4bf7412292..8fa8d250d533 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -471,11 +471,6 @@ Libraries options Build the :mod:`pyexpat` module using an installed ``expat`` library (default is no). -.. cmdoption:: --with-system-ffi - - Build the :mod:`_ctypes` extension module using an installed ``ffi`` - library, see the :mod:`ctypes` module (default is system-dependent). - .. cmdoption:: --with-system-libmpdec Build the ``_decimal`` extension module using an installed ``mpdec`` diff --git a/Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst b/Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst new file mode 100644 index 000000000000..c924ab520f3d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-12-26-15-07-48.gh-issue-100540.l6ToSY.rst @@ -0,0 +1,8 @@ +Removed the ``--with-system-ffi`` ``configure`` option; ``libffi`` must +now always be supplied by the system on all non-Windows platforms. The +option has had no effect on non-Darwin platforms for several releases, and +in 3.11 only had the non-obvious effect of invoking ``pkg-config`` to +find ``libffi`` and never setting ``-DUSING_APPLE_OS_LIBFFI``. Now on +Darwin platforms ``configure`` will first check for the OS ``libffi`` and +then fall back to the same processing as other platforms if it is not +found. diff --git a/configure b/configure index 72299939892d..6afd1e9c367c 100755 --- a/configure +++ b/configure @@ -1064,7 +1064,6 @@ with_hash_algorithm with_tzpath with_libs with_system_expat -with_system_ffi with_system_libmpdec with_decimal_contextvar enable_loadable_sqlite_extensions @@ -1847,9 +1846,6 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-ffi build _ctypes module using an installed ffi library, - see Doc/library/ctypes.rst (default is - system-dependent) --with-system-libmpdec build _decimal module using an installed libmpdec library, see Doc/library/decimal.rst (default is no) --with-decimal-contextvar @@ -12516,43 +12512,77 @@ fi -# Check for use of the system libffi library -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-ffi" >&5 -$as_echo_n "checking for --with-system-ffi... " >&6; } +have_libffi=missing +if test "x$ac_sys_system" = xDarwin; then : -# Check whether --with-system_ffi was given. -if test "${with_system_ffi+set}" = set; then : - withval=$with_system_ffi; -fi + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS -if test "$ac_sys_system" = "Darwin" -then - case "$with_system_ffi" in - "") - with_system_ffi="no" - ;; - yes|no) - ;; - *) - as_fn_error $? "--with-system-ffi accepts no arguments" "$LINENO" 5 - ;; - esac - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_system_ffi" >&5 -$as_echo "$with_system_ffi" >&6; } + CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" + ac_fn_c_check_header_mongrel "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" +if test "x$ac_cv_header_ffi_h" = xyes; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_call in -lffi" >&5 +$as_echo_n "checking for ffi_call in -lffi... " >&6; } +if ${ac_cv_lib_ffi_ffi_call+:} false; then : + $as_echo_n "(cached) " >&6 else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - if test "$with_system_ffi" != "" - then - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with(out)-system-ffi is ignored on this platform" >&5 -$as_echo "$as_me: WARNING: --with(out)-system-ffi is ignored on this platform" >&2;} - fi - with_system_ffi="yes" + ac_check_lib_save_LIBS=$LIBS +LIBS="-lffi $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (); +int +main () +{ +return ffi_call (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_ffi_ffi_call=yes +else + ac_cv_lib_ffi_ffi_call=no fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 +$as_echo "$ac_cv_lib_ffi_ffi_call" >&6; } +if test "x$ac_cv_lib_ffi_ffi_call" = xyes; then : -have_libffi=missing -if test "x$with_system_ffi" = xyes; then : + have_libffi=yes + LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" + LIBFFI_LIBS="-lffi" + +fi + + +fi + + + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + +fi +if test "x$have_libffi" = xmissing; then : pkg_failed=no @@ -12762,80 +12792,6 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } have_libffi=yes -fi - -else - - if test "x$ac_sys_system" = xDarwin; then : - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" - ac_fn_c_check_header_mongrel "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" -if test "x$ac_cv_header_ffi_h" = xyes; then : - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_call in -lffi" >&5 -$as_echo_n "checking for ffi_call in -lffi... " >&6; } -if ${ac_cv_lib_ffi_ffi_call+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lffi $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char ffi_call (); -int -main () -{ -return ffi_call (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_ffi_ffi_call=yes -else - ac_cv_lib_ffi_ffi_call=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 -$as_echo "$ac_cv_lib_ffi_ffi_call" >&6; } -if test "x$ac_cv_lib_ffi_ffi_call" = xyes; then : - - have_libffi=yes - LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" - LIBFFI_LIBS="-lffi" - -else - have_libffi=no -fi - - -fi - - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - fi fi diff --git a/configure.ac b/configure.ac index d6499be4c007..48736649a98e 100644 --- a/configure.ac +++ b/configure.ac @@ -3705,36 +3705,22 @@ AS_VAR_IF([with_system_expat], [yes], [ AC_SUBST([LIBEXPAT_CFLAGS]) AC_SUBST([LIBEXPAT_INTERNAL]) -# Check for use of the system libffi library -AC_MSG_CHECKING(for --with-system-ffi) -AC_ARG_WITH(system_ffi, - AS_HELP_STRING([--with-system-ffi], [build _ctypes module using an installed ffi library, see Doc/library/ctypes.rst (default is system-dependent)]),,,) - -if test "$ac_sys_system" = "Darwin" -then - case "$with_system_ffi" in - "") - with_system_ffi="no" - ;; - yes|no) - ;; - *) - AC_MSG_ERROR([--with-system-ffi accepts no arguments]) - ;; - esac - AC_MSG_RESULT($with_system_ffi) -else - AC_MSG_RESULT(yes) - if test "$with_system_ffi" != "" - then - AC_MSG_WARN([--with(out)-system-ffi is ignored on this platform]) - fi - with_system_ffi="yes" -fi - dnl detect libffi have_libffi=missing -AS_VAR_IF([with_system_ffi], [yes], [ +AS_VAR_IF([ac_sys_system], [Darwin], [ + WITH_SAVE_ENV([ + CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" + AC_CHECK_HEADER([ffi.h], [ + AC_CHECK_LIB([ffi], [ffi_call], [ + dnl use ffi from SDK root + have_libffi=yes + LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" + LIBFFI_LIBS="-lffi" + ]) + ]) + ]) +]) +AS_VAR_IF([have_libffi], [missing], [ PKG_CHECK_MODULES([LIBFFI], [libffi], [have_libffi=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" @@ -3748,20 +3734,6 @@ AS_VAR_IF([with_system_ffi], [yes], [ ]) ]) ]) -], [ - AS_VAR_IF([ac_sys_system], [Darwin], [ - WITH_SAVE_ENV([ - CFLAGS="-I${SDKROOT}/usr/include/ffi $CFLAGS" - AC_CHECK_HEADER([ffi.h], [ - AC_CHECK_LIB([ffi], [ffi_call], [ - dnl use ffi from SDK root - have_libffi=yes - LIBFFI_CFLAGS="-I${SDKROOT}/usr/include/ffi -DUSING_APPLE_OS_LIBFFI=1" - LIBFFI_LIBS="-lffi" - ], [have_libffi=no]) - ]) - ]) - ]) ]) AS_VAR_IF([have_libffi], [yes], [ From webhook-mailer at python.org Wed Dec 28 14:38:58 2022 From: webhook-mailer at python.org (zware) Date: Wed, 28 Dec 2022 19:38:58 -0000 Subject: [Python-checkins] gh-100540: Remove unused Modules/_ctypes/libffi_osx/ (GH-100543) Message-ID: <mailman.3120.1672256339.3313.python-checkins@python.org> https://github.com/python/cpython/commit/025b5c37800ce7a0e487a522171b479bf8235267 commit: 025b5c37800ce7a0e487a522171b479bf8235267 branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-28T13:38:52-06:00 summary: gh-100540: Remove unused Modules/_ctypes/libffi_osx/ (GH-100543) It was an ancient, modified copy of libffi that has not been in use since GH-22855. files: D Modules/_ctypes/libffi_osx/LICENSE D Modules/_ctypes/libffi_osx/README D Modules/_ctypes/libffi_osx/README.pyobjc D Modules/_ctypes/libffi_osx/ffi.c D Modules/_ctypes/libffi_osx/include/ffi.h D Modules/_ctypes/libffi_osx/include/ffi_common.h D Modules/_ctypes/libffi_osx/include/fficonfig.h D Modules/_ctypes/libffi_osx/include/ffitarget.h D Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h D Modules/_ctypes/libffi_osx/include/x86-ffitarget.h D Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S D Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h D Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S D Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c D Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S D Modules/_ctypes/libffi_osx/types.c D Modules/_ctypes/libffi_osx/x86/darwin64.S D Modules/_ctypes/libffi_osx/x86/x86-darwin.S D Modules/_ctypes/libffi_osx/x86/x86-ffi64.c D Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c M Tools/c-analyzer/cpython/_parser.py M Tools/patchcheck/patchcheck.py diff --git a/Modules/_ctypes/libffi_osx/LICENSE b/Modules/_ctypes/libffi_osx/LICENSE deleted file mode 100644 index f591795152d6..000000000000 --- a/Modules/_ctypes/libffi_osx/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -libffi - Copyright (c) 1996-2003 Red Hat, Inc. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -``Software''), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. diff --git a/Modules/_ctypes/libffi_osx/README b/Modules/_ctypes/libffi_osx/README deleted file mode 100644 index 54f00e3ec1ff..000000000000 --- a/Modules/_ctypes/libffi_osx/README +++ /dev/null @@ -1,500 +0,0 @@ -This directory contains the libffi package, which is not part of GCC but -shipped with GCC as convenience. - -Status -====== - -libffi-2.00 has not been released yet! This is a development snapshot! - -libffi-1.20 was released on October 5, 1998. Check the libffi web -page for updates: <URL:http://sources.redhat.com/libffi/>. - - -What is libffi? -=============== - -Compilers for high level languages generate code that follow certain -conventions. These conventions are necessary, in part, for separate -compilation to work. One such convention is the "calling -convention". The "calling convention" is essentially a set of -assumptions made by the compiler about where function arguments will -be found on entry to a function. A "calling convention" also specifies -where the return value for a function is found. - -Some programs may not know at the time of compilation what arguments -are to be passed to a function. For instance, an interpreter may be -told at run-time about the number and types of arguments used to call -a given function. Libffi can be used in such programs to provide a -bridge from the interpreter program to compiled code. - -The libffi library provides a portable, high level programming -interface to various calling conventions. This allows a programmer to -call any function specified by a call interface description at run -time. - -Ffi stands for Foreign Function Interface. A foreign function -interface is the popular name for the interface that allows code -written in one language to call code written in another language. The -libffi library really only provides the lowest, machine dependent -layer of a fully featured foreign function interface. A layer must -exist above libffi that handles type conversions for values passed -between the two languages. - - -Supported Platforms and Prerequisites -===================================== - -Libffi has been ported to: - - SunOS 4.1.3 & Solaris 2.x (SPARC-V8, SPARC-V9) - - Irix 5.3 & 6.2 (System V/o32 & n32) - - Intel x86 - Linux (System V ABI) - - Alpha - Linux and OSF/1 - - m68k - Linux (System V ABI) - - PowerPC - Linux (System V ABI, Darwin, AIX) - - ARM - Linux (System V ABI) - -Libffi has been tested with the egcs 1.0.2 gcc compiler. Chances are -that other versions will work. Libffi has also been built and tested -with the SGI compiler tools. - -On PowerPC, the tests failed (see the note below). - -You must use GNU make to build libffi. SGI's make will not work. -Sun's probably won't either. - -If you port libffi to another platform, please let me know! I assume -that some will be easy (x86 NetBSD), and others will be more difficult -(HP). - - -Installing libffi -================= - -[Note: before actually performing any of these installation steps, - you may wish to read the "Platform Specific Notes" below.] - -First you must configure the distribution for your particular -system. Go to the directory you wish to build libffi in and run the -"configure" program found in the root directory of the libffi source -distribution. - -You may want to tell configure where to install the libffi library and -header files. To do that, use the --prefix configure switch. Libffi -will install under /usr/local by default. - -If you want to enable extra run-time debugging checks use the the ---enable-debug configure switch. This is useful when your program dies -mysteriously while using libffi. - -Another useful configure switch is --enable-purify-safety. Using this -will add some extra code which will suppress certain warnings when you -are using Purify with libffi. Only use this switch when using -Purify, as it will slow down the library. - -Configure has many other options. Use "configure --help" to see them all. - -Once configure has finished, type "make". Note that you must be using -GNU make. SGI's make will not work. Sun's probably won't either. -You can ftp GNU make from prep.ai.mit.edu:/pub/gnu. - -To ensure that libffi is working as advertised, type "make test". - -To install the library and header files, type "make install". - - -Using libffi -============ - - The Basics - ---------- - -Libffi assumes that you have a pointer to the function you wish to -call and that you know the number and types of arguments to pass it, -as well as the return type of the function. - -The first thing you must do is create an ffi_cif object that matches -the signature of the function you wish to call. The cif in ffi_cif -stands for Call InterFace. To prepare a call interface object, use the -following function: - -ffi_status ffi_prep_cif(ffi_cif *cif, ffi_abi abi, - unsigned int nargs, - ffi_type *rtype, ffi_type **atypes); - - CIF is a pointer to the call interface object you wish - to initialize. - - ABI is an enum that specifies the calling convention - to use for the call. FFI_DEFAULT_ABI defaults - to the system's native calling convention. Other - ABI's may be used with care. They are system - specific. - - NARGS is the number of arguments this function accepts. - libffi does not yet support vararg functions. - - RTYPE is a pointer to an ffi_type structure that represents - the return type of the function. Ffi_type objects - describe the types of values. libffi provides - ffi_type objects for many of the native C types: - signed int, unsigned int, signed char, unsigned char, - etc. There is also a pointer ffi_type object and - a void ffi_type. Use &ffi_type_void for functions that - don't return values. - - ATYPES is a vector of ffi_type pointers. ARGS must be NARGS long. - If NARGS is 0, this is ignored. - - -ffi_prep_cif will return a status code that you are responsible -for checking. It will be one of the following: - - FFI_OK - All is good. - - FFI_BAD_TYPEDEF - One of the ffi_type objects that ffi_prep_cif - came across is bad. - - -Before making the call, the VALUES vector should be initialized -with pointers to the appropriate argument values. - -To call the function using the initialized ffi_cif, use the -ffi_call function: - -void ffi_call(ffi_cif *cif, void *fn, void *rvalue, void **avalues); - - CIF is a pointer to the ffi_cif initialized specifically - for this function. - - FN is a pointer to the function you want to call. - - RVALUE is a pointer to a chunk of memory that is to hold the - result of the function call. Currently, it must be - at least one word in size (except for the n32 version - under Irix 6.x, which must be a pointer to an 8 byte - aligned value (a long long). It must also be at least - word aligned (depending on the return type, and the - system's alignment requirements). If RTYPE is - &ffi_type_void, this is ignored. If RVALUE is NULL, - the return value is discarded. - - AVALUES is a vector of void* that point to the memory locations - holding the argument values for a call. - If NARGS is 0, this is ignored. - - -If you are expecting a return value from FN it will have been stored -at RVALUE. - - - - An Example - ---------- - -Here is a trivial example that calls puts() a few times. - - #include <stdio.h> - #include <ffi.h> - - int main() - { - ffi_cif cif; - ffi_type *args[1]; - void *values[1]; - char *s; - int rc; - - /* Initialize the argument info vectors */ - args[0] = &ffi_type_uint; - values[0] = &s; - - /* Initialize the cif */ - if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, 1, - &ffi_type_uint, args) == FFI_OK) - { - s = "Hello World!"; - ffi_call(&cif, puts, &rc, values); - /* rc now holds the result of the call to puts */ - - /* values holds a pointer to the function's arg, so to - call puts() again all we need to do is change the - value of s */ - s = "This is cool!"; - ffi_call(&cif, puts, &rc, values); - } - - return 0; - } - - - - Aggregate Types - --------------- - -Although libffi has no special support for unions or bit-fields, it is -perfectly happy passing structures back and forth. You must first -describe the structure to libffi by creating a new ffi_type object -for it. Here is the definition of ffi_type: - - typedef struct _ffi_type - { - unsigned size; - short alignment; - short type; - struct _ffi_type **elements; - } ffi_type; - -All structures must have type set to FFI_TYPE_STRUCT. You may set -size and alignment to 0. These will be calculated and reset to the -appropriate values by ffi_prep_cif(). - -elements is a NULL terminated array of pointers to ffi_type objects -that describe the type of the structure elements. These may, in turn, -be structure elements. - -The following example initializes a ffi_type object representing the -tm struct from Linux's time.h: - - struct tm { - int tm_sec; - int tm_min; - int tm_hour; - int tm_mday; - int tm_mon; - int tm_year; - int tm_wday; - int tm_yday; - int tm_isdst; - /* Those are for future use. */ - long int __tm_gmtoff__; - __const char *__tm_zone__; - }; - - { - ffi_type tm_type; - ffi_type *tm_type_elements[12]; - int i; - - tm_type.size = tm_type.alignment = 0; - tm_type.elements = &tm_type_elements; - - for (i = 0; i < 9; i++) - tm_type_elements[i] = &ffi_type_sint; - - tm_type_elements[9] = &ffi_type_slong; - tm_type_elements[10] = &ffi_type_pointer; - tm_type_elements[11] = NULL; - - /* tm_type can now be used to represent tm argument types and - return types for ffi_prep_cif() */ - } - - - -Platform Specific Notes -======================= - - Intel x86 - --------- - -There are no known problems with the x86 port. - - Sun SPARC - SunOS 4.1.3 & Solaris 2.x - ------------------------------------- - -You must use GNU Make to build libffi on Sun platforms. - - MIPS - Irix 5.3 & 6.x - --------------------- - -Irix 6.2 and better supports three different calling conventions: o32, -n32 and n64. Currently, libffi only supports both o32 and n32 under -Irix 6.x, but only o32 under Irix 5.3. Libffi will automatically be -configured for whichever calling convention it was built for. - -By default, the configure script will try to build libffi with the GNU -development tools. To build libffi with the SGI development tools, set -the environment variable CC to either "cc -32" or "cc -n32" before -running configure under Irix 6.x (depending on whether you want an o32 -or n32 library), or just "cc" for Irix 5.3. - -With the n32 calling convention, when returning structures smaller -than 16 bytes, be sure to provide an RVALUE that is 8 byte aligned. -Here's one way of forcing this: - - double struct_storage[2]; - my_small_struct *s = (my_small_struct *) struct_storage; - /* Use s for RVALUE */ - -If you don't do this you are liable to get spurious bus errors. - -"long long" values are not supported yet. - -You must use GNU Make to build libffi on SGI platforms. - - ARM - System V ABI - ------------------ - -The ARM port was performed on a NetWinder running ARM Linux ELF -(2.0.31) and gcc 2.8.1. - - - - PowerPC System V ABI - -------------------- - -There are two `System V ABI's which libffi implements for PowerPC. -They differ only in how small structures are returned from functions. - -In the FFI_SYSV version, structures that are 8 bytes or smaller are -returned in registers. This is what GCC does when it is configured -for solaris, and is what the System V ABI I have (dated September -1995) says. - -In the FFI_GCC_SYSV version, all structures are returned the same way: -by passing a pointer as the first argument to the function. This is -what GCC does when it is configured for linux or a generic sysv -target. - -EGCS 1.0.1 (and probably other versions of EGCS/GCC) also has a -inconsistency with the SysV ABI: When a procedure is called with many -floating-point arguments, some of them get put on the stack. They are -all supposed to be stored in double-precision format, even if they are -only single-precision, but EGCS stores single-precision arguments as -single-precision anyway. This causes one test to fail (the `many -arguments' test). - - -What's With The Cryptic Comments? -================================= - -You might notice a number of cryptic comments in the code, delimited -by /*@ and @*/. These are annotations read by the program LCLint, a -tool for statically checking C programs. You can read all about it at -<http://larch-www.lcs.mit.edu:8001/larch/lclint/index.html>. - - -History -======= - -1.20 Oct-5-98 - Raffaele Sena produces ARM port. - -1.19 Oct-5-98 - Fixed x86 long double and long long return support. - m68k bug fixes from Andreas Schwab. - Patch for DU assembler compatibility for the Alpha from Richard - Henderson. - -1.18 Apr-17-98 - Bug fixes and MIPS configuration changes. - -1.17 Feb-24-98 - Bug fixes and m68k port from Andreas Schwab. PowerPC port from - Geoffrey Keating. Various bug x86, Sparc and MIPS bug fixes. - -1.16 Feb-11-98 - Richard Henderson produces Alpha port. - -1.15 Dec-4-97 - Fixed an n32 ABI bug. New libtool, auto* support. - -1.14 May-13-97 - libtool is now used to generate shared and static libraries. - Fixed a minor portability problem reported by Russ McManus - <mcmanr at eq.gs.com>. - -1.13 Dec-2-96 - Added --enable-purify-safety to keep Purify from complaining - about certain low level code. - Sparc fix for calling functions with < 6 args. - Linux x86 a.out fix. - -1.12 Nov-22-96 - Added missing ffi_type_void, needed for supporting void return - types. Fixed test case for non MIPS machines. Cygnus Support - is now Cygnus Solutions. - -1.11 Oct-30-96 - Added notes about GNU make. - -1.10 Oct-29-96 - Added configuration fix for non GNU compilers. - -1.09 Oct-29-96 - Added --enable-debug configure switch. Clean-ups based on LCLint - feedback. ffi_mips.h is always installed. Many configuration - fixes. Fixed ffitest.c for sparc builds. - -1.08 Oct-15-96 - Fixed n32 problem. Many clean-ups. - -1.07 Oct-14-96 - Gordon Irlam rewrites v8.S again. Bug fixes. - -1.06 Oct-14-96 - Gordon Irlam improved the sparc port. - -1.05 Oct-14-96 - Interface changes based on feedback. - -1.04 Oct-11-96 - Sparc port complete (modulo struct passing bug). - -1.03 Oct-10-96 - Passing struct args, and returning struct values works for - all architectures/calling conventions. Expanded tests. - -1.02 Oct-9-96 - Added SGI n32 support. Fixed bugs in both o32 and Linux support. - Added "make test". - -1.01 Oct-8-96 - Fixed float passing bug in mips version. Restructured some - of the code. Builds cleanly with SGI tools. - -1.00 Oct-7-96 - First release. No public announcement. - - -Authors & Credits -================= - -libffi was written by Anthony Green <green at cygnus.com>. - -Portions of libffi were derived from Gianni Mariani's free gencall -library for Silicon Graphics machines. - -The closure mechanism was designed and implemented by Kresten Krab -Thorup. - -The Sparc port was derived from code contributed by the fine folks at -Visible Decisions Inc <http://www.vdi.com>. Further enhancements were -made by Gordon Irlam at Cygnus Solutions <http://www.cygnus.com>. - -The Alpha port was written by Richard Henderson at Cygnus Solutions. - -Andreas Schwab ported libffi to m68k Linux and provided a number of -bug fixes. - -Geoffrey Keating ported libffi to the PowerPC. - -Raffaele Sena ported libffi to the ARM. - -Jesper Skov and Andrew Haley both did more than their fair share of -stepping through the code and tracking down bugs. - -Thanks also to Tom Tromey for bug fixes and configuration help. - -Thanks to Jim Blandy, who provided some useful feedback on the libffi -interface. - -If you have a problem, or have found a bug, please send a note to -green at cygnus.com. diff --git a/Modules/_ctypes/libffi_osx/README.pyobjc b/Modules/_ctypes/libffi_osx/README.pyobjc deleted file mode 100644 index 405d85fed2de..000000000000 --- a/Modules/_ctypes/libffi_osx/README.pyobjc +++ /dev/null @@ -1,5 +0,0 @@ -This directory contains a slightly modified version of libffi, extracted from -the GCC source-tree. - -The only modifications are those that are necessary to compile libffi using -the Apple provided compiler and outside of the GCC source tree. diff --git a/Modules/_ctypes/libffi_osx/ffi.c b/Modules/_ctypes/libffi_osx/ffi.c deleted file mode 100644 index 1776b795e2f8..000000000000 --- a/Modules/_ctypes/libffi_osx/ffi.c +++ /dev/null @@ -1,227 +0,0 @@ -/* ----------------------------------------------------------------------- - prep_cif.c - Copyright (c) 1996, 1998 Red Hat, Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdbool.h> -#include <stdlib.h> - -/* Round up to FFI_SIZEOF_ARG. */ -#define STACK_ARG_SIZE(x) ALIGN(x, FFI_SIZEOF_ARG) - -/* Perform machine independent initialization of aggregate type - specifications. */ - -static ffi_status -initialize_aggregate( -/*@out@*/ ffi_type* arg) -{ -/*@-usedef@*/ - ffi_type** ptr; - - if (arg == NULL || arg->elements == NULL || - arg->size != 0 || arg->alignment != 0) - return FFI_BAD_TYPEDEF; - - ptr = &(arg->elements[0]); - - while ((*ptr) != NULL) - { - if (((*ptr)->size == 0) && (initialize_aggregate(*ptr) != FFI_OK)) - return FFI_BAD_TYPEDEF; - - /* Perform a sanity check on the argument type */ - FFI_ASSERT_VALID_TYPE(*ptr); - -#ifdef POWERPC_DARWIN - int curalign = (*ptr)->alignment; - - if (ptr != &(arg->elements[0])) - { - if (curalign > 4 && curalign != 16) - curalign = 4; - } - - arg->size = ALIGN(arg->size, curalign); - arg->size += (*ptr)->size; - arg->alignment = (arg->alignment > curalign) ? - arg->alignment : curalign; -#else - arg->size = ALIGN(arg->size, (*ptr)->alignment); - arg->size += (*ptr)->size; - arg->alignment = (arg->alignment > (*ptr)->alignment) ? - arg->alignment : (*ptr)->alignment; -#endif - - ptr++; - } - - /* Structure size includes tail padding. This is important for - structures that fit in one register on ABIs like the PowerPC64 - Linux ABI that right justify small structs in a register. - It's also needed for nested structure layout, for example - struct A { long a; char b; }; struct B { struct A x; char y; }; - should find y at an offset of 2*sizeof(long) and result in a - total size of 3*sizeof(long). */ - arg->size = ALIGN(arg->size, arg->alignment); - - if (arg->size == 0) - return FFI_BAD_TYPEDEF; - - return FFI_OK; - -/*@=usedef@*/ -} - -#ifndef __CRIS__ -/* The CRIS ABI specifies structure elements to have byte - alignment only, so it completely overrides this functions, - which assumes "natural" alignment and padding. */ - -/* Perform machine independent ffi_cif preparation, then call - machine dependent routine. */ - -#if defined(X86_DARWIN) && !defined __x86_64__ - -static inline bool -struct_on_stack( - int size) -{ - if (size > 8) - return true; - - /* This is not what the ABI says, but is what is really implemented */ - switch (size) - { - case 1: - case 2: - case 4: - case 8: - return false; - - default: - return true; - } -} - -#endif // defined(X86_DARWIN) && !defined __x86_64__ - -// Arguments' ffi_type->alignment must be nonzero. -ffi_status -ffi_prep_cif( -/*@out@*/ /*@partial@*/ ffi_cif* cif, - ffi_abi abi, - unsigned int nargs, -/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, -/*@dependent@*/ ffi_type** atypes) -{ - unsigned int bytes = 0; - unsigned int i; - ffi_type** ptr; - - if (cif == NULL) - return FFI_BAD_TYPEDEF; - - if (abi <= FFI_FIRST_ABI || abi > FFI_DEFAULT_ABI) - return FFI_BAD_ABI; - - cif->abi = abi; - cif->arg_types = atypes; - cif->nargs = nargs; - cif->rtype = rtype; - cif->flags = 0; - - /* Initialize the return type if necessary */ - /*@-usedef@*/ - if ((cif->rtype->size == 0) && (initialize_aggregate(cif->rtype) != FFI_OK)) - return FFI_BAD_TYPEDEF; - /*@=usedef@*/ - - /* Perform a sanity check on the return type */ - FFI_ASSERT_VALID_TYPE(cif->rtype); - - /* x86-64 and s390 stack space allocation is handled in prep_machdep. */ -#if !defined M68K && !defined __x86_64__ && !defined S390 && !defined PA - /* Make space for the return structure pointer */ - if (cif->rtype->type == FFI_TYPE_STRUCT -#ifdef SPARC - && (cif->abi != FFI_V9 || cif->rtype->size > 32) -#endif -#ifdef X86_DARWIN - && (struct_on_stack(cif->rtype->size)) -#endif - ) - bytes = STACK_ARG_SIZE(sizeof(void*)); -#endif - - for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++) - { - /* Initialize any uninitialized aggregate type definitions */ - if (((*ptr)->size == 0) && (initialize_aggregate((*ptr)) != FFI_OK)) - return FFI_BAD_TYPEDEF; - - if ((*ptr)->alignment == 0) - return FFI_BAD_TYPEDEF; - - /* Perform a sanity check on the argument type, do this - check after the initialization. */ - FFI_ASSERT_VALID_TYPE(*ptr); - -#if defined(X86_DARWIN) - { - int align = (*ptr)->alignment; - - if (align > 4) - align = 4; - - if ((align - 1) & bytes) - bytes = ALIGN(bytes, align); - - bytes += STACK_ARG_SIZE((*ptr)->size); - } -#elif !defined __x86_64__ && !defined S390 && !defined PA -#ifdef SPARC - if (((*ptr)->type == FFI_TYPE_STRUCT - && ((*ptr)->size > 16 || cif->abi != FFI_V9)) - || ((*ptr)->type == FFI_TYPE_LONGDOUBLE - && cif->abi != FFI_V9)) - bytes += sizeof(void*); - else -#endif - { - /* Add any padding if necessary */ - if (((*ptr)->alignment - 1) & bytes) - bytes = ALIGN(bytes, (*ptr)->alignment); - - bytes += STACK_ARG_SIZE((*ptr)->size); - } -#endif - } - - cif->bytes = bytes; - - /* Perform machine dependent cif processing */ - return ffi_prep_cif_machdep(cif); -} -#endif /* not __CRIS__ */ diff --git a/Modules/_ctypes/libffi_osx/include/ffi.h b/Modules/_ctypes/libffi_osx/include/ffi.h deleted file mode 100644 index c104a5c89350..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ffi.h +++ /dev/null @@ -1,355 +0,0 @@ -/* -----------------------------------------------------------------*-C-*- - libffi PyOBJC - Copyright (c) 1996-2003 Red Hat, Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - - ----------------------------------------------------------------------- */ - -/* ------------------------------------------------------------------- - The basic API is described in the README file. - - The raw API is designed to bypass some of the argument packing - and unpacking on architectures for which it can be avoided. - - The closure API allows interpreted functions to be packaged up - inside a C function pointer, so that they can be called as C functions, - with no understanding on the client side that they are interpreted. - It can also be used in other cases in which it is necessary to package - up a user specified parameter and a function pointer as a single - function pointer. - - The closure API must be implemented in order to get its functionality, - e.g. for use by gij. Routines are provided to emulate the raw API - if the underlying platform doesn't allow faster implementation. - - More details on the raw and closure API can be found in: - - http://gcc.gnu.org/ml/java/1999-q3/msg00138.html - - and - - http://gcc.gnu.org/ml/java/1999-q3/msg00174.html - -------------------------------------------------------------------- */ - -#ifndef LIBFFI_H -#define LIBFFI_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* Specify which architecture libffi is configured for. */ -#ifdef MACOSX -# if defined(__i386__) || defined(__x86_64__) -# define X86_DARWIN -# elif defined(__ppc__) || defined(__ppc64__) -# define POWERPC_DARWIN -# else -# error "Unsupported MacOS X CPU type" -# endif -#else -#error "Unsupported OS type" -#endif - -/* ---- System configuration information --------------------------------- */ - -#include "ffitarget.h" -#include "fficonfig.h" - -#ifndef LIBFFI_ASM - -#include <stddef.h> -#include <limits.h> - -/* LONG_LONG_MAX is not always defined (not if STRICT_ANSI, for example). - But we can find it either under the correct ANSI name, or under GNU - C's internal name. */ -#ifdef LONG_LONG_MAX -# define FFI_LONG_LONG_MAX LONG_LONG_MAX -#else -# ifdef LLONG_MAX -# define FFI_LONG_LONG_MAX LLONG_MAX -# else -# ifdef __GNUC__ -# define FFI_LONG_LONG_MAX __LONG_LONG_MAX__ -# endif -# endif -#endif - -#if SCHAR_MAX == 127 -# define ffi_type_uchar ffi_type_uint8 -# define ffi_type_schar ffi_type_sint8 -#else -#error "char size not supported" -#endif - -#if SHRT_MAX == 32767 -# define ffi_type_ushort ffi_type_uint16 -# define ffi_type_sshort ffi_type_sint16 -#elif SHRT_MAX == 2147483647 -# define ffi_type_ushort ffi_type_uint32 -# define ffi_type_sshort ffi_type_sint32 -#else -#error "short size not supported" -#endif - -#if INT_MAX == 32767 -# define ffi_type_uint ffi_type_uint16 -# define ffi_type_sint ffi_type_sint16 -#elif INT_MAX == 2147483647 -# define ffi_type_uint ffi_type_uint32 -# define ffi_type_sint ffi_type_sint32 -#elif INT_MAX == 9223372036854775807 -# define ffi_type_uint ffi_type_uint64 -# define ffi_type_sint ffi_type_sint64 -#else -#error "int size not supported" -#endif - -#define ffi_type_ulong ffi_type_uint64 -#define ffi_type_slong ffi_type_sint64 - -#if LONG_MAX == 2147483647 -# if FFI_LONG_LONG_MAX != 9223372036854775807 -# error "no 64-bit data type supported" -# endif -#elif LONG_MAX != 9223372036854775807 -#error "long size not supported" -#endif - -/* The closure code assumes that this works on pointers, i.e. a size_t - can hold a pointer. */ - -typedef struct _ffi_type { - size_t size; - unsigned short alignment; - unsigned short type; -/*@null@*/ struct _ffi_type** elements; -} ffi_type; - -/* These are defined in types.c */ -extern ffi_type ffi_type_void; -extern ffi_type ffi_type_uint8; -extern ffi_type ffi_type_sint8; -extern ffi_type ffi_type_uint16; -extern ffi_type ffi_type_sint16; -extern ffi_type ffi_type_uint32; -extern ffi_type ffi_type_sint32; -extern ffi_type ffi_type_uint64; -extern ffi_type ffi_type_sint64; -extern ffi_type ffi_type_float; -extern ffi_type ffi_type_double; -extern ffi_type ffi_type_longdouble; -extern ffi_type ffi_type_pointer; - -typedef enum ffi_status { - FFI_OK = 0, - FFI_BAD_TYPEDEF, - FFI_BAD_ABI -} ffi_status; - -typedef unsigned FFI_TYPE; - -typedef struct ffi_cif { - ffi_abi abi; - unsigned nargs; -/*@dependent@*/ ffi_type** arg_types; -/*@dependent@*/ ffi_type* rtype; - unsigned bytes; - unsigned flags; -#ifdef FFI_EXTRA_CIF_FIELDS - FFI_EXTRA_CIF_FIELDS; -#endif -} ffi_cif; - -/* ---- Definitions for the raw API -------------------------------------- */ - -#ifndef FFI_SIZEOF_ARG -# if LONG_MAX == 2147483647 -# define FFI_SIZEOF_ARG 4 -# elif LONG_MAX == 9223372036854775807 -# define FFI_SIZEOF_ARG 8 -# endif -#endif - -typedef union { - ffi_sarg sint; - ffi_arg uint; - float flt; - char data[FFI_SIZEOF_ARG]; - void* ptr; -} ffi_raw; - -void -ffi_raw_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ ffi_raw* avalue); - -void -ffi_ptrarray_to_raw( - ffi_cif* cif, - void** args, - ffi_raw* raw); - -void -ffi_raw_to_ptrarray( - ffi_cif* cif, - ffi_raw* raw, - void** args); - -size_t -ffi_raw_size( - ffi_cif* cif); - -/* This is analogous to the raw API, except it uses Java parameter - packing, even on 64-bit machines. I.e. on 64-bit machines - longs and doubles are followed by an empty 64-bit word. */ -void -ffi_java_raw_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ ffi_raw* avalue); - -void -ffi_java_ptrarray_to_raw( - ffi_cif* cif, - void** args, - ffi_raw* raw); - -void -ffi_java_raw_to_ptrarray( - ffi_cif* cif, - ffi_raw* raw, - void** args); - -size_t -ffi_java_raw_size( - ffi_cif* cif); - -/* ---- Definitions for closures ----------------------------------------- */ - -#if FFI_CLOSURES - -typedef struct ffi_closure { - char tramp[FFI_TRAMPOLINE_SIZE]; - ffi_cif* cif; - void (*fun)(ffi_cif*,void*,void**,void*); - void* user_data; -} ffi_closure; - -ffi_status -ffi_prep_closure( - ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,void**,void*), - void* user_data); - -void ffi_closure_free(void *); -void *ffi_closure_alloc (size_t size, void **code); - -typedef struct ffi_raw_closure { - char tramp[FFI_TRAMPOLINE_SIZE]; - ffi_cif* cif; - -#if !FFI_NATIVE_RAW_API - /* if this is enabled, then a raw closure has the same layout - as a regular closure. We use this to install an intermediate - handler to do the transaltion, void** -> ffi_raw*. */ - void (*translate_args)(ffi_cif*,void*,void**,void*); - void* this_closure; -#endif - - void (*fun)(ffi_cif*,void*,ffi_raw*,void*); - void* user_data; -} ffi_raw_closure; - -ffi_status -ffi_prep_raw_closure( - ffi_raw_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,ffi_raw*,void*), - void* user_data); - -ffi_status -ffi_prep_java_raw_closure( - ffi_raw_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,ffi_raw*,void*), - void* user_data); - -#endif // FFI_CLOSURES - -/* ---- Public interface definition -------------------------------------- */ - -ffi_status -ffi_prep_cif( -/*@out@*/ /*@partial@*/ ffi_cif* cif, - ffi_abi abi, - unsigned int nargs, -/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, -/*@dependent@*/ ffi_type** atypes); - -void -ffi_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ void** avalue); - -/* Useful for eliminating compiler warnings */ -#define FFI_FN(f) ((void (*)(void))f) - -#endif // #ifndef LIBFFI_ASM -/* ---- Definitions shared with assembly code ---------------------------- */ - -/* If these change, update src/mips/ffitarget.h. */ -#define FFI_TYPE_VOID 0 -#define FFI_TYPE_INT 1 -#define FFI_TYPE_FLOAT 2 -#define FFI_TYPE_DOUBLE 3 - -#ifdef HAVE_LONG_DOUBLE -# define FFI_TYPE_LONGDOUBLE 4 -#else -# define FFI_TYPE_LONGDOUBLE FFI_TYPE_DOUBLE -#endif - -#define FFI_TYPE_UINT8 5 -#define FFI_TYPE_SINT8 6 -#define FFI_TYPE_UINT16 7 -#define FFI_TYPE_SINT16 8 -#define FFI_TYPE_UINT32 9 -#define FFI_TYPE_SINT32 10 -#define FFI_TYPE_UINT64 11 -#define FFI_TYPE_SINT64 12 -#define FFI_TYPE_STRUCT 13 -#define FFI_TYPE_POINTER 14 - -/* This should always refer to the last type code (for sanity checks) */ -#define FFI_TYPE_LAST FFI_TYPE_POINTER - -#ifdef __cplusplus -} -#endif - -#endif // #ifndef LIBFFI_H diff --git a/Modules/_ctypes/libffi_osx/include/ffi_common.h b/Modules/_ctypes/libffi_osx/include/ffi_common.h deleted file mode 100644 index 685a3580f4fe..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ffi_common.h +++ /dev/null @@ -1,102 +0,0 @@ -/* ----------------------------------------------------------------------- - ffi_common.h - Copyright (c) 1996 Red Hat, Inc. - - Common internal definitions and macros. Only necessary for building - libffi. - ----------------------------------------------------------------------- */ - -#ifndef FFI_COMMON_H -#define FFI_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "fficonfig.h" - -/* Do not move this. Some versions of AIX are very picky about where - this is positioned. */ -#ifdef __GNUC__ -# define alloca __builtin_alloca -#else -# if HAVE_ALLOCA_H -# include <alloca.h> -# else -# ifdef _AIX -# pragma alloca -# else -# ifndef alloca /* predefined by HP cc +Olibcalls */ -char* alloca(); -# endif -# endif -# endif -#endif - -/* Check for the existence of memcpy. */ -#if STDC_HEADERS -# include <string.h> -#else -# ifndef HAVE_MEMCPY -# define memcpy(d, s, n) bcopy((s), (d), (n)) -# endif -#endif - -/*#if defined(FFI_DEBUG) -#include <stdio.h> -#endif*/ - -#ifdef FFI_DEBUG -#include <stdio.h> - -/*@exits@*/ void -ffi_assert( -/*@temp@*/ char* expr, -/*@temp@*/ char* file, - int line); -void -ffi_stop_here(void); -void -ffi_type_test( -/*@temp@*/ /*@out@*/ ffi_type* a, -/*@temp@*/ char* file, - int line); - -# define FFI_ASSERT(x) ((x) ? (void)0 : ffi_assert(#x, __FILE__,__LINE__)) -# define FFI_ASSERT_AT(x, f, l) ((x) ? 0 : ffi_assert(#x, (f), (l))) -# define FFI_ASSERT_VALID_TYPE(x) ffi_type_test(x, __FILE__, __LINE__) -#else -# define FFI_ASSERT(x) -# define FFI_ASSERT_AT(x, f, l) -# define FFI_ASSERT_VALID_TYPE(x) -#endif // #ifdef FFI_DEBUG - -#define ALIGN(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1)) - -/* Perform machine dependent cif processing */ -ffi_status -ffi_prep_cif_machdep( - ffi_cif* cif); - -/* Extended cif, used in callback from assembly routine */ -typedef struct extended_cif { -/*@dependent@*/ ffi_cif* cif; -/*@dependent@*/ void* rvalue; -/*@dependent@*/ void** avalue; -} extended_cif; - -/* Terse sized type definitions. */ -typedef unsigned int UINT8 __attribute__((__mode__(__QI__))); -typedef signed int SINT8 __attribute__((__mode__(__QI__))); -typedef unsigned int UINT16 __attribute__((__mode__(__HI__))); -typedef signed int SINT16 __attribute__((__mode__(__HI__))); -typedef unsigned int UINT32 __attribute__((__mode__(__SI__))); -typedef signed int SINT32 __attribute__((__mode__(__SI__))); -typedef unsigned int UINT64 __attribute__((__mode__(__DI__))); -typedef signed int SINT64 __attribute__((__mode__(__DI__))); -typedef float FLOAT32; - -#ifdef __cplusplus -} -#endif - -#endif // #ifndef FFI_COMMON_H \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/fficonfig.h b/Modules/_ctypes/libffi_osx/include/fficonfig.h deleted file mode 100644 index 217249071dcf..000000000000 --- a/Modules/_ctypes/libffi_osx/include/fficonfig.h +++ /dev/null @@ -1,150 +0,0 @@ -/* Manually created fficonfig.h for Darwin on PowerPC or Intel - - This file is manually generated to do away with the need for autoconf and - therefore make it easier to cross-compile and build fat binaries. - - NOTE: This file was added by PyObjC. -*/ - -#ifndef MACOSX -#error "This file is only supported on Mac OS X" -#endif - -#if defined(__i386__) -# define BYTEORDER 1234 -# undef HOST_WORDS_BIG_ENDIAN -# undef WORDS_BIGENDIAN -# define SIZEOF_DOUBLE 8 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 - -#elif defined(__x86_64__) -# define BYTEORDER 1234 -# undef HOST_WORDS_BIG_ENDIAN -# undef WORDS_BIGENDIAN -# define SIZEOF_DOUBLE 8 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 - -#elif defined(__ppc__) -# define BYTEORDER 4321 -# define HOST_WORDS_BIG_ENDIAN 1 -# define WORDS_BIGENDIAN 1 -# define SIZEOF_DOUBLE 8 -# if __GNUC__ >= 4 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 -# else -# undef HAVE_LONG_DOUBLE -# define SIZEOF_LONG_DOUBLE 8 -# endif - -#elif defined(__ppc64__) -# define BYTEORDER 4321 -# define HOST_WORDS_BIG_ENDIAN 1 -# define WORDS_BIGENDIAN 1 -# define SIZEOF_DOUBLE 8 -# define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 - -#else -#error "Unknown CPU type" -#endif - -/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP - systems. This function is required for `alloca.c' support on those systems. */ -#undef CRAY_STACKSEG_END - -/* Define to 1 if using `alloca.c'. */ -/* #undef C_ALLOCA */ - -/* Define to the flags needed for the .section .eh_frame directive. */ -#define EH_FRAME_FLAGS "aw" - -/* Define this if you want extra debugging. */ -/* #undef FFI_DEBUG */ - -/* Define this is you do not want support for the raw API. */ -#define FFI_NO_RAW_API 1 - -/* Define this if you do not want support for aggregate types. */ -/* #undef FFI_NO_STRUCTS */ - -/* Define to 1 if you have `alloca', as a function or macro. */ -#define HAVE_ALLOCA 1 - -/* Define to 1 if you have <alloca.h> and it should be used (not on Ultrix). */ -#define HAVE_ALLOCA_H 1 - -/* Define if your assembler supports .register. */ -/* #undef HAVE_AS_REGISTER_PSEUDO_OP */ - -/* Define if your assembler and linker support unaligned PC relative relocs. */ -/* #undef HAVE_AS_SPARC_UA_PCREL */ - -/* Define to 1 if you have the `memcpy' function. */ -#define HAVE_MEMCPY 1 - -/* Define if mmap with MAP_ANON(YMOUS) works. */ -#define HAVE_MMAP_ANON 1 - -/* Define if mmap of /dev/zero works. */ -/* #undef HAVE_MMAP_DEV_ZERO */ - -/* Define if read-only mmap of a plain file works. */ -#define HAVE_MMAP_FILE 1 - -/* Define if .eh_frame sections should be read-only. */ -/* #undef HAVE_RO_EH_FRAME */ - -/* Define to 1 if your C compiler doesn't accept -c and -o together. */ -/* #undef NO_MINUS_C_MINUS_O */ - -/* Name of package */ -#define PACKAGE "libffi" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "http://gcc.gnu.org/bugs.html" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "libffi" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "libffi 2.1" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "libffi" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "2.1" - -/* If using the C implementation of alloca, define if you know the - direction of stack growth for your system; otherwise it will be - automatically deduced at run-time. - STACK_DIRECTION > 0 => grows toward higher addresses - STACK_DIRECTION < 0 => grows toward lower addresses - STACK_DIRECTION = 0 => direction of growth unknown */ -/* #undef STACK_DIRECTION */ - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Define this if you are using Purify and want to suppress spurious messages. */ -/* #undef USING_PURIFY */ - -/* Version number of package */ -#define VERSION "2.1-pyobjc" - -#ifdef HAVE_HIDDEN_VISIBILITY_ATTRIBUTE -# ifdef LIBFFI_ASM -# define FFI_HIDDEN(name) .hidden name -# else -# define FFI_HIDDEN __attribute__((visibility ("hidden"))) -# endif -#else -# ifdef LIBFFI_ASM -# define FFI_HIDDEN(name) -# else -# define FFI_HIDDEN -# endif -#endif \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/ffitarget.h b/Modules/_ctypes/libffi_osx/include/ffitarget.h deleted file mode 100644 index faaa30de6f6f..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ffitarget.h +++ /dev/null @@ -1,13 +0,0 @@ -/* Dispatch to the right ffitarget file. This file is PyObjC specific; in a - normal build, the build environment copies the file to the right location or - sets up the right include flags. We want to do neither because that would - make building fat binaries harder. -*/ - -#if defined(__i386__) || defined(__x86_64__) -#include "x86-ffitarget.h" -#elif defined(__ppc__) || defined(__ppc64__) -#include "ppc-ffitarget.h" -#else -#error "Unsupported CPU type" -#endif \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h b/Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h deleted file mode 100644 index 231842199056..000000000000 --- a/Modules/_ctypes/libffi_osx/include/ppc-ffitarget.h +++ /dev/null @@ -1,104 +0,0 @@ -/* -----------------------------------------------------------------*-C-*- - ppc-ffitarget.h - Copyright (c) 1996-2003 Red Hat, Inc. - Target configuration macros for PowerPC. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#ifndef LIBFFI_TARGET_H -#define LIBFFI_TARGET_H - -/* ---- System specific configurations ----------------------------------- */ - -#if (defined(POWERPC) && defined(__powerpc64__)) || \ - (defined(POWERPC_DARWIN) && defined(__ppc64__)) -#define POWERPC64 -#endif - -#ifndef LIBFFI_ASM - -typedef unsigned long ffi_arg; -typedef signed long ffi_sarg; - -typedef enum ffi_abi { - FFI_FIRST_ABI = 0, - -#ifdef POWERPC - FFI_SYSV, - FFI_GCC_SYSV, - FFI_LINUX64, -# ifdef POWERPC64 - FFI_DEFAULT_ABI = FFI_LINUX64, -# else - FFI_DEFAULT_ABI = FFI_GCC_SYSV, -# endif -#endif - -#ifdef POWERPC_AIX - FFI_AIX, - FFI_DARWIN, - FFI_DEFAULT_ABI = FFI_AIX, -#endif - -#ifdef POWERPC_DARWIN - FFI_AIX, - FFI_DARWIN, - FFI_DEFAULT_ABI = FFI_DARWIN, -#endif - -#ifdef POWERPC_FREEBSD - FFI_SYSV, - FFI_GCC_SYSV, - FFI_LINUX64, - FFI_DEFAULT_ABI = FFI_SYSV, -#endif - - FFI_LAST_ABI = FFI_DEFAULT_ABI + 1 -} ffi_abi; - -#endif // #ifndef LIBFFI_ASM - -/* ---- Definitions for closures ----------------------------------------- */ - -#define FFI_CLOSURES 1 -#define FFI_NATIVE_RAW_API 0 - -/* Needed for FFI_SYSV small structure returns. */ -#define FFI_SYSV_TYPE_SMALL_STRUCT (FFI_TYPE_LAST) - -#if defined(POWERPC64) /*|| defined(POWERPC_AIX)*/ -# define FFI_TRAMPOLINE_SIZE 48 -#elif defined(POWERPC_AIX) -# define FFI_TRAMPOLINE_SIZE 24 -#else -# define FFI_TRAMPOLINE_SIZE 40 -#endif - -#ifndef LIBFFI_ASM -# if defined(POWERPC_DARWIN) || defined(POWERPC_AIX) -typedef struct ffi_aix_trampoline_struct { - void* code_pointer; /* Pointer to ffi_closure_ASM */ - void* toc; /* TOC */ - void* static_chain; /* Pointer to closure */ -} ffi_aix_trampoline_struct; -# endif -#endif // #ifndef LIBFFI_ASM - -#endif // #ifndef LIBFFI_TARGET_H \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h b/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h deleted file mode 100644 index 55c2b6c50cd9..000000000000 --- a/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h +++ /dev/null @@ -1,88 +0,0 @@ -/* -----------------------------------------------------------------*-C-*- - x86-ffitarget.h - Copyright (c) 1996-2003 Red Hat, Inc. - Target configuration macros for x86 and x86-64. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - - ----------------------------------------------------------------------- */ - -#ifndef LIBFFI_TARGET_H -#define LIBFFI_TARGET_H - -/* ---- System specific configurations ----------------------------------- */ - -#if defined(X86_64) && defined(__i386__) -# undef X86_64 -# define X86 -#endif - -#if defined(__x86_64__) -# ifndef X86_64 -# define X86_64 -# endif -#endif - -/* ---- Generic type definitions ----------------------------------------- */ - -#ifndef LIBFFI_ASM - -typedef unsigned long ffi_arg; -typedef signed long ffi_sarg; - -typedef enum ffi_abi { - FFI_FIRST_ABI = 0, - - /* ---- Intel x86 Win32 ---------- */ -#ifdef X86_WIN32 - FFI_SYSV, - FFI_STDCALL, - /* TODO: Add fastcall support for the sake of completeness */ - FFI_DEFAULT_ABI = FFI_SYSV, -#endif - - /* ---- Intel x86 and AMD x86-64 - */ -#if !defined(X86_WIN32) && (defined(__i386__) || defined(__x86_64__)) - FFI_SYSV, - FFI_UNIX64, /* Unix variants all use the same ABI for x86-64 */ -# ifdef __i386__ - FFI_DEFAULT_ABI = FFI_SYSV, -# else - FFI_DEFAULT_ABI = FFI_UNIX64, -# endif -#endif - - FFI_LAST_ABI = FFI_DEFAULT_ABI + 1 -} ffi_abi; - -#endif // #ifndef LIBFFI_ASM - -/* ---- Definitions for closures ----------------------------------------- */ - -#define FFI_CLOSURES 1 - -#if defined(X86_64) || (defined(__x86_64__) && defined(X86_DARWIN)) -# define FFI_TRAMPOLINE_SIZE 24 -# define FFI_NATIVE_RAW_API 0 -#else -# define FFI_TRAMPOLINE_SIZE 10 -# define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ -#endif - -#endif // #ifndef LIBFFI_TARGET_H \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S b/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S deleted file mode 100644 index f143dbd28c3d..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.S +++ /dev/null @@ -1,365 +0,0 @@ -#if defined(__ppc__) || defined(__ppc64__) - -/* ----------------------------------------------------------------------- - ppc-darwin.S - Copyright (c) 2000 John Hornkvist - Copyright (c) 2004 Free Software Foundation, Inc. - - PowerPC Assembly glue. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define LIBFFI_ASM - -#include <fficonfig.h> -#include <ffi.h> -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - -.text - .align 2 -.globl _ffi_prep_args - -.text - .align 2 -.globl _ffi_call_DARWIN - -.text - .align 2 -_ffi_call_DARWIN: -LFB0: - mr r12,r8 /* We only need r12 until the call, - so it doesn't have to be saved. */ - -LFB1: - /* Save the old stack pointer as AP. */ - mr r8,r1 - -LCFI0: -#if defined(__ppc64__) - /* Allocate the stack space we need. - r4 (size of input data) - 48 bytes (linkage area) - 40 bytes (saved registers) - 8 bytes (extra FPR) - r4 + 96 bytes total - */ - - addi r4,r4,-96 // Add our overhead. - li r0,-32 // Align to 32 bytes. - and r4,r4,r0 -#endif - stgux r1,r1,r4 // Grow the stack. - mflr r9 - - /* Save registers we use. */ -#if defined(__ppc64__) - std r27,-40(r8) -#endif - stg r28,MODE_CHOICE(-16,-32)(r8) - stg r29,MODE_CHOICE(-12,-24)(r8) - stg r30,MODE_CHOICE(-8,-16)(r8) - stg r31,MODE_CHOICE(-4,-8)(r8) - stg r9,SF_RETURN(r8) /* return address */ -#if !defined(POWERPC_DARWIN) /* TOC unused in OS X */ - stg r2,MODE_CHOICE(20,40)(r1) -#endif - -LCFI1: -#if defined(__ppc64__) - mr r27,r3 // our extended_cif -#endif - /* Save arguments over call. */ - mr r31,r5 /* flags, */ - mr r30,r6 /* rvalue, */ - mr r29,r7 /* function address, */ - mr r28,r8 /* our AP. */ - -LCFI2: - /* Call ffi_prep_args. */ - mr r4,r1 - li r9,0 - mtctr r12 /* r12 holds address of _ffi_prep_args. */ - bctrl -#if !defined(POWERPC_DARWIN) /* TOC unused in OS X */ - lg r2,MODE_CHOICE(20,40)(r1) -#endif - - /* Now do the call. - Set up cr1 with bits 4-7 of the flags. */ - mtcrf 0x40,r31 - - /* Load all those argument registers. - We have set up a nice stack frame, just load it into registers. */ - lg r3,SF_ARG1(r1) - lg r4,SF_ARG2(r1) - lg r5,SF_ARG3(r1) - lg r6,SF_ARG4(r1) - nop - lg r7,SF_ARG5(r1) - lg r8,SF_ARG6(r1) - lg r9,SF_ARG7(r1) - lg r10,SF_ARG8(r1) - - /* Load all the FP registers. */ - bf 6,L2 /* No floats to load. */ -#if defined(__ppc64__) - lfd f1,MODE_CHOICE(-16,-40)-(14*8)(r28) - lfd f2,MODE_CHOICE(-16,-40)-(13*8)(r28) - lfd f3,MODE_CHOICE(-16,-40)-(12*8)(r28) - lfd f4,MODE_CHOICE(-16,-40)-(11*8)(r28) - nop - lfd f5,MODE_CHOICE(-16,-40)-(10*8)(r28) - lfd f6,MODE_CHOICE(-16,-40)-(9*8)(r28) - lfd f7,MODE_CHOICE(-16,-40)-(8*8)(r28) - lfd f8,MODE_CHOICE(-16,-40)-(7*8)(r28) - nop - lfd f9,MODE_CHOICE(-16,-40)-(6*8)(r28) - lfd f10,MODE_CHOICE(-16,-40)-(5*8)(r28) - lfd f11,MODE_CHOICE(-16,-40)-(4*8)(r28) - lfd f12,MODE_CHOICE(-16,-40)-(3*8)(r28) - nop - lfd f13,MODE_CHOICE(-16,-40)-(2*8)(r28) - lfd f14,MODE_CHOICE(-16,-40)-(1*8)(r28) -#elif defined(__ppc__) - lfd f1,MODE_CHOICE(-16,-40)-(13*8)(r28) - lfd f2,MODE_CHOICE(-16,-40)-(12*8)(r28) - lfd f3,MODE_CHOICE(-16,-40)-(11*8)(r28) - lfd f4,MODE_CHOICE(-16,-40)-(10*8)(r28) - nop - lfd f5,MODE_CHOICE(-16,-40)-(9*8)(r28) - lfd f6,MODE_CHOICE(-16,-40)-(8*8)(r28) - lfd f7,MODE_CHOICE(-16,-40)-(7*8)(r28) - lfd f8,MODE_CHOICE(-16,-40)-(6*8)(r28) - nop - lfd f9,MODE_CHOICE(-16,-40)-(5*8)(r28) - lfd f10,MODE_CHOICE(-16,-40)-(4*8)(r28) - lfd f11,MODE_CHOICE(-16,-40)-(3*8)(r28) - lfd f12,MODE_CHOICE(-16,-40)-(2*8)(r28) - nop - lfd f13,MODE_CHOICE(-16,-40)-(1*8)(r28) -#else -#error undefined architecture -#endif - -L2: - mr r12,r29 // Put the target address in r12 as specified. - mtctr r12 // Get the address to call into CTR. - nop - nop - bctrl // Make the call. - - // Deal with the return value. -#if defined(__ppc64__) - mtcrf 0x3,r31 // flags in cr6 and cr7 - bt 27,L(st_return_value) -#elif defined(__ppc__) - mtcrf 0x1,r31 // flags in cr7 -#else -#error undefined architecture -#endif - - bt 30,L(done_return_value) - bt 29,L(fp_return_value) - stg r3,0(r30) -#if defined(__ppc__) - bf 28,L(done_return_value) // Store the second long if necessary. - stg r4,4(r30) -#endif - // Fall through - -L(done_return_value): - lg r1,0(r1) // Restore stack pointer. - // Restore the registers we used. - lg r9,SF_RETURN(r1) // return address - lg r31,MODE_CHOICE(-4,-8)(r1) - mtlr r9 - lg r30,MODE_CHOICE(-8,-16)(r1) - lg r29,MODE_CHOICE(-12,-24)(r1) - lg r28,MODE_CHOICE(-16,-32)(r1) -#if defined(__ppc64__) - ld r27,-40(r1) -#endif - blr - -#if defined(__ppc64__) -L(st_return_value): - // Grow the stack enough to fit the registers. Leave room for 8 args - // to trample the 1st 8 slots in param area. - stgu r1,-SF_ROUND(280)(r1) // 64 + 104 + 48 + 64 - - // Store GPRs - std r3,SF_ARG9(r1) - std r4,SF_ARG10(r1) - std r5,SF_ARG11(r1) - std r6,SF_ARG12(r1) - nop - std r7,SF_ARG13(r1) - std r8,SF_ARG14(r1) - std r9,SF_ARG15(r1) - std r10,SF_ARG16(r1) - - // Store FPRs - nop - bf 26,L(call_struct_to_ram_form) - stfd f1,SF_ARG17(r1) - stfd f2,SF_ARG18(r1) - stfd f3,SF_ARG19(r1) - stfd f4,SF_ARG20(r1) - nop - stfd f5,SF_ARG21(r1) - stfd f6,SF_ARG22(r1) - stfd f7,SF_ARG23(r1) - stfd f8,SF_ARG24(r1) - nop - stfd f9,SF_ARG25(r1) - stfd f10,SF_ARG26(r1) - stfd f11,SF_ARG27(r1) - stfd f12,SF_ARG28(r1) - nop - stfd f13,SF_ARG29(r1) - -L(call_struct_to_ram_form): - ld r3,0(r27) // extended_cif->cif* - ld r3,16(r3) // ffi_cif->rtype* - addi r4,r1,SF_ARG9 // stored GPRs - addi r6,r1,SF_ARG17 // stored FPRs - li r5,0 // GPR size ptr (NULL) - li r7,0 // FPR size ptr (NULL) - li r8,0 // FPR count ptr (NULL) - li r10,0 // struct offset (NULL) - mr r9,r30 // return area - bl Lffi64_struct_to_ram_form$stub - lg r1,0(r1) // Restore stack pointer. - b L(done_return_value) -#endif - -L(fp_return_value): - /* Do we have long double to store? */ - bf 31,L(fd_return_value) - stfd f1,0(r30) - stfd f2,8(r30) - b L(done_return_value) - -L(fd_return_value): - /* Do we have double to store? */ - bf 28,L(float_return_value) - stfd f1,0(r30) - b L(done_return_value) - -L(float_return_value): - /* We only have a float to store. */ - stfs f1,0(r30) - b L(done_return_value) - -LFE1: -/* END(_ffi_call_DARWIN) */ - -/* Provide a null definition of _ffi_call_AIX. */ -.text - .align 2 -.globl _ffi_call_AIX -.text - .align 2 -_ffi_call_AIX: - blr -/* END(_ffi_call_AIX) */ - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 ; Length of Common Information Entry -LSCIE1: - .long 0x0 ; CIE Identifier Tag - .byte 0x1 ; CIE Version - .ascii "zR\0" ; CIE Augmentation - .byte 0x1 ; uleb128 0x1; CIE Code Alignment Factor - .byte 0x7c ; sleb128 -4; CIE Data Alignment Factor - .byte 0x41 ; CIE RA Column - .byte 0x1 ; uleb128 0x1; Augmentation size - .byte 0x10 ; FDE Encoding (pcrel) - .byte 0xc ; DW_CFA_def_cfa - .byte 0x1 ; uleb128 0x1 - .byte 0x0 ; uleb128 0x0 - .align LOG2_GPR_BYTES -LECIE1: -.globl _ffi_call_DARWIN.eh -_ffi_call_DARWIN.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 ; FDE Length - -LASFDE1: - .long LASFDE1-EH_frame1 ; FDE CIE offset - .g_long LFB0-. ; FDE initial location - .set L$set$3,LFE1-LFB0 - .g_long L$set$3 ; FDE address range - .byte 0x0 ; uleb128 0x0; Augmentation size - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$4,LCFI0-LFB1 - .long L$set$4 - .byte 0xd ; DW_CFA_def_cfa_register - .byte 0x08 ; uleb128 0x08 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$5,LCFI1-LCFI0 - .long L$set$5 - .byte 0x11 ; DW_CFA_offset_extended_sf - .byte 0x41 ; uleb128 0x41 - .byte 0x7e ; sleb128 -2 - .byte 0x9f ; DW_CFA_offset, column 0x1f - .byte 0x1 ; uleb128 0x1 - .byte 0x9e ; DW_CFA_offset, column 0x1e - .byte 0x2 ; uleb128 0x2 - .byte 0x9d ; DW_CFA_offset, column 0x1d - .byte 0x3 ; uleb128 0x3 - .byte 0x9c ; DW_CFA_offset, column 0x1c - .byte 0x4 ; uleb128 0x4 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$6,LCFI2-LCFI1 - .long L$set$6 - .byte 0xd ; DW_CFA_def_cfa_register - .byte 0x1c ; uleb128 0x1c - .align LOG2_GPR_BYTES -LEFDE1: - -#if defined(__ppc64__) -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi64_struct_to_ram_form$stub: - .indirect_symbol _ffi64_struct_to_ram_form - mflr r0 - bcl 20,31,LO$ffi64_struct_to_ram_form - -LO$ffi64_struct_to_ram_form: - mflr r11 - addis r11,r11,ha16(L_ffi64_struct_to_ram_form$lazy_ptr - LO$ffi64_struct_to_ram_form) - mtlr r0 - lgu r12,lo16(L_ffi64_struct_to_ram_form$lazy_ptr - LO$ffi64_struct_to_ram_form)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi64_struct_to_ram_form$lazy_ptr: - .indirect_symbol _ffi64_struct_to_ram_form - .g_long dyld_stub_binding_helper - -#endif // __ppc64__ -#endif // __ppc__ || __ppc64__ diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h b/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h deleted file mode 100644 index cf4bd50f93c3..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin.h +++ /dev/null @@ -1,85 +0,0 @@ -/* ----------------------------------------------------------------------- - ppc-darwin.h - Copyright (c) 2002, 2003, 2004, Free Software Foundation, - Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define L(x) x - -#define SF_ARG9 MODE_CHOICE(56,112) -#define SF_ARG10 MODE_CHOICE(60,120) -#define SF_ARG11 MODE_CHOICE(64,128) -#define SF_ARG12 MODE_CHOICE(68,136) -#define SF_ARG13 MODE_CHOICE(72,144) -#define SF_ARG14 MODE_CHOICE(76,152) -#define SF_ARG15 MODE_CHOICE(80,160) -#define SF_ARG16 MODE_CHOICE(84,168) -#define SF_ARG17 MODE_CHOICE(88,176) -#define SF_ARG18 MODE_CHOICE(92,184) -#define SF_ARG19 MODE_CHOICE(96,192) -#define SF_ARG20 MODE_CHOICE(100,200) -#define SF_ARG21 MODE_CHOICE(104,208) -#define SF_ARG22 MODE_CHOICE(108,216) -#define SF_ARG23 MODE_CHOICE(112,224) -#define SF_ARG24 MODE_CHOICE(116,232) -#define SF_ARG25 MODE_CHOICE(120,240) -#define SF_ARG26 MODE_CHOICE(124,248) -#define SF_ARG27 MODE_CHOICE(128,256) -#define SF_ARG28 MODE_CHOICE(132,264) -#define SF_ARG29 MODE_CHOICE(136,272) - -#define ASM_NEEDS_REGISTERS 4 -#define NUM_GPR_ARG_REGISTERS 8 -#define NUM_FPR_ARG_REGISTERS 13 - -#define FFI_TYPE_1_BYTE(x) ((x) == FFI_TYPE_UINT8 || (x) == FFI_TYPE_SINT8) -#define FFI_TYPE_2_BYTE(x) ((x) == FFI_TYPE_UINT16 || (x) == FFI_TYPE_SINT16) -#define FFI_TYPE_4_BYTE(x) \ - ((x) == FFI_TYPE_UINT32 || (x) == FFI_TYPE_SINT32 ||\ - (x) == FFI_TYPE_INT || (x) == FFI_TYPE_FLOAT) - -#if !defined(LIBFFI_ASM) - -enum { - FLAG_RETURNS_NOTHING = 1 << (31 - 30), // cr7 - FLAG_RETURNS_FP = 1 << (31 - 29), - FLAG_RETURNS_64BITS = 1 << (31 - 28), - FLAG_RETURNS_128BITS = 1 << (31 - 31), - - FLAG_RETURNS_STRUCT = 1 << (31 - 27), // cr6 - FLAG_STRUCT_CONTAINS_FP = 1 << (31 - 26), - - FLAG_ARG_NEEDS_COPY = 1 << (31 - 7), - FLAG_FP_ARGUMENTS = 1 << (31 - 6), // cr1.eq; specified by ABI - FLAG_4_GPR_ARGUMENTS = 1 << (31 - 5), - FLAG_RETVAL_REFERENCE = 1 << (31 - 4) -}; - -#if defined(__ppc64__) -void ffi64_struct_to_ram_form(const ffi_type*, const char*, unsigned int*, - const char*, unsigned int*, unsigned int*, char*, unsigned int*); -void ffi64_struct_to_reg_form(const ffi_type*, const char*, unsigned int*, - unsigned int*, char*, unsigned int*, char*, unsigned int*); -bool ffi64_stret_needs_ptr(const ffi_type* inType, - unsigned short*, unsigned short*); -#endif - -#endif // !defined(LIBFFI_ASM) \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S b/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S deleted file mode 100644 index c3d30c25254c..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-darwin_closure.S +++ /dev/null @@ -1,308 +0,0 @@ -#if defined(__ppc__) - -/* ----------------------------------------------------------------------- - ppc-darwin_closure.S - Copyright (c) 2002, 2003, 2004, Free Software Foundation, - Inc. based on ppc_closure.S - - PowerPC Assembly glue. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define LIBFFI_ASM - -#include <ffi.h> -#include <ppc-ffitarget.h> // for FFI_TRAMPOLINE_SIZE -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - - .file "ppc-darwin_closure.S" -.text - .align LOG2_GPR_BYTES - .globl _ffi_closure_ASM - -.text - .align LOG2_GPR_BYTES - -_ffi_closure_ASM: -LFB1: - mflr r0 // Save return address - stg r0,SF_RETURN(r1) - -LCFI0: - /* 24/48 bytes (Linkage Area) - 32/64 bytes (outgoing parameter area, always reserved) - 104 bytes (13*8 from FPR) - 16/32 bytes (result) - 176/232 total bytes */ - - /* skip over caller save area and keep stack aligned to 16/32. */ - stgu r1,-SF_ROUND(176)(r1) - -LCFI1: - /* We want to build up an area for the parameters passed - in registers. (both floating point and integer) */ - - /* 176/256 bytes (callee stack frame aligned to 16/32) - 24/48 bytes (caller linkage area) - 200/304 (start of caller parameter area aligned to 4/8) - */ - - /* Save GPRs 3 - 10 (aligned to 4/8) - in the parents outgoing area. */ - stg r3,200(r1) - stg r4,204(r1) - stg r5,208(r1) - stg r6,212(r1) - stg r7,216(r1) - stg r8,220(r1) - stg r9,224(r1) - stg r10,228(r1) - - /* Save FPRs 1 - 13. (aligned to 8) */ - stfd f1,56(r1) - stfd f2,64(r1) - stfd f3,72(r1) - stfd f4,80(r1) - stfd f5,88(r1) - stfd f6,96(r1) - stfd f7,104(r1) - stfd f8,112(r1) - stfd f9,120(r1) - stfd f10,128(r1) - stfd f11,136(r1) - stfd f12,144(r1) - stfd f13,152(r1) - - // Set up registers for the routine that actually does the work. - mr r3,r11 // context pointer from the trampoline - addi r4,r1,160 // result storage - addi r5,r1,200 // saved GPRs - addi r6,r1,56 // saved FPRs - bl Lffi_closure_helper_DARWIN$stub - - /* Now r3 contains the return type. Use it to look up in a table - so we know how to deal with each type. */ - addi r5,r1,160 // Copy result storage pointer. - bl Lget_ret_type0_addr // Get pointer to Lret_type0 into LR. - mflr r4 // Move to r4. - slwi r3,r3,4 // Multiply return type by 16. - add r3,r3,r4 // Add contents of table to table address. - mtctr r3 - bctr - -LFE1: -/* Each of the ret_typeX code fragments has to be exactly 16 bytes long - (4 instructions). For cache effectiveness we align to a 16 byte boundary - first. */ - .align 4 - nop - nop - nop - -Lget_ret_type0_addr: - blrl - -/* case FFI_TYPE_VOID */ -Lret_type0: - b Lfinish - nop - nop - nop - -/* case FFI_TYPE_INT */ -Lret_type1: - lwz r3,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_FLOAT */ -Lret_type2: - lfs f1,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_DOUBLE */ -Lret_type3: - lfd f1,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_LONGDOUBLE */ -Lret_type4: - lfd f1,0(r5) - lfd f2,8(r5) - b Lfinish - nop - -/* case FFI_TYPE_UINT8 */ -Lret_type5: - lbz r3,3(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_SINT8 */ -Lret_type6: - lbz r3,3(r5) - extsb r3,r3 - b Lfinish - nop - -/* case FFI_TYPE_UINT16 */ -Lret_type7: - lhz r3,2(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_SINT16 */ -Lret_type8: - lha r3,2(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_UINT32 */ -Lret_type9: // same as Lret_type1 - lwz r3,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_SINT32 */ -Lret_type10: // same as Lret_type1 - lwz r3,0(r5) - b Lfinish - nop - nop - -/* case FFI_TYPE_UINT64 */ -Lret_type11: - lwz r3,0(r5) - lwz r4,4(r5) - b Lfinish - nop - -/* case FFI_TYPE_SINT64 */ -Lret_type12: // same as Lret_type11 - lwz r3,0(r5) - lwz r4,4(r5) - b Lfinish - nop - -/* case FFI_TYPE_STRUCT */ -Lret_type13: - b Lfinish - nop - nop - nop - -/* End 16-byte aligned cases */ -/* case FFI_TYPE_POINTER */ -// This case assumes that FFI_TYPE_POINTER == FFI_TYPE_LAST. If more types -// are added in future, the following code will need to be updated and -// padded to 16 bytes. -Lret_type14: - lg r3,0(r5) - // fall through - -/* case done */ -Lfinish: - addi r1,r1,SF_ROUND(176) // Restore stack pointer. - lg r0,SF_RETURN(r1) // Restore return address. - mtlr r0 // Restore link register. - blr - -/* END(ffi_closure_ASM) */ - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 ; Length of Common Information Entry -LSCIE1: - .long 0x0 ; CIE Identifier Tag - .byte 0x1 ; CIE Version - .ascii "zR\0" ; CIE Augmentation - .byte 0x1 ; uleb128 0x1; CIE Code Alignment Factor - .byte 0x7c ; sleb128 -4; CIE Data Alignment Factor - .byte 0x41 ; CIE RA Column - .byte 0x1 ; uleb128 0x1; Augmentation size - .byte 0x10 ; FDE Encoding (pcrel) - .byte 0xc ; DW_CFA_def_cfa - .byte 0x1 ; uleb128 0x1 - .byte 0x0 ; uleb128 0x0 - .align LOG2_GPR_BYTES -LECIE1: -.globl _ffi_closure_ASM.eh -_ffi_closure_ASM.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 ; FDE Length - -LASFDE1: - .long LASFDE1-EH_frame1 ; FDE CIE offset - .g_long LFB1-. ; FDE initial location - .set L$set$3,LFE1-LFB1 - .g_long L$set$3 ; FDE address range - .byte 0x0 ; uleb128 0x0; Augmentation size - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$3,LCFI1-LCFI0 - .long L$set$3 - .byte 0xe ; DW_CFA_def_cfa_offset - .byte 176,1 ; uleb128 176 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$4,LCFI0-LFB1 - .long L$set$4 - .byte 0x11 ; DW_CFA_offset_extended_sf - .byte 0x41 ; uleb128 0x41 - .byte 0x7e ; sleb128 -2 - .align LOG2_GPR_BYTES - -LEFDE1: -.data - .align LOG2_GPR_BYTES -LDFCM0: -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi_closure_helper_DARWIN$stub: - .indirect_symbol _ffi_closure_helper_DARWIN - mflr r0 - bcl 20,31,LO$ffi_closure_helper_DARWIN - -LO$ffi_closure_helper_DARWIN: - mflr r11 - addis r11,r11,ha16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN) - mtlr r0 - lgu r12,lo16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi_closure_helper_DARWIN$lazy_ptr: - .indirect_symbol _ffi_closure_helper_DARWIN - .g_long dyld_stub_binding_helper - - -#endif // __ppc__ diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c deleted file mode 100644 index 8953d5fda358..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c +++ /dev/null @@ -1,1776 +0,0 @@ -#if defined(__ppc__) || defined(__ppc64__) - -/* ----------------------------------------------------------------------- - ffi.c - Copyright (c) 1998 Geoffrey Keating - - PowerPC Foreign Function Interface - - Darwin ABI support (c) 2001 John Hornkvist - AIX ABI support (c) 2002 Free Software Foundation, Inc. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdbool.h> -#include <stdio.h> -#include <stdlib.h> -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - -#if 0 -#if defined(POWERPC_DARWIN) -#include <libkern/OSCacheControl.h> // for sys_icache_invalidate() -#endif - -#else - -#pragma weak sys_icache_invalidate -extern void sys_icache_invalidate(void *start, size_t len); - -#endif - - -extern void ffi_closure_ASM(void); - -// The layout of a function descriptor. A C function pointer really -// points to one of these. -typedef struct aix_fd_struct { - void* code_pointer; - void* toc; -} aix_fd; - -/* ffi_prep_args is called by the assembly routine once stack space - has been allocated for the function's arguments. - - The stack layout we want looks like this: - - | Return address from ffi_call_DARWIN | higher addresses - |--------------------------------------------| - | Previous backchain pointer 4/8 | stack pointer here - |--------------------------------------------|-\ <<< on entry to - | Saved r28-r31 (4/8)*4 | | ffi_call_DARWIN - |--------------------------------------------| | - | Parameters (at least 8*(4/8)=32/64) | | (176) +112 - +288 - |--------------------------------------------| | - | Space for GPR2 4/8 | | - |--------------------------------------------| | stack | - | Reserved (4/8)*2 | | grows | - |--------------------------------------------| | down V - | Space for callee's LR 4/8 | | - |--------------------------------------------| | lower addresses - | Saved CR 4/8 | | - |--------------------------------------------| | stack pointer here - | Current backchain pointer 4/8 | | during - |--------------------------------------------|-/ <<< ffi_call_DARWIN - - Note: ppc64 CR is saved in the low word of a long on the stack. -*/ - -/*@-exportheader@*/ -void -ffi_prep_args( - extended_cif* inEcif, - unsigned *const stack) -/*@=exportheader@*/ -{ - /* Copy the ecif to a local var so we can trample the arg. - BC note: test this with GP later for possible problems... */ - volatile extended_cif* ecif = inEcif; - - const unsigned bytes = ecif->cif->bytes; - const unsigned flags = ecif->cif->flags; - - /* Cast the stack arg from int* to long*. sizeof(long) == 4 in 32-bit mode - and 8 in 64-bit mode. */ - unsigned long *const longStack = (unsigned long *const)stack; - - /* 'stacktop' points at the previous backchain pointer. */ -#if defined(__ppc64__) - // In ppc-darwin.s, an extra 96 bytes is reserved for the linkage area, - // saved registers, and an extra FPR. - unsigned long *const stacktop = - (unsigned long *)(unsigned long)((char*)longStack + bytes + 96); -#elif defined(__ppc__) - unsigned long *const stacktop = longStack + (bytes / sizeof(long)); -#else -#error undefined architecture -#endif - - /* 'fpr_base' points at the space for fpr1, and grows upwards as - we use FPR registers. */ - double* fpr_base = (double*)(stacktop - ASM_NEEDS_REGISTERS) - - NUM_FPR_ARG_REGISTERS; - -#if defined(__ppc64__) - // 64-bit saves an extra register, and uses an extra FPR. Knock fpr_base - // down a couple pegs. - fpr_base -= 2; -#endif - - unsigned int fparg_count = 0; - - /* 'next_arg' grows up as we put parameters in it. */ - unsigned long* next_arg = longStack + 6; /* 6 reserved positions. */ - - int i; - double double_tmp; - void** p_argv = ecif->avalue; - unsigned long gprvalue; - ffi_type** ptr = ecif->cif->arg_types; - - /* Check that everything starts aligned properly. */ - FFI_ASSERT(stack == SF_ROUND(stack)); - FFI_ASSERT(stacktop == SF_ROUND(stacktop)); - FFI_ASSERT(bytes == SF_ROUND(bytes)); - - /* Deal with return values that are actually pass-by-reference. - Rule: - Return values are referenced by r3, so r4 is the first parameter. */ - - if (flags & FLAG_RETVAL_REFERENCE) - *next_arg++ = (unsigned long)(char*)ecif->rvalue; - - /* Now for the arguments. */ - for (i = ecif->cif->nargs; i > 0; i--, ptr++, p_argv++) - { - switch ((*ptr)->type) - { - /* If a floating-point parameter appears before all of the general- - purpose registers are filled, the corresponding GPRs that match - the size of the floating-point parameter are shadowed for the - benefit of vararg and pre-ANSI functions. */ - case FFI_TYPE_FLOAT: - double_tmp = *(float*)*p_argv; - - if (fparg_count < NUM_FPR_ARG_REGISTERS) - *fpr_base++ = double_tmp; - - *(double*)next_arg = double_tmp; - - next_arg++; - fparg_count++; - FFI_ASSERT(flags & FLAG_FP_ARGUMENTS); - - break; - - case FFI_TYPE_DOUBLE: - double_tmp = *(double*)*p_argv; - - if (fparg_count < NUM_FPR_ARG_REGISTERS) - *fpr_base++ = double_tmp; - - *(double*)next_arg = double_tmp; - - next_arg += MODE_CHOICE(2,1); - fparg_count++; - FFI_ASSERT(flags & FLAG_FP_ARGUMENTS); - - break; - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - case FFI_TYPE_LONGDOUBLE: -#if defined(__ppc64__) - if (fparg_count < NUM_FPR_ARG_REGISTERS) - *(long double*)fpr_base = *(long double*)*p_argv; -#elif defined(__ppc__) - if (fparg_count < NUM_FPR_ARG_REGISTERS - 1) - *(long double*)fpr_base = *(long double*)*p_argv; - else if (fparg_count == NUM_FPR_ARG_REGISTERS - 1) - *(double*)fpr_base = *(double*)*p_argv; -#else -#error undefined architecture -#endif - - *(long double*)next_arg = *(long double*)*p_argv; - fparg_count += 2; - fpr_base += 2; - next_arg += MODE_CHOICE(4,2); - FFI_ASSERT(flags & FLAG_FP_ARGUMENTS); - - break; -#endif // FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: -#if defined(__ppc64__) - gprvalue = *(long long*)*p_argv; - goto putgpr; -#elif defined(__ppc__) - *(long long*)next_arg = *(long long*)*p_argv; - next_arg += 2; - break; -#else -#error undefined architecture -#endif - - case FFI_TYPE_POINTER: - gprvalue = *(unsigned long*)*p_argv; - goto putgpr; - - case FFI_TYPE_UINT8: - gprvalue = *(unsigned char*)*p_argv; - goto putgpr; - - case FFI_TYPE_SINT8: - gprvalue = *(signed char*)*p_argv; - goto putgpr; - - case FFI_TYPE_UINT16: - gprvalue = *(unsigned short*)*p_argv; - goto putgpr; - - case FFI_TYPE_SINT16: - gprvalue = *(signed short*)*p_argv; - goto putgpr; - - case FFI_TYPE_STRUCT: - { -#if defined(__ppc64__) - unsigned int gprSize = 0; - unsigned int fprSize = 0; - - ffi64_struct_to_reg_form(*ptr, (char*)*p_argv, NULL, &fparg_count, - (char*)next_arg, &gprSize, (char*)fpr_base, &fprSize); - next_arg += gprSize / sizeof(long); - fpr_base += fprSize / sizeof(double); - -#elif defined(__ppc__) - char* dest_cpy = (char*)next_arg; - - /* Structures that match the basic modes (QI 1 byte, HI 2 bytes, - SI 4 bytes) are aligned as if they were those modes. - Structures with 3 byte in size are padded upwards. */ - unsigned size_al = (*ptr)->size; - - /* If the first member of the struct is a double, then align - the struct to double-word. */ - if ((*ptr)->elements[0]->type == FFI_TYPE_DOUBLE) - size_al = ALIGN((*ptr)->size, 8); - - if (ecif->cif->abi == FFI_DARWIN) - { - if (size_al < 3) - dest_cpy += 4 - size_al; - } - - memcpy((char*)dest_cpy, (char*)*p_argv, size_al); - next_arg += (size_al + 3) / 4; -#else -#error undefined architecture -#endif - break; - } - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - gprvalue = *(unsigned*)*p_argv; - -putgpr: - *next_arg++ = gprvalue; - break; - - default: - break; - } - } - - /* Check that we didn't overrun the stack... */ - //FFI_ASSERT(gpr_base <= stacktop - ASM_NEEDS_REGISTERS); - //FFI_ASSERT((unsigned *)fpr_base - // <= stacktop - ASM_NEEDS_REGISTERS - NUM_GPR_ARG_REGISTERS); - //FFI_ASSERT(flags & FLAG_4_GPR_ARGUMENTS || intarg_count <= 4); -} - -#if defined(__ppc64__) - -bool -ffi64_struct_contains_fp( - const ffi_type* inType) -{ - bool containsFP = false; - unsigned int i; - - for (i = 0; inType->elements[i] != NULL && !containsFP; i++) - { - if (inType->elements[i]->type == FFI_TYPE_FLOAT || - inType->elements[i]->type == FFI_TYPE_DOUBLE || - inType->elements[i]->type == FFI_TYPE_LONGDOUBLE) - containsFP = true; - else if (inType->elements[i]->type == FFI_TYPE_STRUCT) - containsFP = ffi64_struct_contains_fp(inType->elements[i]); - } - - return containsFP; -} - -#endif // defined(__ppc64__) - -/* Perform machine dependent cif processing. */ -ffi_status -ffi_prep_cif_machdep( - ffi_cif* cif) -{ - /* All this is for the DARWIN ABI. */ - int i; - ffi_type** ptr; - int intarg_count = 0; - int fparg_count = 0; - unsigned int flags = 0; - unsigned int size_al = 0; - - /* All the machine-independent calculation of cif->bytes will be wrong. - Redo the calculation for DARWIN. */ - - /* Space for the frame pointer, callee's LR, CR, etc, and for - the asm's temp regs. */ - unsigned int bytes = (6 + ASM_NEEDS_REGISTERS) * sizeof(long); - - /* Return value handling. The rules are as follows: - - 32-bit (or less) integer values are returned in gpr3; - - Structures of size <= 4 bytes also returned in gpr3; - - 64-bit integer values and structures between 5 and 8 bytes are - returned in gpr3 and gpr4; - - Single/double FP values are returned in fpr1; - - Long double FP (if not equivalent to double) values are returned in - fpr1 and fpr2; - - Larger structures values are allocated space and a pointer is passed - as the first argument. */ - switch (cif->rtype->type) - { -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - case FFI_TYPE_LONGDOUBLE: - flags |= FLAG_RETURNS_128BITS; - flags |= FLAG_RETURNS_FP; - break; -#endif // FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_DOUBLE: - flags |= FLAG_RETURNS_64BITS; - /* Fall through. */ - case FFI_TYPE_FLOAT: - flags |= FLAG_RETURNS_FP; - break; - -#if defined(__ppc64__) - case FFI_TYPE_POINTER: -#endif - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - flags |= FLAG_RETURNS_64BITS; - break; - - case FFI_TYPE_STRUCT: - { -#if defined(__ppc64__) - - if (ffi64_stret_needs_ptr(cif->rtype, NULL, NULL)) - { - flags |= FLAG_RETVAL_REFERENCE; - flags |= FLAG_RETURNS_NOTHING; - intarg_count++; - } - else - { - flags |= FLAG_RETURNS_STRUCT; - - if (ffi64_struct_contains_fp(cif->rtype)) - flags |= FLAG_STRUCT_CONTAINS_FP; - } - -#elif defined(__ppc__) - - flags |= FLAG_RETVAL_REFERENCE; - flags |= FLAG_RETURNS_NOTHING; - intarg_count++; - -#else -#error undefined architecture -#endif - break; - } - - case FFI_TYPE_VOID: - flags |= FLAG_RETURNS_NOTHING; - break; - - default: - /* Returns 32-bit integer, or similar. Nothing to do here. */ - break; - } - - /* The first NUM_GPR_ARG_REGISTERS words of integer arguments, and the - first NUM_FPR_ARG_REGISTERS fp arguments, go in registers; the rest - goes on the stack. Structures are passed as a pointer to a copy of - the structure. Stuff on the stack needs to keep proper alignment. */ - for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++) - { - switch ((*ptr)->type) - { - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - fparg_count++; - /* If this FP arg is going on the stack, it must be - 8-byte-aligned. */ - if (fparg_count > NUM_FPR_ARG_REGISTERS - && intarg_count % 2 != 0) - intarg_count++; - break; - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - case FFI_TYPE_LONGDOUBLE: - fparg_count += 2; - /* If this FP arg is going on the stack, it must be - 8-byte-aligned. */ - - if ( -#if defined(__ppc64__) - fparg_count > NUM_FPR_ARG_REGISTERS + 1 -#elif defined(__ppc__) - fparg_count > NUM_FPR_ARG_REGISTERS -#else -#error undefined architecture -#endif - && intarg_count % 2 != 0) - intarg_count++; - - intarg_count += 2; - break; -#endif // FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - /* 'long long' arguments are passed as two words, but - either both words must fit in registers or both go - on the stack. If they go on the stack, they must - be 8-byte-aligned. */ - if (intarg_count == NUM_GPR_ARG_REGISTERS - 1 - || (intarg_count >= NUM_GPR_ARG_REGISTERS - && intarg_count % 2 != 0)) - intarg_count++; - - intarg_count += MODE_CHOICE(2,1); - - break; - - case FFI_TYPE_STRUCT: - size_al = (*ptr)->size; - /* If the first member of the struct is a double, then align - the struct to double-word. */ - if ((*ptr)->elements[0]->type == FFI_TYPE_DOUBLE) - size_al = ALIGN((*ptr)->size, 8); - -#if defined(__ppc64__) - // Look for FP struct members. - unsigned int j; - - for (j = 0; (*ptr)->elements[j] != NULL; j++) - { - if ((*ptr)->elements[j]->type == FFI_TYPE_FLOAT || - (*ptr)->elements[j]->type == FFI_TYPE_DOUBLE) - { - fparg_count++; - - if (fparg_count > NUM_FPR_ARG_REGISTERS) - intarg_count++; - } - else if ((*ptr)->elements[j]->type == FFI_TYPE_LONGDOUBLE) - { - fparg_count += 2; - - if (fparg_count > NUM_FPR_ARG_REGISTERS + 1) - intarg_count += 2; - } - else - intarg_count++; - } -#elif defined(__ppc__) - intarg_count += (size_al + 3) / 4; -#else -#error undefined architecture -#endif - - break; - - default: - /* Everything else is passed as a 4/8-byte word in a GPR, either - the object itself or a pointer to it. */ - intarg_count++; - break; - } - } - - /* Space for the FPR registers, if needed. */ - if (fparg_count != 0) - { - flags |= FLAG_FP_ARGUMENTS; -#if defined(__ppc64__) - bytes += (NUM_FPR_ARG_REGISTERS + 1) * sizeof(double); -#elif defined(__ppc__) - bytes += NUM_FPR_ARG_REGISTERS * sizeof(double); -#else -#error undefined architecture -#endif - } - - /* Stack space. */ -#if defined(__ppc64__) - if ((intarg_count + fparg_count) > NUM_GPR_ARG_REGISTERS) - bytes += (intarg_count + fparg_count) * sizeof(long); -#elif defined(__ppc__) - if ((intarg_count + 2 * fparg_count) > NUM_GPR_ARG_REGISTERS) - bytes += (intarg_count + 2 * fparg_count) * sizeof(long); -#else -#error undefined architecture -#endif - else - bytes += NUM_GPR_ARG_REGISTERS * sizeof(long); - - /* The stack space allocated needs to be a multiple of 16/32 bytes. */ - bytes = SF_ROUND(bytes); - - cif->flags = flags; - cif->bytes = bytes; - - return FFI_OK; -} - -/*@-declundef@*/ -/*@-exportheader@*/ -extern void -ffi_call_AIX( -/*@out@*/ extended_cif*, - unsigned, - unsigned, -/*@out@*/ unsigned*, - void (*fn)(void), - void (*fn2)(extended_cif*, unsigned *const)); - -extern void -ffi_call_DARWIN( -/*@out@*/ extended_cif*, - unsigned long, - unsigned, -/*@out@*/ unsigned*, - void (*fn)(void), - void (*fn2)(extended_cif*, unsigned *const)); -/*@=declundef@*/ -/*@=exportheader@*/ - -void -ffi_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, -/*@dependent@*/ void** avalue) -{ - extended_cif ecif; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return - value address then we need to make one. */ - if ((rvalue == NULL) && - (cif->rtype->type == FFI_TYPE_STRUCT)) - { - /*@-sysunrecog@*/ - ecif.rvalue = alloca(cif->rtype->size); - /*@=sysunrecog@*/ - } - else - ecif.rvalue = rvalue; - - switch (cif->abi) - { - case FFI_AIX: - /*@-usedef@*/ - ffi_call_AIX(&ecif, -cif->bytes, - cif->flags, ecif.rvalue, fn, ffi_prep_args); - /*@=usedef@*/ - break; - - case FFI_DARWIN: - /*@-usedef@*/ - ffi_call_DARWIN(&ecif, -(long)cif->bytes, - cif->flags, ecif.rvalue, fn, ffi_prep_args); - /*@=usedef@*/ - break; - - default: - FFI_ASSERT(0); - break; - } -} - -/* here I'd like to add the stack frame layout we use in darwin_closure.S - and aix_clsoure.S - - SP previous -> +---------------------------------------+ <--- child frame - | back chain to caller 4 | - +---------------------------------------+ 4 - | saved CR 4 | - +---------------------------------------+ 8 - | saved LR 4 | - +---------------------------------------+ 12 - | reserved for compilers 4 | - +---------------------------------------+ 16 - | reserved for binders 4 | - +---------------------------------------+ 20 - | saved TOC pointer 4 | - +---------------------------------------+ 24 - | always reserved 8*4=32 (previous GPRs)| - | according to the linkage convention | - | from AIX | - +---------------------------------------+ 56 - | our FPR area 13*8=104 | - | f1 | - | . | - | f13 | - +---------------------------------------+ 160 - | result area 8 | - +---------------------------------------+ 168 - | alignement to the next multiple of 16 | -SP current --> +---------------------------------------+ 176 <- parent frame - | back chain to caller 4 | - +---------------------------------------+ 180 - | saved CR 4 | - +---------------------------------------+ 184 - | saved LR 4 | - +---------------------------------------+ 188 - | reserved for compilers 4 | - +---------------------------------------+ 192 - | reserved for binders 4 | - +---------------------------------------+ 196 - | saved TOC pointer 4 | - +---------------------------------------+ 200 - | always reserved 8*4=32 we store our | - | GPRs here | - | r3 | - | . | - | r10 | - +---------------------------------------+ 232 - | overflow part | - +---------------------------------------+ xxx - | ???? | - +---------------------------------------+ xxx -*/ - -#if !defined(POWERPC_DARWIN) - -#define MIN_LINE_SIZE 32 - -static void -flush_icache( - char* addr) -{ -#ifndef _AIX - __asm__ volatile ( - "dcbf 0,%0\n" - "sync\n" - "icbi 0,%0\n" - "sync\n" - "isync" - : : "r" (addr) : "memory"); -#endif -} - -static void -flush_range( - char* addr, - int size) -{ - int i; - - for (i = 0; i < size; i += MIN_LINE_SIZE) - flush_icache(addr + i); - - flush_icache(addr + size - 1); -} - -#endif // !defined(POWERPC_DARWIN) - -ffi_status -ffi_prep_closure( - ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*, void*, void**, void*), - void* user_data) -{ - switch (cif->abi) - { - case FFI_DARWIN: - { - FFI_ASSERT (cif->abi == FFI_DARWIN); - - unsigned int* tramp = (unsigned int*)&closure->tramp[0]; - -#if defined(__ppc64__) - tramp[0] = 0x7c0802a6; // mflr r0 - tramp[1] = 0x429f0005; // bcl 20,31,+0x8 - tramp[2] = 0x7d6802a6; // mflr r11 - tramp[3] = 0x7c0803a6; // mtlr r0 - tramp[4] = 0xe98b0018; // ld r12,24(r11) - tramp[5] = 0x7d8903a6; // mtctr r12 - tramp[6] = 0xe96b0020; // ld r11,32(r11) - tramp[7] = 0x4e800420; // bctr - *(unsigned long*)&tramp[8] = (unsigned long)ffi_closure_ASM; - *(unsigned long*)&tramp[10] = (unsigned long)closure; -#elif defined(__ppc__) - tramp[0] = 0x7c0802a6; // mflr r0 - tramp[1] = 0x429f0005; // bcl 20,31,+0x8 - tramp[2] = 0x7d6802a6; // mflr r11 - tramp[3] = 0x7c0803a6; // mtlr r0 - tramp[4] = 0x818b0018; // lwz r12,24(r11) - tramp[5] = 0x7d8903a6; // mtctr r12 - tramp[6] = 0x816b001c; // lwz r11,28(r11) - tramp[7] = 0x4e800420; // bctr - tramp[8] = (unsigned long)ffi_closure_ASM; - tramp[9] = (unsigned long)closure; -#else -#error undefined architecture -#endif - - closure->cif = cif; - closure->fun = fun; - closure->user_data = user_data; - - // Flush the icache. Only necessary on Darwin. -#if defined(POWERPC_DARWIN) - sys_icache_invalidate(closure->tramp, FFI_TRAMPOLINE_SIZE); -#else - flush_range(closure->tramp, FFI_TRAMPOLINE_SIZE); -#endif - - break; - } - - case FFI_AIX: - { - FFI_ASSERT (cif->abi == FFI_AIX); - - ffi_aix_trampoline_struct* tramp_aix = - (ffi_aix_trampoline_struct*)(closure->tramp); - aix_fd* fd = (aix_fd*)(void*)ffi_closure_ASM; - - tramp_aix->code_pointer = fd->code_pointer; - tramp_aix->toc = fd->toc; - tramp_aix->static_chain = closure; - closure->cif = cif; - closure->fun = fun; - closure->user_data = user_data; - break; - } - - default: - return FFI_BAD_ABI; - } - - return FFI_OK; -} - -#if defined(__ppc__) - typedef double ldbits[2]; - - typedef union - { - ldbits lb; - long double ld; - } ldu; -#endif - -typedef union -{ - float f; - double d; -} ffi_dblfl; - -/* The trampoline invokes ffi_closure_ASM, and on entry, r11 holds the - address of the closure. After storing the registers that could possibly - contain parameters to be passed into the stack frame and setting up space - for a return value, ffi_closure_ASM invokes the following helper function - to do most of the work. */ -int -ffi_closure_helper_DARWIN( - ffi_closure* closure, - void* rvalue, - unsigned long* pgr, - ffi_dblfl* pfr) -{ - /* rvalue is the pointer to space for return value in closure assembly - pgr is the pointer to where r3-r10 are stored in ffi_closure_ASM - pfr is the pointer to where f1-f13 are stored in ffi_closure_ASM. */ - -#if defined(__ppc__) - ldu temp_ld; -#endif - - double temp; - unsigned int i; - unsigned int nf = 0; /* number of FPRs already used. */ - unsigned int ng = 0; /* number of GPRs already used. */ - ffi_cif* cif = closure->cif; - long avn = cif->nargs; - void** avalue = alloca(cif->nargs * sizeof(void*)); - ffi_type** arg_types = cif->arg_types; - - /* Copy the caller's structure return value address so that the closure - returns the data directly to the caller. */ -#if defined(__ppc64__) - if (cif->rtype->type == FFI_TYPE_STRUCT && - ffi64_stret_needs_ptr(cif->rtype, NULL, NULL)) -#elif defined(__ppc__) - if (cif->rtype->type == FFI_TYPE_STRUCT) -#else -#error undefined architecture -#endif - { - rvalue = (void*)*pgr; - pgr++; - ng++; - } - - /* Grab the addresses of the arguments from the stack frame. */ - for (i = 0; i < avn; i++) - { - switch (arg_types[i]->type) - { - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT8: - avalue[i] = (char*)pgr + MODE_CHOICE(3,7); - ng++; - pgr++; - break; - - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT16: - avalue[i] = (char*)pgr + MODE_CHOICE(2,6); - ng++; - pgr++; - break; - -#if defined(__ppc__) - case FFI_TYPE_POINTER: -#endif - case FFI_TYPE_SINT32: - case FFI_TYPE_UINT32: - avalue[i] = (char*)pgr + MODE_CHOICE(0,4); - ng++; - pgr++; - - break; - - case FFI_TYPE_STRUCT: - if (cif->abi == FFI_DARWIN) - { -#if defined(__ppc64__) - unsigned int gprSize = 0; - unsigned int fprSize = 0; - unsigned int savedFPRSize = fprSize; - - avalue[i] = alloca(arg_types[i]->size); - ffi64_struct_to_ram_form(arg_types[i], (const char*)pgr, - &gprSize, (const char*)pfr, &fprSize, &nf, avalue[i], NULL); - - ng += gprSize / sizeof(long); - pgr += gprSize / sizeof(long); - pfr += (fprSize - savedFPRSize) / sizeof(double); - -#elif defined(__ppc__) - /* Structures that match the basic modes (QI 1 byte, HI 2 bytes, - SI 4 bytes) are aligned as if they were those modes. */ - unsigned int size_al = size_al = arg_types[i]->size; - - /* If the first member of the struct is a double, then align - the struct to double-word. */ - if (arg_types[i]->elements[0]->type == FFI_TYPE_DOUBLE) - size_al = ALIGN(arg_types[i]->size, 8); - - if (size_al < 3) - avalue[i] = (void*)pgr + MODE_CHOICE(4,8) - size_al; - else - avalue[i] = (void*)pgr; - - ng += (size_al + 3) / sizeof(long); - pgr += (size_al + 3) / sizeof(long); -#else -#error undefined architecture -#endif - } - - break; - -#if defined(__ppc64__) - case FFI_TYPE_POINTER: -#endif - case FFI_TYPE_SINT64: - case FFI_TYPE_UINT64: - /* Long long ints are passed in 1 or 2 GPRs. */ - avalue[i] = pgr; - ng += MODE_CHOICE(2,1); - pgr += MODE_CHOICE(2,1); - - break; - - case FFI_TYPE_FLOAT: - /* A float value consumes a GPR. - There are 13 64-bit floating point registers. */ - if (nf < NUM_FPR_ARG_REGISTERS) - { - temp = pfr->d; - pfr->f = (float)temp; - avalue[i] = pfr; - pfr++; - } - else - avalue[i] = pgr; - - nf++; - ng++; - pgr++; - break; - - case FFI_TYPE_DOUBLE: - /* A double value consumes one or two GPRs. - There are 13 64bit floating point registers. */ - if (nf < NUM_FPR_ARG_REGISTERS) - { - avalue[i] = pfr; - pfr++; - } - else - avalue[i] = pgr; - - nf++; - ng += MODE_CHOICE(2,1); - pgr += MODE_CHOICE(2,1); - - break; - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - - case FFI_TYPE_LONGDOUBLE: -#if defined(__ppc64__) - if (nf < NUM_FPR_ARG_REGISTERS) - { - avalue[i] = pfr; - pfr += 2; - } -#elif defined(__ppc__) - /* A long double value consumes 2/4 GPRs and 2 FPRs. - There are 13 64bit floating point registers. */ - if (nf < NUM_FPR_ARG_REGISTERS - 1) - { - avalue[i] = pfr; - pfr += 2; - } - /* Here we have the situation where one part of the long double - is stored in fpr13 and the other part is already on the stack. - We use a union to pass the long double to avalue[i]. */ - else if (nf == NUM_FPR_ARG_REGISTERS - 1) - { - memcpy (&temp_ld.lb[0], pfr, sizeof(temp_ld.lb[0])); - memcpy (&temp_ld.lb[1], pgr + 2, sizeof(temp_ld.lb[1])); - avalue[i] = &temp_ld.ld; - } -#else -#error undefined architecture -#endif - else - avalue[i] = pgr; - - nf += 2; - ng += MODE_CHOICE(4,2); - pgr += MODE_CHOICE(4,2); - - break; - -#endif /* FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE */ - - default: - FFI_ASSERT(0); - break; - } - } - - (closure->fun)(cif, rvalue, avalue, closure->user_data); - - /* Tell ffi_closure_ASM to perform return type promotions. */ - return cif->rtype->type; -} - -#if defined(__ppc64__) - -/* ffi64_struct_to_ram_form - - Rebuild a struct's natural layout from buffers of concatenated registers. - Return the number of registers used. - inGPRs[0-7] == r3, inFPRs[0-7] == f1 ... -*/ -void -ffi64_struct_to_ram_form( - const ffi_type* inType, - const char* inGPRs, - unsigned int* ioGPRMarker, - const char* inFPRs, - unsigned int* ioFPRMarker, - unsigned int* ioFPRsUsed, - char* outStruct, // caller-allocated - unsigned int* ioStructMarker) -{ - unsigned int srcGMarker = 0; - unsigned int srcFMarker = 0; - unsigned int savedFMarker = 0; - unsigned int fprsUsed = 0; - unsigned int savedFPRsUsed = 0; - unsigned int destMarker = 0; - - static unsigned int recurseCount = 0; - - if (ioGPRMarker) - srcGMarker = *ioGPRMarker; - - if (ioFPRMarker) - { - srcFMarker = *ioFPRMarker; - savedFMarker = srcFMarker; - } - - if (ioFPRsUsed) - { - fprsUsed = *ioFPRsUsed; - savedFPRsUsed = fprsUsed; - } - - if (ioStructMarker) - destMarker = *ioStructMarker; - - size_t i; - - switch (inType->size) - { - case 1: case 2: case 4: - srcGMarker += 8 - inType->size; - break; - - default: - break; - } - - for (i = 0; inType->elements[i] != NULL; i++) - { - switch (inType->elements[i]->type) - { - case FFI_TYPE_FLOAT: - srcFMarker = ALIGN(srcFMarker, 4); - srcGMarker = ALIGN(srcGMarker, 4); - destMarker = ALIGN(destMarker, 4); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - *(float*)&outStruct[destMarker] = - (float)*(double*)&inFPRs[srcFMarker]; - srcFMarker += 8; - fprsUsed++; - } - else - *(float*)&outStruct[destMarker] = - (float)*(double*)&inGPRs[srcGMarker]; - - srcGMarker += 4; - destMarker += 4; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (destMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 4)) - srcGMarker = ALIGN(srcGMarker, 8); - } - - break; - - case FFI_TYPE_DOUBLE: - srcFMarker = ALIGN(srcFMarker, 8); - destMarker = ALIGN(destMarker, 8); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - *(double*)&outStruct[destMarker] = - *(double*)&inFPRs[srcFMarker]; - srcFMarker += 8; - fprsUsed++; - } - else - *(double*)&outStruct[destMarker] = - *(double*)&inGPRs[srcGMarker]; - - destMarker += 8; - - // Skip next GPR - srcGMarker += 8; - srcGMarker = ALIGN(srcGMarker, 8); - - break; - - case FFI_TYPE_LONGDOUBLE: - destMarker = ALIGN(destMarker, 16); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - srcFMarker = ALIGN(srcFMarker, 8); - srcGMarker = ALIGN(srcGMarker, 8); - *(long double*)&outStruct[destMarker] = - *(long double*)&inFPRs[srcFMarker]; - srcFMarker += 16; - fprsUsed += 2; - } - else - { - srcFMarker = ALIGN(srcFMarker, 16); - srcGMarker = ALIGN(srcGMarker, 16); - *(long double*)&outStruct[destMarker] = - *(long double*)&inGPRs[srcGMarker]; - } - - destMarker += 16; - - // Skip next 2 GPRs - srcGMarker += 16; - srcGMarker = ALIGN(srcGMarker, 8); - - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - { - if (inType->alignment == 1) // chars only - { - if (inType->size == 1) - outStruct[destMarker++] = inGPRs[srcGMarker++]; - else if (inType->size == 2) - { - outStruct[destMarker++] = inGPRs[srcGMarker++]; - outStruct[destMarker++] = inGPRs[srcGMarker++]; - i++; - } - else - { - memcpy(&outStruct[destMarker], - &inGPRs[srcGMarker], inType->size); - srcGMarker += inType->size; - destMarker += inType->size; - i += inType->size - 1; - } - } - else // chars and other stuff - { - outStruct[destMarker++] = inGPRs[srcGMarker++]; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (srcGMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(srcGMarker, 8) - srcGMarker) < 4)) - srcGMarker = ALIGN(srcGMarker, inType->alignment); // was 8 - } - } - - break; - } - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - srcGMarker = ALIGN(srcGMarker, 2); - destMarker = ALIGN(destMarker, 2); - - *(short*)&outStruct[destMarker] = - *(short*)&inGPRs[srcGMarker]; - srcGMarker += 2; - destMarker += 2; - - break; - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - srcGMarker = ALIGN(srcGMarker, 4); - destMarker = ALIGN(destMarker, 4); - - *(int*)&outStruct[destMarker] = - *(int*)&inGPRs[srcGMarker]; - srcGMarker += 4; - destMarker += 4; - - break; - - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - srcGMarker = ALIGN(srcGMarker, 8); - destMarker = ALIGN(destMarker, 8); - - *(long long*)&outStruct[destMarker] = - *(long long*)&inGPRs[srcGMarker]; - srcGMarker += 8; - destMarker += 8; - - break; - - case FFI_TYPE_STRUCT: - recurseCount++; - ffi64_struct_to_ram_form(inType->elements[i], inGPRs, - &srcGMarker, inFPRs, &srcFMarker, &fprsUsed, - outStruct, &destMarker); - recurseCount--; - break; - - default: - FFI_ASSERT(0); // unknown element type - break; - } - } - - srcGMarker = ALIGN(srcGMarker, inType->alignment); - - // Take care of the special case for 16-byte structs, but not for - // nested structs. - if (recurseCount == 0 && srcGMarker == 16) - { - *(long double*)&outStruct[0] = *(long double*)&inGPRs[0]; - srcFMarker = savedFMarker; - fprsUsed = savedFPRsUsed; - } - - if (ioGPRMarker) - *ioGPRMarker = ALIGN(srcGMarker, 8); - - if (ioFPRMarker) - *ioFPRMarker = srcFMarker; - - if (ioFPRsUsed) - *ioFPRsUsed = fprsUsed; - - if (ioStructMarker) - *ioStructMarker = ALIGN(destMarker, 8); -} - -/* ffi64_struct_to_reg_form - - Copy a struct's elements into buffers that can be sliced into registers. - Return the sizes of the output buffers in bytes. Pass NULL buffer pointers - to calculate size only. - outGPRs[0-7] == r3, outFPRs[0-7] == f1 ... -*/ -void -ffi64_struct_to_reg_form( - const ffi_type* inType, - const char* inStruct, - unsigned int* ioStructMarker, - unsigned int* ioFPRsUsed, - char* outGPRs, // caller-allocated - unsigned int* ioGPRSize, - char* outFPRs, // caller-allocated - unsigned int* ioFPRSize) -{ - size_t i; - unsigned int srcMarker = 0; - unsigned int destGMarker = 0; - unsigned int destFMarker = 0; - unsigned int savedFMarker = 0; - unsigned int fprsUsed = 0; - unsigned int savedFPRsUsed = 0; - - static unsigned int recurseCount = 0; - - if (ioStructMarker) - srcMarker = *ioStructMarker; - - if (ioFPRsUsed) - { - fprsUsed = *ioFPRsUsed; - savedFPRsUsed = fprsUsed; - } - - if (ioGPRSize) - destGMarker = *ioGPRSize; - - if (ioFPRSize) - { - destFMarker = *ioFPRSize; - savedFMarker = destFMarker; - } - - switch (inType->size) - { - case 1: case 2: case 4: - destGMarker += 8 - inType->size; - break; - - default: - break; - } - - for (i = 0; inType->elements[i] != NULL; i++) - { - switch (inType->elements[i]->type) - { - // Shadow floating-point types in GPRs for vararg and pre-ANSI - // functions. - case FFI_TYPE_FLOAT: - // Nudge markers to next 4/8-byte boundary - srcMarker = ALIGN(srcMarker, 4); - destGMarker = ALIGN(destGMarker, 4); - destFMarker = ALIGN(destFMarker, 8); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - if (outFPRs != NULL && inStruct != NULL) - *(double*)&outFPRs[destFMarker] = - (double)*(float*)&inStruct[srcMarker]; - - destFMarker += 8; - fprsUsed++; - } - - if (outGPRs != NULL && inStruct != NULL) - *(double*)&outGPRs[destGMarker] = - (double)*(float*)&inStruct[srcMarker]; - - srcMarker += 4; - destGMarker += 4; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (srcMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 4)) - destGMarker = ALIGN(destGMarker, 8); - } - - break; - - case FFI_TYPE_DOUBLE: - srcMarker = ALIGN(srcMarker, 8); - destFMarker = ALIGN(destFMarker, 8); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - if (outFPRs != NULL && inStruct != NULL) - *(double*)&outFPRs[destFMarker] = - *(double*)&inStruct[srcMarker]; - - destFMarker += 8; - fprsUsed++; - } - - if (outGPRs != NULL && inStruct != NULL) - *(double*)&outGPRs[destGMarker] = - *(double*)&inStruct[srcMarker]; - - srcMarker += 8; - - // Skip next GPR - destGMarker += 8; - destGMarker = ALIGN(destGMarker, 8); - - break; - - case FFI_TYPE_LONGDOUBLE: - srcMarker = ALIGN(srcMarker, 16); - - if (fprsUsed < NUM_FPR_ARG_REGISTERS) - { - destFMarker = ALIGN(destFMarker, 8); - destGMarker = ALIGN(destGMarker, 8); - - if (outFPRs != NULL && inStruct != NULL) - *(long double*)&outFPRs[destFMarker] = - *(long double*)&inStruct[srcMarker]; - - if (outGPRs != NULL && inStruct != NULL) - *(long double*)&outGPRs[destGMarker] = - *(long double*)&inStruct[srcMarker]; - - destFMarker += 16; - fprsUsed += 2; - } - else - { - destGMarker = ALIGN(destGMarker, 16); - - if (outGPRs != NULL && inStruct != NULL) - *(long double*)&outGPRs[destGMarker] = - *(long double*)&inStruct[srcMarker]; - } - - srcMarker += 16; - destGMarker += 16; // Skip next 2 GPRs - destGMarker = ALIGN(destGMarker, 8); // was 16 - - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - if (inType->alignment == 1) // bytes only - { - if (inType->size == 1) - { - if (outGPRs != NULL && inStruct != NULL) - outGPRs[destGMarker] = inStruct[srcMarker]; - - srcMarker++; - destGMarker++; - } - else if (inType->size == 2) - { - if (outGPRs != NULL && inStruct != NULL) - { - outGPRs[destGMarker] = inStruct[srcMarker]; - outGPRs[destGMarker + 1] = inStruct[srcMarker + 1]; - } - - srcMarker += 2; - destGMarker += 2; - - i++; - } - else - { - if (outGPRs != NULL && inStruct != NULL) - { - // Avoid memcpy for small chunks. - if (inType->size <= sizeof(long)) - *(long*)&outGPRs[destGMarker] = - *(long*)&inStruct[srcMarker]; - else - memcpy(&outGPRs[destGMarker], - &inStruct[srcMarker], inType->size); - } - - srcMarker += inType->size; - destGMarker += inType->size; - i += inType->size - 1; - } - } - else // bytes and other stuff - { - if (outGPRs != NULL && inStruct != NULL) - outGPRs[destGMarker] = inStruct[srcMarker]; - - srcMarker++; - destGMarker++; - - // Skip to next GPR if next element won't fit and we're - // not already at a register boundary. - if (inType->elements[i + 1] != NULL && (destGMarker % 8)) - { - if (!FFI_TYPE_1_BYTE(inType->elements[i + 1]->type) && - (!FFI_TYPE_2_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 2) && - (!FFI_TYPE_4_BYTE(inType->elements[i + 1]->type) || - (ALIGN(destGMarker, 8) - destGMarker) < 4)) - destGMarker = ALIGN(destGMarker, inType->alignment); // was 8 - } - } - - break; - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - srcMarker = ALIGN(srcMarker, 2); - destGMarker = ALIGN(destGMarker, 2); - - if (outGPRs != NULL && inStruct != NULL) - *(short*)&outGPRs[destGMarker] = - *(short*)&inStruct[srcMarker]; - - srcMarker += 2; - destGMarker += 2; - - if (inType->elements[i + 1] == NULL) - destGMarker = ALIGN(destGMarker, inType->alignment); - - break; - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - srcMarker = ALIGN(srcMarker, 4); - destGMarker = ALIGN(destGMarker, 4); - - if (outGPRs != NULL && inStruct != NULL) - *(int*)&outGPRs[destGMarker] = - *(int*)&inStruct[srcMarker]; - - srcMarker += 4; - destGMarker += 4; - - break; - - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - srcMarker = ALIGN(srcMarker, 8); - destGMarker = ALIGN(destGMarker, 8); - - if (outGPRs != NULL && inStruct != NULL) - *(long long*)&outGPRs[destGMarker] = - *(long long*)&inStruct[srcMarker]; - - srcMarker += 8; - destGMarker += 8; - - if (inType->elements[i + 1] == NULL) - destGMarker = ALIGN(destGMarker, inType->alignment); - - break; - - case FFI_TYPE_STRUCT: - recurseCount++; - ffi64_struct_to_reg_form(inType->elements[i], - inStruct, &srcMarker, &fprsUsed, outGPRs, - &destGMarker, outFPRs, &destFMarker); - recurseCount--; - break; - - default: - FFI_ASSERT(0); - break; - } - } - - destGMarker = ALIGN(destGMarker, inType->alignment); - - // Take care of the special case for 16-byte structs, but not for - // nested structs. - if (recurseCount == 0 && destGMarker == 16) - { - if (outGPRs != NULL && inStruct != NULL) - *(long double*)&outGPRs[0] = *(long double*)&inStruct[0]; - - destFMarker = savedFMarker; - fprsUsed = savedFPRsUsed; - } - - if (ioStructMarker) - *ioStructMarker = ALIGN(srcMarker, 8); - - if (ioFPRsUsed) - *ioFPRsUsed = fprsUsed; - - if (ioGPRSize) - *ioGPRSize = ALIGN(destGMarker, 8); - - if (ioFPRSize) - *ioFPRSize = ALIGN(destFMarker, 8); -} - -/* ffi64_stret_needs_ptr - - Determine whether a returned struct needs a pointer in r3 or can fit - in registers. -*/ - -bool -ffi64_stret_needs_ptr( - const ffi_type* inType, - unsigned short* ioGPRCount, - unsigned short* ioFPRCount) -{ - // Obvious case first- struct is larger than combined FPR size. - if (inType->size > 14 * 8) - return true; - - // Now the struct can physically fit in registers, determine if it - // also fits logically. - bool needsPtr = false; - unsigned short gprsUsed = 0; - unsigned short fprsUsed = 0; - size_t i; - - if (ioGPRCount) - gprsUsed = *ioGPRCount; - - if (ioFPRCount) - fprsUsed = *ioFPRCount; - - for (i = 0; inType->elements[i] != NULL && !needsPtr; i++) - { - switch (inType->elements[i]->type) - { - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - gprsUsed++; - fprsUsed++; - - if (fprsUsed > 13) - needsPtr = true; - - break; - - case FFI_TYPE_LONGDOUBLE: - gprsUsed += 2; - fprsUsed += 2; - - if (fprsUsed > 14) - needsPtr = true; - - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - { - gprsUsed++; - - if (gprsUsed > 8) - { - needsPtr = true; - break; - } - - if (inType->elements[i + 1] == NULL) // last byte in the struct - break; - - // Count possible contiguous bytes ahead, up to 8. - unsigned short j; - - for (j = 1; j < 8; j++) - { - if (inType->elements[i + j] == NULL || - !FFI_TYPE_1_BYTE(inType->elements[i + j]->type)) - break; - } - - i += j - 1; // allow for i++ before the test condition - - break; - } - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - gprsUsed++; - - if (gprsUsed > 8) - needsPtr = true; - - break; - - case FFI_TYPE_STRUCT: - needsPtr = ffi64_stret_needs_ptr( - inType->elements[i], &gprsUsed, &fprsUsed); - - break; - - default: - FFI_ASSERT(0); - break; - } - } - - if (ioGPRCount) - *ioGPRCount = gprsUsed; - - if (ioFPRCount) - *ioFPRCount = fprsUsed; - - return needsPtr; -} - -/* ffi64_data_size - - Calculate the size in bytes of an ffi type. -*/ - -unsigned int -ffi64_data_size( - const ffi_type* inType) -{ - unsigned int size = 0; - - switch (inType->type) - { - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - size = 1; - break; - - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - size = 2; - break; - - case FFI_TYPE_INT: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_FLOAT: - size = 4; - break; - - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - case FFI_TYPE_DOUBLE: - size = 8; - break; - - case FFI_TYPE_LONGDOUBLE: - size = 16; - break; - - case FFI_TYPE_STRUCT: - ffi64_struct_to_reg_form( - inType, NULL, NULL, NULL, NULL, &size, NULL, NULL); - break; - - case FFI_TYPE_VOID: - break; - - default: - FFI_ASSERT(0); - break; - } - - return size; -} - -#endif /* defined(__ppc64__) */ -#endif /* __ppc__ || __ppc64__ */ diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S b/Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S deleted file mode 100644 index 7162fa1dda65..000000000000 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc64-darwin_closure.S +++ /dev/null @@ -1,418 +0,0 @@ -#if defined(__ppc64__) - -/* ----------------------------------------------------------------------- - ppc64-darwin_closure.S - Copyright (c) 2002, 2003, 2004, Free Software Foundation, - Inc. based on ppc_closure.S - - PowerPC Assembly glue. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#define LIBFFI_ASM - -#include <ffi.h> -#include <ppc-ffitarget.h> // for FFI_TRAMPOLINE_SIZE -#include <ppc-darwin.h> -#include <architecture/ppc/mode_independent_asm.h> - - .file "ppc64-darwin_closure.S" -.text - .align LOG2_GPR_BYTES - .globl _ffi_closure_ASM - -.text - .align LOG2_GPR_BYTES - -_ffi_closure_ASM: -LFB1: - mflr r0 - stg r0,SF_RETURN(r1) // save return address - - // Save GPRs 3 - 10 (aligned to 8) in the parents outgoing area. - stg r3,SF_ARG1(r1) - stg r4,SF_ARG2(r1) - stg r5,SF_ARG3(r1) - stg r6,SF_ARG4(r1) - stg r7,SF_ARG5(r1) - stg r8,SF_ARG6(r1) - stg r9,SF_ARG7(r1) - stg r10,SF_ARG8(r1) - -LCFI0: -/* 48 bytes (Linkage Area) - 64 bytes (outgoing parameter area, always reserved) - 112 bytes (14*8 for incoming FPR) - ? bytes (result) - 112 bytes (14*8 for outgoing FPR) - 16 bytes (2 saved registers) - 352 + ? total bytes -*/ - - std r31,-8(r1) // Save registers we use. - std r30,-16(r1) - mr r30,r1 // Save the old SP. - mr r31,r11 // Save the ffi_closure around ffi64_data_size. - - // Calculate the space we need. - stdu r1,-SF_MINSIZE(r1) - ld r3,FFI_TRAMPOLINE_SIZE(r31) // ffi_closure->cif* - ld r3,16(r3) // ffi_cif->rtype* - bl Lffi64_data_size$stub - ld r1,0(r1) - - addi r3,r3,352 // Add our overhead. - neg r3,r3 - li r0,-32 // Align to 32 bytes. - and r3,r3,r0 - stdux r1,r1,r3 // Grow the stack. - - mr r11,r31 // Copy the ffi_closure back. - -LCFI1: - // We want to build up an area for the parameters passed - // in registers. (both floating point and integer) - -/* 320 bytes (callee stack frame aligned to 32) - 48 bytes (caller linkage area) - 368 (start of caller parameter area aligned to 8) -*/ - - // Save FPRs 1 - 14. (aligned to 8) - stfd f1,112(r1) - stfd f2,120(r1) - stfd f3,128(r1) - stfd f4,136(r1) - stfd f5,144(r1) - stfd f6,152(r1) - stfd f7,160(r1) - stfd f8,168(r1) - stfd f9,176(r1) - stfd f10,184(r1) - stfd f11,192(r1) - stfd f12,200(r1) - stfd f13,208(r1) - stfd f14,216(r1) - - // Set up registers for the routine that actually does the work. - mr r3,r11 // context pointer from the trampoline - addi r4,r1,224 // result storage - addi r5,r30,SF_ARG1 // saved GPRs - addi r6,r1,112 // saved FPRs - bl Lffi_closure_helper_DARWIN$stub - - // Look the proper starting point in table - // by using return type as an offset. - addi r5,r1,224 // Get pointer to results area. - bl Lget_ret_type0_addr // Get pointer to Lret_type0 into LR. - mflr r4 // Move to r4. - slwi r3,r3,4 // Now multiply return type by 16. - add r3,r3,r4 // Add contents of table to table address. - mtctr r3 - bctr - -LFE1: - // Each of the ret_typeX code fragments has to be exactly 16 bytes long - // (4 instructions). For cache effectiveness we align to a 16 byte - // boundary first. - .align 4 - nop - nop - nop - -Lget_ret_type0_addr: - blrl - -// case FFI_TYPE_VOID -Lret_type0: - b Lfinish - nop - nop - nop - -// case FFI_TYPE_INT -Lret_type1: - lwz r3,4(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_FLOAT -Lret_type2: - lfs f1,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_DOUBLE -Lret_type3: - lfd f1,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_LONGDOUBLE -Lret_type4: - lfd f1,0(r5) - lfd f2,8(r5) - b Lfinish - nop - -// case FFI_TYPE_UINT8 -Lret_type5: - lbz r3,7(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT8 -Lret_type6: - lbz r3,7(r5) - extsb r3,r3 - b Lfinish - nop - -// case FFI_TYPE_UINT16 -Lret_type7: - lhz r3,6(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT16 -Lret_type8: - lha r3,6(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_UINT32 -Lret_type9: // same as Lret_type1 - lwz r3,4(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT32 -Lret_type10: // same as Lret_type1 - lwz r3,4(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_UINT64 -Lret_type11: - ld r3,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_SINT64 -Lret_type12: // same as Lret_type11 - ld r3,0(r5) - b Lfinish - nop - nop - -// case FFI_TYPE_STRUCT -Lret_type13: - b Lret_struct - nop - nop - nop - -// ** End 16-byte aligned cases ** -// case FFI_TYPE_POINTER -// This case assumes that FFI_TYPE_POINTER == FFI_TYPE_LAST. If more types -// are added in future, the following code will need to be updated and -// padded to 16 bytes. -Lret_type14: - lg r3,0(r5) - b Lfinish - -// copy struct into registers -Lret_struct: - ld r31,FFI_TRAMPOLINE_SIZE(r31) // ffi_closure->cif* - ld r3,16(r31) // ffi_cif->rtype* - ld r31,24(r31) // ffi_cif->flags - mr r4,r5 // copy struct* to 2nd arg - addi r7,r1,SF_ARG9 // GPR return area - addi r9,r30,-16-(14*8) // FPR return area - li r5,0 // struct offset ptr (NULL) - li r6,0 // FPR used count ptr (NULL) - li r8,0 // GPR return area size ptr (NULL) - li r10,0 // FPR return area size ptr (NULL) - bl Lffi64_struct_to_reg_form$stub - - // Load GPRs - ld r3,SF_ARG9(r1) - ld r4,SF_ARG10(r1) - ld r5,SF_ARG11(r1) - ld r6,SF_ARG12(r1) - nop - ld r7,SF_ARG13(r1) - ld r8,SF_ARG14(r1) - ld r9,SF_ARG15(r1) - ld r10,SF_ARG16(r1) - nop - - // Load FPRs - mtcrf 0x2,r31 - bf 26,Lfinish - lfd f1,-16-(14*8)(r30) - lfd f2,-16-(13*8)(r30) - lfd f3,-16-(12*8)(r30) - lfd f4,-16-(11*8)(r30) - nop - lfd f5,-16-(10*8)(r30) - lfd f6,-16-(9*8)(r30) - lfd f7,-16-(8*8)(r30) - lfd f8,-16-(7*8)(r30) - nop - lfd f9,-16-(6*8)(r30) - lfd f10,-16-(5*8)(r30) - lfd f11,-16-(4*8)(r30) - lfd f12,-16-(3*8)(r30) - nop - lfd f13,-16-(2*8)(r30) - lfd f14,-16-(1*8)(r30) - // Fall through - -// case done -Lfinish: - lg r1,0(r1) // Restore stack pointer. - ld r31,-8(r1) // Restore registers we used. - ld r30,-16(r1) - lg r0,SF_RETURN(r1) // Get return address. - mtlr r0 // Reset link register. - blr - -// END(ffi_closure_ASM) - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 ; Length of Common Information Entry -LSCIE1: - .long 0x0 ; CIE Identifier Tag - .byte 0x1 ; CIE Version - .ascii "zR\0" ; CIE Augmentation - .byte 0x1 ; uleb128 0x1; CIE Code Alignment Factor - .byte 0x7c ; sleb128 -4; CIE Data Alignment Factor - .byte 0x41 ; CIE RA Column - .byte 0x1 ; uleb128 0x1; Augmentation size - .byte 0x10 ; FDE Encoding (pcrel) - .byte 0xc ; DW_CFA_def_cfa - .byte 0x1 ; uleb128 0x1 - .byte 0x0 ; uleb128 0x0 - .align LOG2_GPR_BYTES -LECIE1: -.globl _ffi_closure_ASM.eh -_ffi_closure_ASM.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 ; FDE Length - -LASFDE1: - .long LASFDE1-EH_frame1 ; FDE CIE offset - .g_long LFB1-. ; FDE initial location - .set L$set$3,LFE1-LFB1 - .g_long L$set$3 ; FDE address range - .byte 0x0 ; uleb128 0x0; Augmentation size - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$3,LCFI1-LCFI0 - .long L$set$3 - .byte 0xe ; DW_CFA_def_cfa_offset - .byte 176,1 ; uleb128 176 - .byte 0x4 ; DW_CFA_advance_loc4 - .set L$set$4,LCFI0-LFB1 - .long L$set$4 - .byte 0x11 ; DW_CFA_offset_extended_sf - .byte 0x41 ; uleb128 0x41 - .byte 0x7e ; sleb128 -2 - .align LOG2_GPR_BYTES - -LEFDE1: -.data - .align LOG2_GPR_BYTES -LDFCM0: -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi_closure_helper_DARWIN$stub: - .indirect_symbol _ffi_closure_helper_DARWIN - mflr r0 - bcl 20,31,LO$ffi_closure_helper_DARWIN - -LO$ffi_closure_helper_DARWIN: - mflr r11 - addis r11,r11,ha16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN) - mtlr r0 - lgu r12,lo16(L_ffi_closure_helper_DARWIN$lazy_ptr - LO$ffi_closure_helper_DARWIN)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi_closure_helper_DARWIN$lazy_ptr: - .indirect_symbol _ffi_closure_helper_DARWIN - .g_long dyld_stub_binding_helper - -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi64_struct_to_reg_form$stub: - .indirect_symbol _ffi64_struct_to_reg_form - mflr r0 - bcl 20,31,LO$ffi64_struct_to_reg_form - -LO$ffi64_struct_to_reg_form: - mflr r11 - addis r11,r11,ha16(L_ffi64_struct_to_reg_form$lazy_ptr - LO$ffi64_struct_to_reg_form) - mtlr r0 - lgu r12,lo16(L_ffi64_struct_to_reg_form$lazy_ptr - LO$ffi64_struct_to_reg_form)(r11) - mtctr r12 - bctr - -.section __TEXT,__picsymbolstub1,symbol_stubs,pure_instructions,32 - .align LOG2_GPR_BYTES - -Lffi64_data_size$stub: - .indirect_symbol _ffi64_data_size - mflr r0 - bcl 20,31,LO$ffi64_data_size - -LO$ffi64_data_size: - mflr r11 - addis r11,r11,ha16(L_ffi64_data_size$lazy_ptr - LO$ffi64_data_size) - mtlr r0 - lgu r12,lo16(L_ffi64_data_size$lazy_ptr - LO$ffi64_data_size)(r11) - mtctr r12 - bctr - -.lazy_symbol_pointer -L_ffi64_struct_to_reg_form$lazy_ptr: - .indirect_symbol _ffi64_struct_to_reg_form - .g_long dyld_stub_binding_helper - -L_ffi64_data_size$lazy_ptr: - .indirect_symbol _ffi64_data_size - .g_long dyld_stub_binding_helper - -#endif // __ppc64__ diff --git a/Modules/_ctypes/libffi_osx/types.c b/Modules/_ctypes/libffi_osx/types.c deleted file mode 100644 index 44806aeeb75d..000000000000 --- a/Modules/_ctypes/libffi_osx/types.c +++ /dev/null @@ -1,115 +0,0 @@ -/* ----------------------------------------------------------------------- - types.c - Copyright (c) 1996, 1998 Red Hat, Inc. - - Predefined ffi_types needed by libffi. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -/* Type definitions */ -#define FFI_INTEGRAL_TYPEDEF(n, s, a, t) \ - ffi_type ffi_type_##n = { s, a, t, NULL } -#define FFI_AGGREGATE_TYPEDEF(n, e) \ - ffi_type ffi_type_##n = { 0, 0, FFI_TYPE_STRUCT, e } - -FFI_INTEGRAL_TYPEDEF(uint8, 1, 1, FFI_TYPE_UINT8); -FFI_INTEGRAL_TYPEDEF(sint8, 1, 1, FFI_TYPE_SINT8); -FFI_INTEGRAL_TYPEDEF(uint16, 2, 2, FFI_TYPE_UINT16); -FFI_INTEGRAL_TYPEDEF(sint16, 2, 2, FFI_TYPE_SINT16); -FFI_INTEGRAL_TYPEDEF(uint32, 4, 4, FFI_TYPE_UINT32); -FFI_INTEGRAL_TYPEDEF(sint32, 4, 4, FFI_TYPE_SINT32); -FFI_INTEGRAL_TYPEDEF(float, 4, 4, FFI_TYPE_FLOAT); - -/* Size and alignment are fake here. They must not be 0. */ -FFI_INTEGRAL_TYPEDEF(void, 1, 1, FFI_TYPE_VOID); - -#if defined ALPHA || defined SPARC64 || defined X86_64 || \ - defined S390X || defined IA64 || defined POWERPC64 -FFI_INTEGRAL_TYPEDEF(pointer, 8, 8, FFI_TYPE_POINTER); -#else -FFI_INTEGRAL_TYPEDEF(pointer, 4, 4, FFI_TYPE_POINTER); -#endif - -#if defined X86 || defined ARM || defined M68K || defined(X86_DARWIN) - -# ifdef X86_64 - FFI_INTEGRAL_TYPEDEF(uint64, 8, 8, FFI_TYPE_UINT64); - FFI_INTEGRAL_TYPEDEF(sint64, 8, 8, FFI_TYPE_SINT64); -# else - FFI_INTEGRAL_TYPEDEF(uint64, 8, 4, FFI_TYPE_UINT64); - FFI_INTEGRAL_TYPEDEF(sint64, 8, 4, FFI_TYPE_SINT64); -# endif - -#elif defined(POWERPC_DARWIN) -FFI_INTEGRAL_TYPEDEF(uint64, 8, 8, FFI_TYPE_UINT64); -FFI_INTEGRAL_TYPEDEF(sint64, 8, 8, FFI_TYPE_SINT64); -#elif defined SH -FFI_INTEGRAL_TYPEDEF(uint64, 8, 4, FFI_TYPE_UINT64); -FFI_INTEGRAL_TYPEDEF(sint64, 8, 4, FFI_TYPE_SINT64); -#else -FFI_INTEGRAL_TYPEDEF(uint64, 8, 8, FFI_TYPE_UINT64); -FFI_INTEGRAL_TYPEDEF(sint64, 8, 8, FFI_TYPE_SINT64); -#endif - -#if defined X86 || defined X86_WIN32 || defined M68K || defined(X86_DARWIN) - -# if defined X86_WIN32 || defined X86_64 - FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(double, 8, 4, FFI_TYPE_DOUBLE); -# endif - -# ifdef X86_DARWIN - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(longdouble, 12, 4, FFI_TYPE_LONGDOUBLE); -# endif - -#elif defined ARM || defined SH || defined POWERPC_AIX -FFI_INTEGRAL_TYPEDEF(double, 8, 4, FFI_TYPE_DOUBLE); -FFI_INTEGRAL_TYPEDEF(longdouble, 8, 4, FFI_TYPE_LONGDOUBLE); -#elif defined POWERPC_DARWIN -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); - -# if __GNUC__ >= 4 - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(longdouble, 8, 8, FFI_TYPE_LONGDOUBLE); -# endif - -#elif defined SPARC -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); - -# ifdef SPARC64 - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -# else - FFI_INTEGRAL_TYPEDEF(longdouble, 16, 8, FFI_TYPE_LONGDOUBLE); -# endif - -#elif defined X86_64 || defined POWERPC64 -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); -FFI_INTEGRAL_TYPEDEF(longdouble, 16, 16, FFI_TYPE_LONGDOUBLE); -#else -FFI_INTEGRAL_TYPEDEF(double, 8, 8, FFI_TYPE_DOUBLE); -FFI_INTEGRAL_TYPEDEF(longdouble, 8, 8, FFI_TYPE_LONGDOUBLE); -#endif \ No newline at end of file diff --git a/Modules/_ctypes/libffi_osx/x86/darwin64.S b/Modules/_ctypes/libffi_osx/x86/darwin64.S deleted file mode 100644 index 1286d33f83e0..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/darwin64.S +++ /dev/null @@ -1,417 +0,0 @@ -/* ----------------------------------------------------------------------- - darwin64.S - Copyright (c) 2006 Free Software Foundation, Inc. - derived from unix64.S - - x86-64 Foreign Function Interface for Darwin. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#ifdef __x86_64__ -#define LIBFFI_ASM -#include <fficonfig.h> -#include <ffi.h> - - .file "darwin64.S" -.text - -/* ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags, - void *raddr, void (*fnaddr)()); - - Bit o trickiness here -- ARGS+BYTES is the base of the stack frame - for this function. This has been allocated by ffi_call. We also - deallocate some of the stack that has been alloca'd. */ - - .align 3 - .globl _ffi_call_unix64 - -_ffi_call_unix64: -LUW0: - movq (%rsp), %r10 /* Load return address. */ - movq %rdi, %r12 /* Save a copy of the register area. */ - leaq (%rdi, %rsi), %rax /* Find local stack base. */ - movq %rdx, (%rax) /* Save flags. */ - movq %rcx, 8(%rax) /* Save raddr. */ - movq %rbp, 16(%rax) /* Save old frame pointer. */ - movq %r10, 24(%rax) /* Relocate return address. */ - movq %rax, %rbp /* Finalize local stack frame. */ -LUW1: - /* movq %rdi, %r10 // Save a copy of the register area. */ - movq %r12, %r10 - movq %r8, %r11 /* Save a copy of the target fn. */ - movl %r9d, %eax /* Set number of SSE registers. */ - - /* Load up all argument registers. */ - movq (%r10), %rdi - movq 8(%r10), %rsi - movq 16(%r10), %rdx - movq 24(%r10), %rcx - movq 32(%r10), %r8 - movq 40(%r10), %r9 - testl %eax, %eax - jnz Lload_sse -Lret_from_load_sse: - - /* Deallocate the reg arg area. */ - leaq 176(%r10), %rsp - - /* Call the user function. */ - call *%r11 - - /* Deallocate stack arg area; local stack frame in redzone. */ - leaq 24(%rbp), %rsp - - movq 0(%rbp), %rcx /* Reload flags. */ - movq 8(%rbp), %rdi /* Reload raddr. */ - movq 16(%rbp), %rbp /* Reload old frame pointer. */ -LUW2: - - /* The first byte of the flags contains the FFI_TYPE. */ - movzbl %cl, %r10d - leaq Lstore_table(%rip), %r11 - movslq (%r11, %r10, 4), %r10 - addq %r11, %r10 - jmp *%r10 - -Lstore_table: - .long Lst_void-Lstore_table /* FFI_TYPE_VOID */ - .long Lst_sint32-Lstore_table /* FFI_TYPE_INT */ - .long Lst_float-Lstore_table /* FFI_TYPE_FLOAT */ - .long Lst_double-Lstore_table /* FFI_TYPE_DOUBLE */ - .long Lst_ldouble-Lstore_table /* FFI_TYPE_LONGDOUBLE */ - .long Lst_uint8-Lstore_table /* FFI_TYPE_UINT8 */ - .long Lst_sint8-Lstore_table /* FFI_TYPE_SINT8 */ - .long Lst_uint16-Lstore_table /* FFI_TYPE_UINT16 */ - .long Lst_sint16-Lstore_table /* FFI_TYPE_SINT16 */ - .long Lst_uint32-Lstore_table /* FFI_TYPE_UINT32 */ - .long Lst_sint32-Lstore_table /* FFI_TYPE_SINT32 */ - .long Lst_int64-Lstore_table /* FFI_TYPE_UINT64 */ - .long Lst_int64-Lstore_table /* FFI_TYPE_SINT64 */ - .long Lst_struct-Lstore_table /* FFI_TYPE_STRUCT */ - .long Lst_int64-Lstore_table /* FFI_TYPE_POINTER */ - - .text - .align 3 -Lst_void: - ret - .align 3 -Lst_uint8: - movzbq %al, %rax - movq %rax, (%rdi) - ret - .align 3 -Lst_sint8: - movsbq %al, %rax - movq %rax, (%rdi) - ret - .align 3 -Lst_uint16: - movzwq %ax, %rax - movq %rax, (%rdi) - .align 3 -Lst_sint16: - movswq %ax, %rax - movq %rax, (%rdi) - ret - .align 3 -Lst_uint32: - movl %eax, %eax - movq %rax, (%rdi) - .align 3 -Lst_sint32: - cltq - movq %rax, (%rdi) - ret - .align 3 -Lst_int64: - movq %rax, (%rdi) - ret - .align 3 -Lst_float: - movss %xmm0, (%rdi) - ret - .align 3 -Lst_double: - movsd %xmm0, (%rdi) - ret -Lst_ldouble: - fstpt (%rdi) - ret - .align 3 -Lst_struct: - leaq -20(%rsp), %rsi /* Scratch area in redzone. */ - - /* We have to locate the values now, and since we don't want to - write too much data into the user's return value, we spill the - value to a 16 byte scratch area first. Bits 8, 9, and 10 - control where the values are located. Only one of the three - bits will be set; see ffi_prep_cif_machdep for the pattern. */ - movd %xmm0, %r10 - movd %xmm1, %r11 - testl $0x100, %ecx - cmovnz %rax, %rdx - cmovnz %r10, %rax - testl $0x200, %ecx - cmovnz %r10, %rdx - testl $0x400, %ecx - cmovnz %r10, %rax - cmovnz %r11, %rdx - movq %rax, (%rsi) - movq %rdx, 8(%rsi) - - /* Bits 12-31 contain the true size of the structure. Copy from - the scratch area to the true destination. */ - shrl $12, %ecx - rep movsb - ret - - /* Many times we can avoid loading any SSE registers at all. - It's not worth an indirect jump to load the exact set of - SSE registers needed; zero or all is a good compromise. */ - .align 3 -LUW3: -Lload_sse: - movdqa 48(%r10), %xmm0 - movdqa 64(%r10), %xmm1 - movdqa 80(%r10), %xmm2 - movdqa 96(%r10), %xmm3 - movdqa 112(%r10), %xmm4 - movdqa 128(%r10), %xmm5 - movdqa 144(%r10), %xmm6 - movdqa 160(%r10), %xmm7 - jmp Lret_from_load_sse - -LUW4: - .align 3 - .globl _ffi_closure_unix64 - -_ffi_closure_unix64: -LUW5: - /* The carry flag is set by the trampoline iff SSE registers - are used. Don't clobber it before the branch instruction. */ - leaq -200(%rsp), %rsp -LUW6: - movq %rdi, (%rsp) - movq %rsi, 8(%rsp) - movq %rdx, 16(%rsp) - movq %rcx, 24(%rsp) - movq %r8, 32(%rsp) - movq %r9, 40(%rsp) - jc Lsave_sse -Lret_from_save_sse: - - movq %r10, %rdi - leaq 176(%rsp), %rsi - movq %rsp, %rdx - leaq 208(%rsp), %rcx - call _ffi_closure_unix64_inner - - /* Deallocate stack frame early; return value is now in redzone. */ - addq $200, %rsp -LUW7: - - /* The first byte of the return value contains the FFI_TYPE. */ - movzbl %al, %r10d - leaq Lload_table(%rip), %r11 - movslq (%r11, %r10, 4), %r10 - addq %r11, %r10 - jmp *%r10 - -Lload_table: - .long Lld_void-Lload_table /* FFI_TYPE_VOID */ - .long Lld_int32-Lload_table /* FFI_TYPE_INT */ - .long Lld_float-Lload_table /* FFI_TYPE_FLOAT */ - .long Lld_double-Lload_table /* FFI_TYPE_DOUBLE */ - .long Lld_ldouble-Lload_table /* FFI_TYPE_LONGDOUBLE */ - .long Lld_int8-Lload_table /* FFI_TYPE_UINT8 */ - .long Lld_int8-Lload_table /* FFI_TYPE_SINT8 */ - .long Lld_int16-Lload_table /* FFI_TYPE_UINT16 */ - .long Lld_int16-Lload_table /* FFI_TYPE_SINT16 */ - .long Lld_int32-Lload_table /* FFI_TYPE_UINT32 */ - .long Lld_int32-Lload_table /* FFI_TYPE_SINT32 */ - .long Lld_int64-Lload_table /* FFI_TYPE_UINT64 */ - .long Lld_int64-Lload_table /* FFI_TYPE_SINT64 */ - .long Lld_struct-Lload_table /* FFI_TYPE_STRUCT */ - .long Lld_int64-Lload_table /* FFI_TYPE_POINTER */ - - .text - .align 3 -Lld_void: - ret - .align 3 -Lld_int8: - movzbl -24(%rsp), %eax - ret - .align 3 -Lld_int16: - movzwl -24(%rsp), %eax - ret - .align 3 -Lld_int32: - movl -24(%rsp), %eax - ret - .align 3 -Lld_int64: - movq -24(%rsp), %rax - ret - .align 3 -Lld_float: - movss -24(%rsp), %xmm0 - ret - .align 3 -Lld_double: - movsd -24(%rsp), %xmm0 - ret - .align 3 -Lld_ldouble: - fldt -24(%rsp) - ret - .align 3 -Lld_struct: - /* There are four possibilities here, %rax/%rdx, %xmm0/%rax, - %rax/%xmm0, %xmm0/%xmm1. We collapse two by always loading - both rdx and xmm1 with the second word. For the remaining, - bit 8 set means xmm0 gets the second word, and bit 9 means - that rax gets the second word. */ - movq -24(%rsp), %rcx - movq -16(%rsp), %rdx - movq -16(%rsp), %xmm1 - testl $0x100, %eax - cmovnz %rdx, %rcx - movd %rcx, %xmm0 - testl $0x200, %eax - movq -24(%rsp), %rax - cmovnz %rdx, %rax - ret - - /* See the comment above Lload_sse; the same logic applies here. */ - .align 3 -LUW8: -Lsave_sse: - movdqa %xmm0, 48(%rsp) - movdqa %xmm1, 64(%rsp) - movdqa %xmm2, 80(%rsp) - movdqa %xmm3, 96(%rsp) - movdqa %xmm4, 112(%rsp) - movdqa %xmm5, 128(%rsp) - movdqa %xmm6, 144(%rsp) - movdqa %xmm7, 160(%rsp) - jmp Lret_from_save_sse - -LUW9: -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 /* CIE Length */ - .long L$set$0 -LSCIE1: - .long 0x0 /* CIE Identifier Tag */ - .byte 0x1 /* CIE Version */ - .ascii "zR\0" /* CIE Augmentation */ - .byte 0x1 /* uleb128 0x1; CIE Code Alignment Factor */ - .byte 0x78 /* sleb128 -8; CIE Data Alignment Factor */ - .byte 0x10 /* CIE RA Column */ - .byte 0x1 /* uleb128 0x1; Augmentation size */ - .byte 0x10 /* FDE Encoding (pcrel sdata4) */ - .byte 0xc /* DW_CFA_def_cfa, %rsp offset 8 */ - .byte 0x7 /* uleb128 0x7 */ - .byte 0x8 /* uleb128 0x8 */ - .byte 0x90 /* DW_CFA_offset, column 0x10 */ - .byte 0x1 - .align 3 -LECIE1: - .globl _ffi_call_unix64.eh -_ffi_call_unix64.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 /* FDE Length */ - .long L$set$1 -LASFDE1: - .long LASFDE1-EH_frame1 /* FDE CIE offset */ - .quad LUW0-. /* FDE initial location */ - .set L$set$2,LUW4-LUW0 /* FDE address range */ - .quad L$set$2 - .byte 0x0 /* Augmentation size */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$3,LUW1-LUW0 - .long L$set$3 - - /* New stack frame based off rbp. This is an itty bit of unwind - trickery in that the CFA *has* changed. There is no easy way - to describe it correctly on entry to the function. Fortunately, - it doesn't matter too much since at all points we can correctly - unwind back to ffi_call. Note that the location to which we - moved the return address is (the new) CFA-8, so from the - perspective of the unwind info, it hasn't moved. */ - .byte 0xc /* DW_CFA_def_cfa, %rbp offset 32 */ - .byte 0x6 - .byte 0x20 - .byte 0x80+6 /* DW_CFA_offset, %rbp offset 2*-8 */ - .byte 0x2 - .byte 0xa /* DW_CFA_remember_state */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$4,LUW2-LUW1 - .long L$set$4 - .byte 0xc /* DW_CFA_def_cfa, %rsp offset 8 */ - .byte 0x7 - .byte 0x8 - .byte 0xc0+6 /* DW_CFA_restore, %rbp */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$5,LUW3-LUW2 - .long L$set$5 - .byte 0xb /* DW_CFA_restore_state */ - - .align 3 -LEFDE1: - .globl _ffi_closure_unix64.eh -_ffi_closure_unix64.eh: -LSFDE3: - .set L$set$6,LEFDE3-LASFDE3 /* FDE Length */ - .long L$set$6 -LASFDE3: - .long LASFDE3-EH_frame1 /* FDE CIE offset */ - .quad LUW5-. /* FDE initial location */ - .set L$set$7,LUW9-LUW5 /* FDE address range */ - .quad L$set$7 - .byte 0x0 /* Augmentation size */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$8,LUW6-LUW5 - .long L$set$8 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 208,1 /* uleb128 208 */ - .byte 0xa /* DW_CFA_remember_state */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$9,LUW7-LUW6 - .long L$set$9 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .set L$set$10,LUW8-LUW7 - .long L$set$10 - .byte 0xb /* DW_CFA_restore_state */ - - .align 3 -LEFDE3: - .subsections_via_symbols - -#endif /* __x86_64__ */ diff --git a/Modules/_ctypes/libffi_osx/x86/x86-darwin.S b/Modules/_ctypes/libffi_osx/x86/x86-darwin.S deleted file mode 100644 index 925a84131661..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/x86-darwin.S +++ /dev/null @@ -1,422 +0,0 @@ -#ifdef __i386__ -/* ----------------------------------------------------------------------- - darwin.S - Copyright (c) 1996, 1998, 2001, 2002, 2003 Red Hat, Inc. - - X86 Foreign Function Interface - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -/* - * This file is based on sysv.S and then hacked up by Ronald who hasn't done - * assembly programming in 8 years. - */ - -#ifndef __x86_64__ - -#define LIBFFI_ASM -#include <fficonfig.h> -#include <ffi.h> - -#ifdef PyObjC_STRICT_DEBUGGING - /* XXX: Debugging of stack alignment, to be removed */ -#define ASSERT_STACK_ALIGNED movdqa -16(%esp), %xmm0 -#else -#define ASSERT_STACK_ALIGNED -#endif - -.text - -.globl _ffi_prep_args - - .align 4 -.globl _ffi_call_SYSV - -_ffi_call_SYSV: -LFB1: - pushl %ebp -LCFI0: - movl %esp,%ebp -LCFI1: - subl $8,%esp - /* Make room for all of the new args. */ - movl 16(%ebp),%ecx - subl %ecx,%esp - - movl %esp,%eax - - /* Place all of the ffi_prep_args in position */ - subl $8,%esp - pushl 12(%ebp) - pushl %eax - call *8(%ebp) - - /* Return stack to previous state and call the function */ - addl $16,%esp - - call *28(%ebp) - - /* Remove the space we pushed for the args */ - movl 16(%ebp),%ecx - addl %ecx,%esp - - /* Load %ecx with the return type code */ - movl 20(%ebp),%ecx - - /* If the return value pointer is NULL, assume no return value. */ - cmpl $0,24(%ebp) - jne Lretint - - /* Even if there is no space for the return value, we are - obliged to handle floating-point values. */ - cmpl $FFI_TYPE_FLOAT,%ecx - jne Lnoretval - fstp %st(0) - - jmp Lepilogue - -Lretint: - cmpl $FFI_TYPE_INT,%ecx - jne Lretfloat - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - jmp Lepilogue - -Lretfloat: - cmpl $FFI_TYPE_FLOAT,%ecx - jne Lretdouble - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstps (%ecx) - jmp Lepilogue - -Lretdouble: - cmpl $FFI_TYPE_DOUBLE,%ecx - jne Lretlongdouble - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpl (%ecx) - jmp Lepilogue - -Lretlongdouble: - cmpl $FFI_TYPE_LONGDOUBLE,%ecx - jne Lretint64 - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpt (%ecx) - jmp Lepilogue - -Lretint64: - cmpl $FFI_TYPE_SINT64,%ecx - jne Lretstruct1b - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - movl %edx,4(%ecx) - jmp Lepilogue - -Lretstruct1b: - cmpl $FFI_TYPE_SINT8,%ecx - jne Lretstruct2b - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movb %al,0(%ecx) - jmp Lepilogue - -Lretstruct2b: - cmpl $FFI_TYPE_SINT16,%ecx - jne Lretstruct - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movw %ax,0(%ecx) - jmp Lepilogue - -Lretstruct: - cmpl $FFI_TYPE_STRUCT,%ecx - jne Lnoretval - /* Nothing to do! */ - addl $4,%esp - popl %ebp - ret - -Lnoretval: -Lepilogue: - addl $8,%esp - movl %ebp,%esp - popl %ebp - ret -LFE1: -.ffi_call_SYSV_end: - - .align 4 -FFI_HIDDEN (ffi_closure_SYSV) -.globl _ffi_closure_SYSV - -_ffi_closure_SYSV: -LFB2: - pushl %ebp -LCFI2: - movl %esp, %ebp -LCFI3: - subl $56, %esp - leal -40(%ebp), %edx - movl %edx, -12(%ebp) /* resp */ - leal 8(%ebp), %edx - movl %edx, 4(%esp) /* args = __builtin_dwarf_cfa () */ - leal -12(%ebp), %edx - movl %edx, (%esp) /* &resp */ - movl %ebx, 8(%esp) -LCFI7: - call L_ffi_closure_SYSV_inner$stub - movl 8(%esp), %ebx - movl -12(%ebp), %ecx - cmpl $FFI_TYPE_INT, %eax - je Lcls_retint - cmpl $FFI_TYPE_FLOAT, %eax - je Lcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je Lcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je Lcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je Lcls_retllong - cmpl $FFI_TYPE_UINT8, %eax - je Lcls_retstruct1 - cmpl $FFI_TYPE_SINT8, %eax - je Lcls_retstruct1 - cmpl $FFI_TYPE_UINT16, %eax - je Lcls_retstruct2 - cmpl $FFI_TYPE_SINT16, %eax - je Lcls_retstruct2 - cmpl $FFI_TYPE_STRUCT, %eax - je Lcls_retstruct -Lcls_epilogue: - movl %ebp, %esp - popl %ebp - ret -Lcls_retint: - movl (%ecx), %eax - jmp Lcls_epilogue -Lcls_retfloat: - flds (%ecx) - jmp Lcls_epilogue -Lcls_retdouble: - fldl (%ecx) - jmp Lcls_epilogue -Lcls_retldouble: - fldt (%ecx) - jmp Lcls_epilogue -Lcls_retllong: - movl (%ecx), %eax - movl 4(%ecx), %edx - jmp Lcls_epilogue -Lcls_retstruct1: - movsbl (%ecx), %eax - jmp Lcls_epilogue -Lcls_retstruct2: - movswl (%ecx), %eax - jmp Lcls_epilogue -Lcls_retstruct: - lea -8(%ebp),%esp - movl %ebp, %esp - popl %ebp - ret $4 -LFE2: - -#if !FFI_NO_RAW_API - -#define RAW_CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) & ~3) -#define RAW_CLOSURE_FUN_OFFSET (RAW_CLOSURE_CIF_OFFSET + 4) -#define RAW_CLOSURE_USER_DATA_OFFSET (RAW_CLOSURE_FUN_OFFSET + 4) -#define CIF_FLAGS_OFFSET 20 - - .align 4 -FFI_HIDDEN (ffi_closure_raw_SYSV) -.globl _ffi_closure_raw_SYSV - -_ffi_closure_raw_SYSV: -LFB3: - pushl %ebp -LCFI4: - movl %esp, %ebp -LCFI5: - pushl %esi -LCFI6: - subl $36, %esp - movl RAW_CLOSURE_CIF_OFFSET(%eax), %esi /* closure->cif */ - movl RAW_CLOSURE_USER_DATA_OFFSET(%eax), %edx /* closure->user_data */ - movl %edx, 12(%esp) /* user_data */ - leal 8(%ebp), %edx /* __builtin_dwarf_cfa () */ - movl %edx, 8(%esp) /* raw_args */ - leal -24(%ebp), %edx - movl %edx, 4(%esp) /* &res */ - movl %esi, (%esp) /* cif */ - call *RAW_CLOSURE_FUN_OFFSET(%eax) /* closure->fun */ - movl CIF_FLAGS_OFFSET(%esi), %eax /* rtype */ - cmpl $FFI_TYPE_INT, %eax - je Lrcls_retint - cmpl $FFI_TYPE_FLOAT, %eax - je Lrcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je Lrcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je Lrcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je Lrcls_retllong -Lrcls_epilogue: - addl $36, %esp - popl %esi - popl %ebp - ret -Lrcls_retint: - movl -24(%ebp), %eax - jmp Lrcls_epilogue -Lrcls_retfloat: - flds -24(%ebp) - jmp Lrcls_epilogue -Lrcls_retdouble: - fldl -24(%ebp) - jmp Lrcls_epilogue -Lrcls_retldouble: - fldt -24(%ebp) - jmp Lrcls_epilogue -Lrcls_retllong: - movl -24(%ebp), %eax - movl -20(%ebp), %edx - jmp Lrcls_epilogue -LFE3: -#endif - -.section __IMPORT,__jump_table,symbol_stubs,self_modifying_code+pure_instructions,5 -L_ffi_closure_SYSV_inner$stub: - .indirect_symbol _ffi_closure_SYSV_inner - hlt ; hlt ; hlt ; hlt ; hlt - - -.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support -EH_frame1: - .set L$set$0,LECIE1-LSCIE1 - .long L$set$0 -LSCIE1: - .long 0x0 - .byte 0x1 - .ascii "zR\0" - .byte 0x1 - .byte 0x7c - .byte 0x8 - .byte 0x1 - .byte 0x10 - .byte 0xc - .byte 0x5 - .byte 0x4 - .byte 0x88 - .byte 0x1 - .align 2 -LECIE1: -.globl _ffi_call_SYSV.eh -_ffi_call_SYSV.eh: -LSFDE1: - .set L$set$1,LEFDE1-LASFDE1 - .long L$set$1 -LASFDE1: - .long LASFDE1-EH_frame1 - .long LFB1-. - .set L$set$2,LFE1-LFB1 - .long L$set$2 - .byte 0x0 - .byte 0x4 - .set L$set$3,LCFI0-LFB1 - .long L$set$3 - .byte 0xe - .byte 0x8 - .byte 0x84 - .byte 0x2 - .byte 0x4 - .set L$set$4,LCFI1-LCFI0 - .long L$set$4 - .byte 0xd - .byte 0x4 - .align 2 -LEFDE1: -.globl _ffi_closure_SYSV.eh -_ffi_closure_SYSV.eh: -LSFDE2: - .set L$set$5,LEFDE2-LASFDE2 - .long L$set$5 -LASFDE2: - .long LASFDE2-EH_frame1 - .long LFB2-. - .set L$set$6,LFE2-LFB2 - .long L$set$6 - .byte 0x0 - .byte 0x4 - .set L$set$7,LCFI2-LFB2 - .long L$set$7 - .byte 0xe - .byte 0x8 - .byte 0x84 - .byte 0x2 - .byte 0x4 - .set L$set$8,LCFI3-LCFI2 - .long L$set$8 - .byte 0xd - .byte 0x4 - .align 2 -LEFDE2: - -#if !FFI_NO_RAW_API - -.globl _ffi_closure_raw_SYSV.eh -_ffi_closure_raw_SYSV.eh: -LSFDE3: - .set L$set$10,LEFDE3-LASFDE3 - .long L$set$10 -LASFDE3: - .long LASFDE3-EH_frame1 - .long LFB3-. - .set L$set$11,LFE3-LFB3 - .long L$set$11 - .byte 0x0 - .byte 0x4 - .set L$set$12,LCFI4-LFB3 - .long L$set$12 - .byte 0xe - .byte 0x8 - .byte 0x84 - .byte 0x2 - .byte 0x4 - .set L$set$13,LCFI5-LCFI4 - .long L$set$13 - .byte 0xd - .byte 0x4 - .byte 0x4 - .set L$set$14,LCFI6-LCFI5 - .long L$set$14 - .byte 0x85 - .byte 0x3 - .align 2 -LEFDE3: - -#endif - -#endif /* ifndef __x86_64__ */ - -#endif /* defined __i386__ */ diff --git a/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c b/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c deleted file mode 100644 index 8e7d01648802..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c +++ /dev/null @@ -1,737 +0,0 @@ -#ifdef __x86_64__ - -/* ----------------------------------------------------------------------- - x86-ffi64.c - Copyright (c) 2002 Bo Thorsen <bo at suse.de> - - x86-64 Foreign Function Interface - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdlib.h> -#include <stdarg.h> - -#define MAX_GPR_REGS 6 -#define MAX_SSE_REGS 8 - -typedef struct RegisterArgs { - /* Registers for argument passing. */ - UINT64 gpr[MAX_GPR_REGS]; - __int128_t sse[MAX_SSE_REGS]; -} RegisterArgs; - -extern void -ffi_call_unix64( - void* args, - unsigned long bytes, - unsigned flags, - void* raddr, - void (*fnaddr)(void), - unsigned ssecount); - -/* All reference to register classes here is identical to the code in - gcc/config/i386/i386.c. Do *not* change one without the other. */ - -/* Register class used for passing given 64bit part of the argument. - These represent classes as documented by the PS ABI, with the exception - of SSESF, SSEDF classes, that are basically SSE class, just gcc will - use SF or DFmode move instead of DImode to avoid reformating penalties. - - Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves - whenever possible (upper half does contain padding). */ -enum x86_64_reg_class -{ - X86_64_NO_CLASS, - X86_64_INTEGER_CLASS, - X86_64_INTEGERSI_CLASS, - X86_64_SSE_CLASS, - X86_64_SSESF_CLASS, - X86_64_SSEDF_CLASS, - X86_64_SSEUP_CLASS, - X86_64_X87_CLASS, - X86_64_X87UP_CLASS, - X86_64_COMPLEX_X87_CLASS, - X86_64_MEMORY_CLASS -}; - -#define MAX_CLASSES 4 -#define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS) - -/* x86-64 register passing implementation. See x86-64 ABI for details. Goal - of this code is to classify each 8bytes of incoming argument by the register - class and assign registers accordingly. */ - -/* Return the union class of CLASS1 and CLASS2. - See the x86-64 PS ABI for details. */ -static enum x86_64_reg_class -merge_classes( - enum x86_64_reg_class class1, - enum x86_64_reg_class class2) -{ - /* Rule #1: If both classes are equal, this is the resulting class. */ - if (class1 == class2) - return class1; - - /* Rule #2: If one of the classes is NO_CLASS, the resulting class is - the other class. */ - if (class1 == X86_64_NO_CLASS) - return class2; - - if (class2 == X86_64_NO_CLASS) - return class1; - - /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */ - if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS) - return X86_64_MEMORY_CLASS; - - /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */ - if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS) - || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS)) - return X86_64_INTEGERSI_CLASS; - - if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS - || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS) - return X86_64_INTEGER_CLASS; - - /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class, - MEMORY is used. */ - if (class1 == X86_64_X87_CLASS - || class1 == X86_64_X87UP_CLASS - || class1 == X86_64_COMPLEX_X87_CLASS - || class2 == X86_64_X87_CLASS - || class2 == X86_64_X87UP_CLASS - || class2 == X86_64_COMPLEX_X87_CLASS) - return X86_64_MEMORY_CLASS; - - /* Rule #6: Otherwise class SSE is used. */ - return X86_64_SSE_CLASS; -} - -/* Classify the argument of type TYPE and mode MODE. - CLASSES will be filled by the register class used to pass each word - of the operand. The number of words is returned. In case the parameter - should be passed in memory, 0 is returned. As a special case for zero - sized containers, classes[0] will be NO_CLASS and 1 is returned. - - See the x86-64 PS ABI for details. */ - -static int -classify_argument( - ffi_type* type, - enum x86_64_reg_class classes[], - size_t byte_offset) -{ - switch (type->type) - { - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - case FFI_TYPE_POINTER: -#if 0 - if (byte_offset + type->size <= 4) - classes[0] = X86_64_INTEGERSI_CLASS; - else - classes[0] = X86_64_INTEGER_CLASS; - - return 1; -#else - { - int size = byte_offset + type->size; - - if (size <= 4) - { - classes[0] = X86_64_INTEGERSI_CLASS; - return 1; - } - else if (size <= 8) - { - classes[0] = X86_64_INTEGER_CLASS; - return 1; - } - else if (size <= 12) - { - classes[0] = X86_64_INTEGER_CLASS; - classes[1] = X86_64_INTEGERSI_CLASS; - return 2; - } - else if (size <= 16) - { - classes[0] = classes[1] = X86_64_INTEGERSI_CLASS; - return 2; - } - else - FFI_ASSERT (0); - } -#endif - - case FFI_TYPE_FLOAT: - if (byte_offset == 0) - classes[0] = X86_64_SSESF_CLASS; - else - classes[0] = X86_64_SSE_CLASS; - - return 1; - - case FFI_TYPE_DOUBLE: - classes[0] = X86_64_SSEDF_CLASS; - return 1; - - case FFI_TYPE_LONGDOUBLE: - classes[0] = X86_64_X87_CLASS; - classes[1] = X86_64_X87UP_CLASS; - return 2; - - case FFI_TYPE_STRUCT: - { - ffi_type** ptr; - int i; - enum x86_64_reg_class subclasses[MAX_CLASSES]; - const int UNITS_PER_WORD = 8; - int words = - (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD; - - /* If the struct is larger than 16 bytes, pass it on the stack. */ - if (type->size > 16) - return 0; - - for (i = 0; i < words; i++) - classes[i] = X86_64_NO_CLASS; - - /* Merge the fields of structure. */ - for (ptr = type->elements; *ptr != NULL; ptr++) - { - int num, pos; - - byte_offset = ALIGN(byte_offset, (*ptr)->alignment); - - num = classify_argument(*ptr, subclasses, byte_offset % 8); - - if (num == 0) - return 0; - - pos = byte_offset / 8; - - for (i = 0; i < num; i++) - { - classes[i + pos] = - merge_classes(subclasses[i], classes[i + pos]); - } - - byte_offset += (*ptr)->size; - } - - if (words > 2) - { - /* When size > 16 bytes, if the first one isn't - X86_64_SSE_CLASS or any other ones aren't - X86_64_SSEUP_CLASS, everything should be passed in - memory. */ - if (classes[0] != X86_64_SSE_CLASS) - return 0; - - for (i = 1; i < words; i++) - if (classes[i] != X86_64_SSEUP_CLASS) - return 0; - } - - - /* Final merger cleanup. */ - for (i = 0; i < words; i++) - { - /* If one class is MEMORY, everything should be passed in - memory. */ - if (classes[i] == X86_64_MEMORY_CLASS) - return 0; - - /* The X86_64_SSEUP_CLASS should be always preceded by - X86_64_SSE_CLASS. */ - if (classes[i] == X86_64_SSEUP_CLASS - && classes[i - 1] != X86_64_SSE_CLASS - && classes[i - 1] != X86_64_SSEUP_CLASS) - { - FFI_ASSERT(i != 0); - classes[i] = X86_64_SSE_CLASS; - } - - /* X86_64_X87UP_CLASS should be preceded by X86_64_X87_CLASS. */ - if (classes[i] == X86_64_X87UP_CLASS - && classes[i - 1] != X86_64_X87_CLASS) - { - FFI_ASSERT(i != 0); - classes[i] = X86_64_SSE_CLASS; - } - } - - return words; - } - - default: - FFI_ASSERT(0); - } - - return 0; /* Never reached. */ -} - -/* Examine the argument and return set number of register required in each - class. Return zero if parameter should be passed in memory, otherwise - the number of registers. */ -static int -examine_argument( - ffi_type* type, - enum x86_64_reg_class classes[MAX_CLASSES], - _Bool in_return, - int* pngpr, - int* pnsse) -{ - int n = classify_argument(type, classes, 0); - int ngpr = 0; - int nsse = 0; - int i; - - if (n == 0) - return 0; - - for (i = 0; i < n; ++i) - { - switch (classes[i]) - { - case X86_64_INTEGER_CLASS: - case X86_64_INTEGERSI_CLASS: - ngpr++; - break; - - case X86_64_SSE_CLASS: - case X86_64_SSESF_CLASS: - case X86_64_SSEDF_CLASS: - nsse++; - break; - - case X86_64_NO_CLASS: - case X86_64_SSEUP_CLASS: - break; - - case X86_64_X87_CLASS: - case X86_64_X87UP_CLASS: - case X86_64_COMPLEX_X87_CLASS: - return in_return != 0; - - default: - abort(); - } - } - - *pngpr = ngpr; - *pnsse = nsse; - - return n; -} - -/* Perform machine dependent cif processing. */ -ffi_status -ffi_prep_cif_machdep( - ffi_cif* cif) -{ - int gprcount = 0; - int ssecount = 0; - int flags = cif->rtype->type; - int i, avn, n, ngpr, nsse; - enum x86_64_reg_class classes[MAX_CLASSES]; - size_t bytes; - - if (flags != FFI_TYPE_VOID) - { - n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); - - if (n == 0) - { - /* The return value is passed in memory. A pointer to that - memory is the first argument. Allocate a register for it. */ - gprcount++; - - /* We don't have to do anything in asm for the return. */ - flags = FFI_TYPE_VOID; - } - else if (flags == FFI_TYPE_STRUCT) - { - /* Mark which registers the result appears in. */ - _Bool sse0 = SSE_CLASS_P(classes[0]); - _Bool sse1 = n == 2 && SSE_CLASS_P(classes[1]); - - if (sse0 && !sse1) - flags |= 1 << 8; - else if (!sse0 && sse1) - flags |= 1 << 9; - else if (sse0 && sse1) - flags |= 1 << 10; - - /* Mark the true size of the structure. */ - flags |= cif->rtype->size << 12; - } - } - - /* Go over all arguments and determine the way they should be passed. - If it's in a register and there is space for it, let that be so. If - not, add it's size to the stack byte count. */ - for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++) - { - if (examine_argument(cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0 - || gprcount + ngpr > MAX_GPR_REGS - || ssecount + nsse > MAX_SSE_REGS) - { - long align = cif->arg_types[i]->alignment; - - if (align < 8) - align = 8; - - bytes = ALIGN(bytes, align); - bytes += cif->arg_types[i]->size; - } - else - { - gprcount += ngpr; - ssecount += nsse; - } - } - - if (ssecount) - flags |= 1 << 11; - - cif->flags = flags; - cif->bytes = bytes; - cif->bytes = ALIGN(bytes,8); - - return FFI_OK; -} - -void -ffi_call( - ffi_cif* cif, - void (*fn)(void), - void* rvalue, - void** avalue) -{ - enum x86_64_reg_class classes[MAX_CLASSES]; - char* stack; - char* argp; - ffi_type** arg_types; - int gprcount, ssecount, ngpr, nsse, i, avn; - _Bool ret_in_memory; - RegisterArgs* reg_args; - - /* Can't call 32-bit mode from 64-bit mode. */ - FFI_ASSERT(cif->abi == FFI_UNIX64); - - /* If the return value is a struct and we don't have a return value - address then we need to make one. Note the setting of flags to - VOID above in ffi_prep_cif_machdep. */ - ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT - && (cif->flags & 0xff) == FFI_TYPE_VOID); - - if (rvalue == NULL && ret_in_memory) - rvalue = alloca (cif->rtype->size); - - /* Allocate the space for the arguments, plus 4 words of temp space. */ - stack = alloca(sizeof(RegisterArgs) + cif->bytes + 4 * 8); - reg_args = (RegisterArgs*)stack; - argp = stack + sizeof(RegisterArgs); - - gprcount = ssecount = 0; - - /* If the return value is passed in memory, add the pointer as the - first integer argument. */ - if (ret_in_memory) - reg_args->gpr[gprcount++] = (long) rvalue; - - avn = cif->nargs; - arg_types = cif->arg_types; - - for (i = 0; i < avn; ++i) - { - size_t size = arg_types[i]->size; - int n; - - n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse); - - if (n == 0 - || gprcount + ngpr > MAX_GPR_REGS - || ssecount + nsse > MAX_SSE_REGS) - { - long align = arg_types[i]->alignment; - - /* Stack arguments are *always* at least 8 byte aligned. */ - if (align < 8) - align = 8; - - /* Pass this argument in memory. */ - argp = (void *) ALIGN (argp, align); - memcpy (argp, avalue[i], size); - argp += size; - } - else - { /* The argument is passed entirely in registers. */ - char *a = (char *) avalue[i]; - int j; - - for (j = 0; j < n; j++, a += 8, size -= 8) - { - switch (classes[j]) - { - case X86_64_INTEGER_CLASS: - case X86_64_INTEGERSI_CLASS: - reg_args->gpr[gprcount] = 0; - switch (arg_types[i]->type) { - case FFI_TYPE_SINT8: - { - int8_t shortval = *(int8_t*)a; - int64_t actval = (int64_t)shortval; - reg_args->gpr[gprcount] = actval; - /*memcpy (®_args->gpr[gprcount], &actval, 8);*/ - break; - } - - case FFI_TYPE_SINT16: - { - int16_t shortval = *(int16_t*)a; - int64_t actval = (int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - case FFI_TYPE_SINT32: - { - int32_t shortval = *(int32_t*)a; - int64_t actval = (int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - case FFI_TYPE_UINT8: - { - u_int8_t shortval = *(u_int8_t*)a; - u_int64_t actval = (u_int64_t)shortval; - /*memcpy (®_args->gpr[gprcount], &actval, 8);*/ - reg_args->gpr[gprcount] = actval; - break; - } - - case FFI_TYPE_UINT16: - { - u_int16_t shortval = *(u_int16_t*)a; - u_int64_t actval = (u_int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - case FFI_TYPE_UINT32: - { - u_int32_t shortval = *(u_int32_t*)a; - u_int64_t actval = (u_int64_t)shortval; - memcpy (®_args->gpr[gprcount], &actval, 8); - break; - } - - default: - //memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8); - reg_args->gpr[gprcount] = *(int64_t*)a; - } - gprcount++; - break; - - case X86_64_SSE_CLASS: - case X86_64_SSEDF_CLASS: - reg_args->sse[ssecount++] = *(UINT64 *) a; - break; - - case X86_64_SSESF_CLASS: - reg_args->sse[ssecount++] = *(UINT32 *) a; - break; - - default: - abort(); - } - } - } - } - - ffi_call_unix64 (stack, cif->bytes + sizeof(RegisterArgs), - cif->flags, rvalue, fn, ssecount); -} - -extern void ffi_closure_unix64(void); - -ffi_status -ffi_prep_closure( - ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*, void*, void**, void*), - void* user_data) -{ - volatile unsigned short* tramp; - - if (cif->abi != FFI_UNIX64) - return FFI_BAD_ABI; - - tramp = (volatile unsigned short*)&closure->tramp[0]; - - tramp[0] = 0xbb49; /* mov <code>, %r11 */ - *(void* volatile*)&tramp[1] = ffi_closure_unix64; - tramp[5] = 0xba49; /* mov <data>, %r10 */ - *(void* volatile*)&tramp[6] = closure; - - /* Set the carry bit if the function uses any sse registers. - This is clc or stc, together with the first byte of the jmp. */ - tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8; - tramp[11] = 0xe3ff; /* jmp *%r11 */ - - closure->cif = cif; - closure->fun = fun; - closure->user_data = user_data; - - return FFI_OK; -} - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wmissing-prototypes" -int -ffi_closure_unix64_inner( - ffi_closure* closure, - void* rvalue, - RegisterArgs* reg_args, - char* argp) -#pragma clang diagnostic pop -{ - ffi_cif* cif = closure->cif; - void** avalue = alloca(cif->nargs * sizeof(void *)); - ffi_type** arg_types; - long i, avn; - int gprcount = 0; - int ssecount = 0; - int ngpr, nsse; - int ret; - - ret = cif->rtype->type; - - if (ret != FFI_TYPE_VOID) - { - enum x86_64_reg_class classes[MAX_CLASSES]; - int n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); - - if (n == 0) - { - /* The return value goes in memory. Arrange for the closure - return value to go directly back to the original caller. */ - rvalue = (void *) reg_args->gpr[gprcount++]; - - /* We don't have to do anything in asm for the return. */ - ret = FFI_TYPE_VOID; - } - else if (ret == FFI_TYPE_STRUCT && n == 2) - { - /* Mark which register the second word of the structure goes in. */ - _Bool sse0 = SSE_CLASS_P (classes[0]); - _Bool sse1 = SSE_CLASS_P (classes[1]); - - if (!sse0 && sse1) - ret |= 1 << 8; - else if (sse0 && !sse1) - ret |= 1 << 9; - } - } - - avn = cif->nargs; - arg_types = cif->arg_types; - - for (i = 0; i < avn; ++i) - { - enum x86_64_reg_class classes[MAX_CLASSES]; - int n; - - n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse); - - if (n == 0 - || gprcount + ngpr > MAX_GPR_REGS - || ssecount + nsse > MAX_SSE_REGS) - { - long align = arg_types[i]->alignment; - - /* Stack arguments are *always* at least 8 byte aligned. */ - if (align < 8) - align = 8; - - /* Pass this argument in memory. */ - argp = (void *) ALIGN (argp, align); - avalue[i] = argp; - argp += arg_types[i]->size; - } - -#if !defined(X86_DARWIN) - /* If the argument is in a single register, or two consecutive - registers, then we can use that address directly. */ - else if (n == 1 || (n == 2 && - SSE_CLASS_P (classes[0]) == SSE_CLASS_P (classes[1]))) - { - // The argument is in a single register. - if (SSE_CLASS_P (classes[0])) - { - avalue[i] = ®_args->sse[ssecount]; - ssecount += n; - } - else - { - avalue[i] = ®_args->gpr[gprcount]; - gprcount += n; - } - } -#endif - - /* Otherwise, allocate space to make them consecutive. */ - else - { - char *a = alloca (16); - int j; - - avalue[i] = a; - - for (j = 0; j < n; j++, a += 8) - { - if (SSE_CLASS_P (classes[j])) - memcpy (a, ®_args->sse[ssecount++], 8); - else - memcpy (a, ®_args->gpr[gprcount++], 8); - } - } - } - - /* Invoke the closure. */ - closure->fun (cif, rvalue, avalue, closure->user_data); - - /* Tell assembly how to perform return type promotions. */ - return ret; -} - -#endif /* __x86_64__ */ diff --git a/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c b/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c deleted file mode 100644 index 706ea0f51206..000000000000 --- a/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c +++ /dev/null @@ -1,438 +0,0 @@ -#ifdef __i386__ -/* ----------------------------------------------------------------------- - ffi.c - Copyright (c) 1996, 1998, 1999, 2001 Red Hat, Inc. - Copyright (c) 2002 Ranjit Mathew - Copyright (c) 2002 Bo Thorsen - Copyright (c) 2002 Roger Sayle - - x86 Foreign Function Interface - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- */ - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdlib.h> - -/* ffi_prep_args is called by the assembly routine once stack space - has been allocated for the function's arguments */ - -void ffi_prep_args(char *stack, extended_cif *ecif); - -void ffi_prep_args(char *stack, extended_cif *ecif) -{ - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - - argp = stack; - - if (ecif->cif->flags == FFI_TYPE_STRUCT) - { - *(void **) argp = ecif->rvalue; - argp += 4; - } - - p_argv = ecif->avalue; - - for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; - i != 0; - i--, p_arg++) - { - size_t z; - - /* Align if necessary */ - if ((sizeof(int) - 1) & (unsigned) argp) - argp = (char *) ALIGN(argp, sizeof(int)); - - z = (*p_arg)->size; - if (z < sizeof(int)) - { - z = sizeof(int); - switch ((*p_arg)->type) - { - case FFI_TYPE_SINT8: - *(signed int *) argp = (signed int)*(SINT8 *)(* p_argv); - break; - - case FFI_TYPE_UINT8: - *(unsigned int *) argp = (unsigned int)*(UINT8 *)(* p_argv); - break; - - case FFI_TYPE_SINT16: - *(signed int *) argp = (signed int)*(SINT16 *)(* p_argv); - break; - - case FFI_TYPE_UINT16: - *(unsigned int *) argp = (unsigned int)*(UINT16 *)(* p_argv); - break; - - case FFI_TYPE_SINT32: - *(signed int *) argp = (signed int)*(SINT32 *)(* p_argv); - break; - - case FFI_TYPE_UINT32: - *(unsigned int *) argp = (unsigned int)*(UINT32 *)(* p_argv); - break; - - case FFI_TYPE_STRUCT: - *(unsigned int *) argp = (unsigned int)*(UINT32 *)(* p_argv); - break; - - default: - FFI_ASSERT(0); - } - } - else - { - memcpy(argp, *p_argv, z); - } - p_argv++; - argp += z; - } - - return; -} - -/* Perform machine dependent cif processing */ -ffi_status ffi_prep_cif_machdep(ffi_cif *cif) -{ - /* Set the return type flag */ - switch (cif->rtype->type) - { - case FFI_TYPE_VOID: -#ifdef X86 - case FFI_TYPE_STRUCT: - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: -#endif - - case FFI_TYPE_SINT64: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: - cif->flags = (unsigned) cif->rtype->type; - break; - - case FFI_TYPE_UINT64: - cif->flags = FFI_TYPE_SINT64; - break; - -#ifndef X86 - case FFI_TYPE_STRUCT: - if (cif->rtype->size == 1) - { - cif->flags = FFI_TYPE_SINT8; /* same as char size */ - } - else if (cif->rtype->size == 2) - { - cif->flags = FFI_TYPE_SINT16; /* same as short size */ - } - else if (cif->rtype->size == 4) - { - cif->flags = FFI_TYPE_INT; /* same as int type */ - } - else if (cif->rtype->size == 8) - { - cif->flags = FFI_TYPE_SINT64; /* same as int64 type */ - } - else - { - cif->flags = FFI_TYPE_STRUCT; - } - break; -#endif - - default: - cif->flags = FFI_TYPE_INT; - break; - } - -#ifdef X86_DARWIN - cif->bytes = (cif->bytes + 15) & ~0xF; -#endif - - return FFI_OK; -} - -extern void ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, - unsigned, unsigned, unsigned *, void (*fn)()); - -#ifdef X86_WIN32 -extern void ffi_call_STDCALL(void (*)(char *, extended_cif *), extended_cif *, - unsigned, unsigned, unsigned *, void (*fn)()); - -#endif /* X86_WIN32 */ - -void ffi_call(ffi_cif *cif, void (*fn)(), void *rvalue, void **avalue) -{ - extended_cif ecif; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ - - if ((rvalue == NULL) && - (cif->flags == FFI_TYPE_STRUCT)) - { - ecif.rvalue = alloca(cif->rtype->size); - } - else - ecif.rvalue = rvalue; - - - switch (cif->abi) - { - case FFI_SYSV: - ffi_call_SYSV(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, - fn); - break; -#ifdef X86_WIN32 - case FFI_STDCALL: - ffi_call_STDCALL(ffi_prep_args, &ecif, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; -#endif /* X86_WIN32 */ - default: - FFI_ASSERT(0); - break; - } -} - - -/** private members **/ - -static void ffi_prep_incoming_args_SYSV (char *stack, void **ret, - void** args, ffi_cif* cif); -void FFI_HIDDEN ffi_closure_SYSV (ffi_closure *) -__attribute__ ((regparm(1))); -unsigned int FFI_HIDDEN ffi_closure_SYSV_inner (ffi_closure *, void **, void *) -__attribute__ ((regparm(1))); -void FFI_HIDDEN ffi_closure_raw_SYSV (ffi_raw_closure *) -__attribute__ ((regparm(1))); - -/* This function is jumped to by the trampoline */ - -unsigned int FFI_HIDDEN -ffi_closure_SYSV_inner (closure, respp, args) -ffi_closure *closure; -void **respp; -void *args; -{ - // our various things... - ffi_cif *cif; - void **arg_area; - - cif = closure->cif; - arg_area = (void**) alloca (cif->nargs * sizeof (void*)); - - /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding - * value on the stack; and if the function returns - * a structure, it will re-set RESP to point to the - * structure return address. */ - - ffi_prep_incoming_args_SYSV(args, respp, arg_area, cif); - - (closure->fun) (cif, *respp, arg_area, closure->user_data); - - return cif->flags; -} - -static void -ffi_prep_incoming_args_SYSV(char *stack, void **rvalue, void **avalue, - ffi_cif *cif) -{ - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - - argp = stack; - - if ( cif->flags == FFI_TYPE_STRUCT ) { - *rvalue = *(void **) argp; - argp += 4; - } - - p_argv = avalue; - - for (i = cif->nargs, p_arg = cif->arg_types; (i != 0); i--, p_arg++) - { - size_t z; - - /* Align if necessary */ - if ((sizeof(int) - 1) & (unsigned) argp) { - argp = (char *) ALIGN(argp, sizeof(int)); - } - - z = (*p_arg)->size; - - /* because we're little endian, this is what it turns into. */ - - *p_argv = (void*) argp; - - p_argv++; - argp += z; - } - - return; -} - -/* How to make a trampoline. Derived from gcc/config/i386/i386.c. */ - -#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX) \ -({ unsigned char *__tramp = (unsigned char*)(TRAMP); \ -unsigned int __fun = (unsigned int)(FUN); \ -unsigned int __ctx = (unsigned int)(CTX); \ -unsigned int __dis = __fun - (__ctx + FFI_TRAMPOLINE_SIZE); \ -*(unsigned char*) &__tramp[0] = 0xb8; \ -*(unsigned int*) &__tramp[1] = __ctx; /* movl __ctx, %eax */ \ -*(unsigned char *) &__tramp[5] = 0xe9; \ -*(unsigned int*) &__tramp[6] = __dis; /* jmp __fun */ \ -}) - - -/* the cif must already be prep'ed */ -ffi_status -ffi_prep_closure (ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,void**,void*), - void *user_data) -{ - if (cif->abi != FFI_SYSV) - return FFI_BAD_ABI; - - FFI_INIT_TRAMPOLINE (&closure->tramp[0], \ - &ffi_closure_SYSV, \ - (void*)closure); - - closure->cif = cif; - closure->user_data = user_data; - closure->fun = fun; - - return FFI_OK; -} - -/* ------- Native raw API support -------------------------------- */ - -#if !FFI_NO_RAW_API - -ffi_status -ffi_prep_raw_closure_loc (ffi_raw_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,ffi_raw*,void*), - void *user_data, - void *codeloc) -{ - int i; - - FFI_ASSERT (cif->abi == FFI_SYSV); - - // we currently don't support certain kinds of arguments for raw - // closures. This should be implemented by a separate assembly language - // routine, since it would require argument processing, something we - // don't do now for performance. - - for (i = cif->nargs-1; i >= 0; i--) - { - FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_STRUCT); - FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_LONGDOUBLE); - } - - - FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_raw_SYSV, - codeloc); - - closure->cif = cif; - closure->user_data = user_data; - closure->fun = fun; - - return FFI_OK; -} - -static void -ffi_prep_args_raw(char *stack, extended_cif *ecif) -{ - memcpy (stack, ecif->avalue, ecif->cif->bytes); -} - -/* we borrow this routine from libffi (it must be changed, though, to - * actually call the function passed in the first argument. as of - * libffi-1.20, this is not the case.) - */ - -extern void -ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)()); - -#ifdef X86_WIN32 -extern void -ffi_call_STDCALL(void (*)(char *, extended_cif *), extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)()); -#endif /* X86_WIN32 */ - -void -ffi_raw_call(ffi_cif *cif, void (*fn)(), void *rvalue, ffi_raw *fake_avalue) -{ - extended_cif ecif; - void **avalue = (void **)fake_avalue; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ - - if ((rvalue == NULL) && - (cif->rtype->type == FFI_TYPE_STRUCT)) - { - ecif.rvalue = alloca(cif->rtype->size); - } - else - ecif.rvalue = rvalue; - - - switch (cif->abi) - { - case FFI_SYSV: - ffi_call_SYSV(ffi_prep_args_raw, &ecif, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; -#ifdef X86_WIN32 - case FFI_STDCALL: - ffi_call_STDCALL(ffi_prep_args_raw, &ecif, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; -#endif /* X86_WIN32 */ - default: - FFI_ASSERT(0); - break; - } -} - -#endif -#endif // __i386__ diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 769e27432f28..ac1721c061f0 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -52,7 +52,6 @@ def clean_lines(text): # OSX #Modules/_ctypes/darwin/*.c -#Modules/_ctypes/libffi_osx/*.c Modules/_scproxy.c # SystemConfiguration/SystemConfiguration.h # Windows diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index a324eafc52b2..6dcf61206619 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -13,11 +13,11 @@ # Excluded directories which are copies of external libraries: # don't check their coding style -EXCLUDE_DIRS = [os.path.join('Modules', '_ctypes', 'libffi_osx'), - os.path.join('Modules', '_ctypes', 'libffi_msvc'), - os.path.join('Modules', '_decimal', 'libmpdec'), - os.path.join('Modules', 'expat'), - os.path.join('Modules', 'zlib')] +EXCLUDE_DIRS = [ + os.path.join('Modules', '_decimal', 'libmpdec'), + os.path.join('Modules', 'expat'), + os.path.join('Modules', 'zlib'), + ] SRCDIR = sysconfig.get_config_var('srcdir') From webhook-mailer at python.org Wed Dec 28 14:43:25 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Wed, 28 Dec 2022 19:43:25 -0000 Subject: [Python-checkins] gh-94172: Update docs for params removed in 3.12 (#100431) Message-ID: <mailman.3121.1672256606.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9eca7235af84bb4cc29a54a16af8730b01878722 commit: 9eca7235af84bb4cc29a54a16af8730b01878722 branch: main author: Hugo van Kemenade <hugovk at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-28T14:43:19-05:00 summary: gh-94172: Update docs for params removed in 3.12 (#100431) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: M Doc/library/ftplib.rst M Doc/library/http.client.rst M Doc/library/imaplib.rst M Doc/library/poplib.rst M Doc/library/smtplib.rst M Doc/whatsnew/3.12.rst M Misc/NEWS.d/3.12.0a2.rst diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index e5ba9eef4074..47054812f9f5 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -85,7 +85,8 @@ The module defines the following items: The *encoding* parameter was added, and the default was changed from Latin-1 to UTF-8 to follow :rfc:`2640`. -.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None, *, encoding='utf-8') +.. class:: FTP_TLS(host='', user='', passwd='', acct='', *, context=None, + timeout=None, source_address=None, encoding='utf-8') A :class:`FTP` subclass which adds TLS support to FTP as described in :rfc:`4217`. @@ -96,10 +97,6 @@ The module defines the following items: options, certificates and private keys into a single (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *context* -- they - can point to PEM-formatted private key and certificate chain files - (respectively) for the SSL connection. - .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -111,7 +108,6 @@ The module defines the following items: :data:`ssl.HAS_SNI`). .. deprecated:: 3.6 - *keyfile* and *certfile* are deprecated in favor of *context*. Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let :func:`ssl.create_default_context` select the system's trusted CA @@ -123,6 +119,9 @@ The module defines the following items: The *encoding* parameter was added, and the default was changed from Latin-1 to UTF-8 to follow :rfc:`2640`. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + Here's a sample session using the :class:`FTP_TLS` class:: >>> ftps = FTP_TLS('ftp.pureftpd.org') diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 06f92510a5e4..48582219695b 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -20,7 +20,7 @@ HTTPS protocols. It is normally not used directly --- the module .. seealso:: - The `Requests package <https://requests.readthedocs.io/en/master/>`_ + The `Requests package <https://requests.readthedocs.io/en/latest/>`_ is recommended for a higher-level HTTP client interface. .. note:: @@ -67,10 +67,9 @@ The module provides the following classes: *blocksize* parameter was added. -.. class:: HTTPSConnection(host, port=None, key_file=None, \ - cert_file=None[, timeout], \ - source_address=None, *, context=None, \ - check_hostname=None, blocksize=8192) +.. class:: HTTPSConnection(host, port=None, *[, timeout], \ + source_address=None, context=None, \ + blocksize=8192) A subclass of :class:`HTTPConnection` that uses SSL for communication with secure servers. Default port is ``443``. If *context* is specified, it @@ -96,6 +95,16 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. + .. deprecated:: 3.6 + *key_file* and *cert_file* are deprecated in favor of *context*. + Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let + :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. + + The *check_hostname* parameter is also deprecated; the + :attr:`ssl.SSLContext.check_hostname` attribute of *context* should + be used instead. + .. versionchanged:: 3.8 This class now enables TLS 1.3 :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or @@ -106,16 +115,9 @@ The module provides the following classes: ``http/1.1`` when no *context* is given. Custom *context* should set ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. - .. deprecated:: 3.6 - - *key_file* and *cert_file* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - The *check_hostname* parameter is also deprecated; the - :attr:`ssl.SSLContext.check_hostname` attribute of *context* should - be used instead. + .. versionchanged:: 3.12 + The deprecated *key_file*, *cert_file* and *check_hostname* parameters + have been removed. .. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) @@ -344,11 +346,11 @@ HTTPConnection Objects Set the host and the port for HTTP Connect Tunnelling. This allows running the connection through a proxy server. - The host and port arguments specify the endpoint of the tunneled connection + The *host* and *port* arguments specify the endpoint of the tunneled connection (i.e. the address included in the CONNECT request, *not* the address of the proxy server). - The headers argument should be a mapping of extra HTTP headers to send with + The *headers* argument should be a mapping of extra HTTP headers to send with the CONNECT request. For example, to tunnel through a HTTPS proxy server running locally on port diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 8c28fce99ff9..f1344ae9bb0a 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -84,8 +84,8 @@ Three exceptions are defined as attributes of the :class:`IMAP4` class: There's also a subclass for secure connections: -.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, keyfile=None, \ - certfile=None, ssl_context=None, timeout=None) +.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, *, ssl_context=None, \ + timeout=None) This is a subclass derived from :class:`IMAP4` that connects over an SSL encrypted socket (to use this class you need a socket module that was compiled @@ -96,12 +96,6 @@ There's also a subclass for secure connections: (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *ssl_context* - they - can point to PEM-formatted private key and certificate chain files for - the SSL connection. Note that the *keyfile*/*certfile* parameters are - mutually exclusive with *ssl_context*, a :class:`ValueError` is raised - if *keyfile*/*certfile* is provided along with *ssl_context*. - The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. If timeout is not given or is None, the global default socket timeout is used. @@ -124,6 +118,9 @@ There's also a subclass for secure connections: .. versionchanged:: 3.9 The optional *timeout* parameter was added. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + The second subclass allows for connections created by a child process: @@ -564,7 +561,7 @@ An :class:`IMAP4` instance has the following methods: ``search``, the searching *charset* argument is mandatory. There is also a ``uid thread`` command which corresponds to ``thread`` the way that ``uid search`` corresponds to ``search``. The ``thread`` command first searches the - mailbox for messages that match the given searching criteria using the charset + mailbox for messages that match the given searching criteria using the *charset* argument for the interpretation of strings in the searching criteria. It then returns the matching messages threaded according to the specified threading algorithm. diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index e22a2e1455e7..d8618ce9b60b 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -53,7 +53,7 @@ The :mod:`poplib` module provides two classes: If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket. -.. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) +.. class:: POP3_SSL(host, port=POP3_SSL_PORT, *, timeout=None, context=None) This is a subclass of :class:`POP3` that connects to the server over an SSL encrypted socket. If *port* is not specified, 995, the standard POP3-over-SSL @@ -63,10 +63,6 @@ The :mod:`poplib` module provides two classes: single (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *context* - they can - point to PEM-formatted private key and certificate chain files, - respectively, for the SSL connection. - .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL .. audit-event:: poplib.putline self,line poplib.POP3_SSL @@ -94,6 +90,9 @@ The :mod:`poplib` module provides two classes: If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + One exception is defined as an attribute of the :mod:`poplib` module: diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 4e50ad1568ed..2539c3d38832 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -66,18 +66,17 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). Support for the :keyword:`with` statement was added. .. versionchanged:: 3.3 - source_address argument was added. + *source_address* argument was added. .. versionadded:: 3.5 The SMTPUTF8 extension (:rfc:`6531`) is now supported. .. versionchanged:: 3.9 If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket + :class:`ValueError` to prevent the creation of a non-blocking socket. -.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \ - certfile=None [, timeout], context=None, \ - source_address=None) +.. class:: SMTP_SSL(host='', port=0, local_hostname=None, * [, timeout], \ + context=None, source_address=None) An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is @@ -90,15 +89,11 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). aspects of the secure connection. Please read :ref:`ssl-security` for best practices. - *keyfile* and *certfile* are a legacy alternative to *context*, and can - point to a PEM formatted private key and certificate chain file for the - SSL connection. - .. versionchanged:: 3.3 *context* was added. .. versionchanged:: 3.3 - source_address argument was added. + The *source_address* argument was added. .. versionchanged:: 3.4 The class now supports hostname check with @@ -116,13 +111,16 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + .. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, \ source_address=None[, timeout]) The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our :meth:`connect` method must support that as well as a regular host:port - server. The optional arguments local_hostname and source_address have the + server. The optional arguments *local_hostname* and *source_address* have the same meaning as they do in the :class:`SMTP` class. To specify a Unix socket, you must use an absolute path for *host*, starting with a '/'. @@ -360,7 +358,7 @@ An :class:`SMTP` instance has the following methods: be used as argument to the ``AUTH`` command; the valid values are those listed in the ``auth`` element of :attr:`esmtp_features`. - *authobject* must be a callable object taking an optional single argument: + *authobject* must be a callable object taking an optional single argument:: data = authobject(challenge=None) @@ -393,7 +391,7 @@ An :class:`SMTP` instance has the following methods: .. versionadded:: 3.5 -.. method:: SMTP.starttls(keyfile=None, certfile=None, context=None) +.. method:: SMTP.starttls(*, context=None) Put the SMTP connection in TLS (Transport Layer Security) mode. All SMTP commands that follow will be encrypted. You should then call :meth:`ehlo` @@ -416,6 +414,9 @@ An :class:`SMTP` instance has the following methods: :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. + :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 69f97debd694..2f50ece4dab3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -631,9 +631,11 @@ Removed <https://github.com/sphinx-contrib/sphinx-lint>`_. (Contributed by Julien Palard in :gh:`98179`.) -* Remove the *keyfile*, *certfile* and *check_hostname* parameters, deprecated - since Python 3.6, in modules: :mod:`ftplib`, :mod:`http.client`, - :mod:`imaplib`, :mod:`poplib` and :mod:`smtplib`. Use the *context* parameter +* Remove the *keyfile* and *certfile* parameters from the + :mod:`ftplib`, :mod:`imaplib`, :mod:`poplib` and :mod:`smtplib` modules, + and the *key_file*, *cert_file* and *check_hostname* parameters from the + :mod:`http.client` module, + all deprecated since Python 3.6. Use the *context* parameter (*ssl_context* in :mod:`imaplib`) instead. (Contributed by Victor Stinner in :gh:`94172`.) diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 4029856ee332..318f3f71f115 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -725,9 +725,11 @@ Fix handling of ``bytes`` :term:`path-like objects <path-like object>` in .. nonce: AXE2IZ .. section: Library -Remove the *keyfile*, *certfile* and *check_hostname* parameters, deprecated -since Python 3.6, in modules: :mod:`ftplib`, :mod:`http.client`, -:mod:`imaplib`, :mod:`poplib` and :mod:`smtplib`. Use the *context* +Remove the *keyfile* and *certfile* parameters from the +:mod:`ftplib`, :mod:`imaplib`, :mod:`poplib` and :mod:`smtplib` modules, +and the *key_file*, *cert_file* and *check_hostname* parameters from the +:mod:`http.client` module, +all deprecated since Python 3.6. Use the *context* parameter (*ssl_context* in :mod:`imaplib`) instead. Patch by Victor Stinner. From webhook-mailer at python.org Wed Dec 28 15:29:33 2022 From: webhook-mailer at python.org (rhettinger) Date: Wed, 28 Dec 2022 20:29:33 -0000 Subject: [Python-checkins] Restore early-out to factor(). Strengthen tests. (GH-100591) Message-ID: <mailman.3122.1672259374.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c4c5790120beabed83ce5855f18d209ab8324434 commit: c4c5790120beabed83ce5855f18d209ab8324434 branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-28T12:29:27-08:00 summary: Restore early-out to factor(). Strengthen tests. (GH-100591) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index f1277dfdbdc6..5d6efabcef60 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -901,12 +901,14 @@ which incur interpreter overhead. "Prime factors of n." # factor(99) --> 3 3 11 for prime in sieve(math.isqrt(n) + 1): - while n >= prime: + while True: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient + if n == 1: + return if n >= 2: yield n @@ -1286,13 +1288,21 @@ which incur interpreter overhead. [3, 3] >>> list(factor(10)) [2, 5] - >>> list(factor(999953*999983)) + >>> list(factor(128_884_753_939)) # large prime + [128884753939] + >>> list(factor(999953 * 999983)) # large semiprime [999953, 999983] - >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(6 ** 20)) == [2] * 20 + [3] * 20 # large power + True + >>> list(factor(909_909_090_909)) # large multiterm composite + [3, 3, 7, 13, 13, 751, 113797] + >>> math.prod([3, 3, 7, 13, 13, 751, 113797]) + 909909090909 + >>> all(math.prod(factor(n)) == n for n in range(1, 2_000)) True - >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(2_000)) True - >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 15:38:03 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 20:38:03 -0000 Subject: [Python-checkins] Restore early-out to factor(). Strengthen tests. (GH-100591) Message-ID: <mailman.3123.1672259884.3313.python-checkins@python.org> https://github.com/python/cpython/commit/9120450b253a718ae295d7e86b5087705c177eae commit: 9120450b253a718ae295d7e86b5087705c177eae branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T12:37:58-08:00 summary: Restore early-out to factor(). Strengthen tests. (GH-100591) (cherry picked from commit c4c5790120beabed83ce5855f18d209ab8324434) Co-authored-by: Raymond Hettinger <rhettinger at users.noreply.github.com> files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 221863cbbcc5..93bc1f792f54 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -862,12 +862,14 @@ which incur interpreter overhead. "Prime factors of n." # factor(99) --> 3 3 11 for prime in sieve(math.isqrt(n) + 1): - while n >= prime: + while True: quotient, remainder = divmod(n, prime) if remainder: break yield prime n = quotient + if n == 1: + return if n >= 2: yield n @@ -1256,13 +1258,21 @@ which incur interpreter overhead. [3, 3] >>> list(factor(10)) [2, 5] - >>> list(factor(999953*999983)) + >>> list(factor(128_884_753_939)) # large prime + [128884753939] + >>> list(factor(999953 * 999983)) # large semiprime [999953, 999983] - >>> all(math.prod(factor(n)) == n for n in range(1, 1000)) + >>> list(factor(6 ** 20)) == [2] * 20 + [3] * 20 # large power + True + >>> list(factor(909_909_090_909)) # large multiterm composite + [3, 3, 7, 13, 13, 751, 113797] + >>> math.prod([3, 3, 7, 13, 13, 751, 113797]) + 909909090909 + >>> all(math.prod(factor(n)) == n for n in range(1, 2_000)) True - >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(1, 1000)) + >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(2_000)) True - >>> all(list(factor(n)) == sorted(factor(n)) for n in range(1, 1000)) + >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) From webhook-mailer at python.org Wed Dec 28 16:24:59 2022 From: webhook-mailer at python.org (brandtbucher) Date: Wed, 28 Dec 2022 21:24:59 -0000 Subject: [Python-checkins] GH-100101: Clarify documentation of zip's strict option (GH-100103) Message-ID: <mailman.3124.1672262700.3313.python-checkins@python.org> https://github.com/python/cpython/commit/cf1c09818032df3080c2cd9e7edb5f657213dc83 commit: cf1c09818032df3080c2cd9e7edb5f657213dc83 branch: main author: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> committer: brandtbucher <brandtbucher at gmail.com> date: 2022-12-28T13:24:50-08:00 summary: GH-100101: Clarify documentation of zip's strict option (GH-100103) files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2e988257d5d3..f00a072e6f41 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1923,14 +1923,24 @@ are always available. They are listed here in alphabetical order. >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) [('a', 1), ('b', 2), ('c', 3)] - Unlike the default behavior, it checks that the lengths of iterables are - identical, raising a :exc:`ValueError` if they aren't: - - >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True)) + Unlike the default behavior, it raises a :exc:`ValueError` if one iterable + is exhausted before the others: + + >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP + ... print(item) + ... + (0, 'fee') + (1, 'fi') + (2, 'fo') Traceback (most recent call last): ... ValueError: zip() argument 2 is longer than argument 1 + .. + This doctest is disabled because doctest does not support capturing + output and exceptions in the same code unit. + https://github.com/python/cpython/issues/65382 + Without the ``strict=True`` argument, any bug that results in iterables of different lengths will be silenced, possibly manifesting as a hard-to-find bug in another part of the program. From webhook-mailer at python.org Wed Dec 28 16:31:49 2022 From: webhook-mailer at python.org (jaraco) Date: Wed, 28 Dec 2022 21:31:49 -0000 Subject: [Python-checkins] gh-100585: Fixed a bug where importlib.resources.as_file was leaving file pointers open (GH-100586) Message-ID: <mailman.3125.1672263110.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f10f503b24a35a43910a632ee50c7568bedd6664 commit: f10f503b24a35a43910a632ee50c7568bedd6664 branch: main author: Samet YASLAN <sametyaslan at gmail.com> committer: jaraco <jaraco at jaraco.com> date: 2022-12-28T16:31:43-05:00 summary: gh-100585: Fixed a bug where importlib.resources.as_file was leaving file pointers open (GH-100586) * gh-100585: Fixed open fp bug in the imporlib module * Added news for gh-100585 files: A Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst M Lib/importlib/resources/_common.py diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index 8c0be7ee547e..92a37e2c12b8 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -155,5 +155,5 @@ def _write_contents(target, source): for item in source.iterdir(): _write_contents(child, item) else: - child.open('wb').write(source.read_bytes()) + child.write_bytes(source.read_bytes()) return child diff --git a/Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst b/Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst new file mode 100644 index 000000000000..0bf33b8bdd02 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-28-17-38-39.gh-issue-100585.BiiTlG.rst @@ -0,0 +1 @@ +Fixed a bug where importlib.resources.as_file was leaving file pointers open From webhook-mailer at python.org Wed Dec 28 16:32:24 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 21:32:24 -0000 Subject: [Python-checkins] GH-100101: Clarify documentation of zip's strict option (GH-100103) Message-ID: <mailman.3126.1672263145.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ac31120de7824581656de7734cbaa7333d555067 commit: ac31120de7824581656de7734cbaa7333d555067 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T13:32:19-08:00 summary: GH-100101: Clarify documentation of zip's strict option (GH-100103) (cherry picked from commit cf1c09818032df3080c2cd9e7edb5f657213dc83) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 304716194d22..c71ebd235cf3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1918,14 +1918,24 @@ are always available. They are listed here in alphabetical order. >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) [('a', 1), ('b', 2), ('c', 3)] - Unlike the default behavior, it checks that the lengths of iterables are - identical, raising a :exc:`ValueError` if they aren't: - - >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True)) + Unlike the default behavior, it raises a :exc:`ValueError` if one iterable + is exhausted before the others: + + >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP + ... print(item) + ... + (0, 'fee') + (1, 'fi') + (2, 'fo') Traceback (most recent call last): ... ValueError: zip() argument 2 is longer than argument 1 + .. + This doctest is disabled because doctest does not support capturing + output and exceptions in the same code unit. + https://github.com/python/cpython/issues/65382 + Without the ``strict=True`` argument, any bug that results in iterables of different lengths will be silenced, possibly manifesting as a hard-to-find bug in another part of the program. From webhook-mailer at python.org Wed Dec 28 16:32:50 2022 From: webhook-mailer at python.org (miss-islington) Date: Wed, 28 Dec 2022 21:32:50 -0000 Subject: [Python-checkins] GH-100101: Clarify documentation of zip's strict option (GH-100103) Message-ID: <mailman.3127.1672263171.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4217fafeac87d31ad7eb40ca0ec2bb9309333211 commit: 4217fafeac87d31ad7eb40ca0ec2bb9309333211 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-28T13:32:44-08:00 summary: GH-100101: Clarify documentation of zip's strict option (GH-100103) (cherry picked from commit cf1c09818032df3080c2cd9e7edb5f657213dc83) Co-authored-by: JustAnotherArchivist <JustAnotherArchivist at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 46e77fdb4155..5400feb35567 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1894,14 +1894,24 @@ are always available. They are listed here in alphabetical order. >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) [('a', 1), ('b', 2), ('c', 3)] - Unlike the default behavior, it checks that the lengths of iterables are - identical, raising a :exc:`ValueError` if they aren't: - - >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True)) + Unlike the default behavior, it raises a :exc:`ValueError` if one iterable + is exhausted before the others: + + >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP + ... print(item) + ... + (0, 'fee') + (1, 'fi') + (2, 'fo') Traceback (most recent call last): ... ValueError: zip() argument 2 is longer than argument 1 + .. + This doctest is disabled because doctest does not support capturing + output and exceptions in the same code unit. + https://github.com/python/cpython/issues/65382 + Without the ``strict=True`` argument, any bug that results in iterables of different lengths will be silenced, possibly manifesting as a hard-to-find bug in another part of the program. From webhook-mailer at python.org Thu Dec 29 10:14:19 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 29 Dec 2022 15:14:19 -0000 Subject: [Python-checkins] gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (#100601) Message-ID: <mailman.3128.1672326860.3313.python-checkins@python.org> https://github.com/python/cpython/commit/76856366d3ece34c3e738f7167329e97bbf52b34 commit: 76856366d3ece34c3e738f7167329e97bbf52b34 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-29T20:43:38+05:30 summary: gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (#100601) files: M Lib/test/test_coroutines.py diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 43a3ff0536fe..6ab19efcc588 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2214,6 +2214,7 @@ async def f(): gen = f() with self.assertWarns(RuntimeWarning): gen.cr_frame.clear() + gen.close() def test_stack_in_coroutine_throw(self): # Regression test for https://github.com/python/cpython/issues/93592 From webhook-mailer at python.org Thu Dec 29 10:17:26 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 29 Dec 2022 15:17:26 -0000 Subject: [Python-checkins] gh-100583: Improve the `pydoc` documentation (#100590) Message-ID: <mailman.3129.1672327047.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7223d50b9785bc7b0cd76dcc68d97dabcbade4b6 commit: 7223d50b9785bc7b0cd76dcc68d97dabcbade4b6 branch: main author: ram vikram singh <ramvikrams243 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-29T20:47:20+05:30 summary: gh-100583: Improve the `pydoc` documentation (#100590) files: M Doc/library/pydoc.rst diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 94daf4a58f9c..03e0915bf6d1 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -33,7 +33,7 @@ as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: - pydoc sys + python -m pydoc sys at a shell prompt will display documentation on the :mod:`sys` module, in a style similar to the manual pages shown by the Unix :program:`man` command. The @@ -65,18 +65,18 @@ manner similar to the Unix :program:`man` command. The synopsis line of a module is the first line of its documentation string. You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting web browsers. :program:`pydoc -p 1234` +that will serve documentation to visiting web browsers. :program:`python -m pydoc -p 1234` will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. -:program:`pydoc -n <hostname>` will start the server listening at the given +:program:`python -m pydoc -n <hostname>` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the server responds to. During development this is especially useful if you want to run pydoc from within a container. -:program:`pydoc -b` will start the server and additionally open a web +:program:`python -m pydoc -b` will start the server and additionally open a web browser to a module index page. Each served page has a navigation bar at the top where you can *Get* help on an individual item, *Search* all modules with a keyword in their synopsis line, and go to the *Module index*, *Topics* and From webhook-mailer at python.org Thu Dec 29 10:24:17 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 29 Dec 2022 15:24:17 -0000 Subject: [Python-checkins] gh-100583: Improve the `pydoc` documentation (GH-100590) Message-ID: <mailman.3130.1672327459.3313.python-checkins@python.org> https://github.com/python/cpython/commit/e180dff4c7fff1d0b2d66e6be3e9c06835a130ef commit: e180dff4c7fff1d0b2d66e6be3e9c06835a130ef branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-29T07:24:11-08:00 summary: gh-100583: Improve the `pydoc` documentation (GH-100590) (cherry picked from commit 7223d50b9785bc7b0cd76dcc68d97dabcbade4b6) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/pydoc.rst diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 94daf4a58f9c..03e0915bf6d1 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -33,7 +33,7 @@ as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: - pydoc sys + python -m pydoc sys at a shell prompt will display documentation on the :mod:`sys` module, in a style similar to the manual pages shown by the Unix :program:`man` command. The @@ -65,18 +65,18 @@ manner similar to the Unix :program:`man` command. The synopsis line of a module is the first line of its documentation string. You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting web browsers. :program:`pydoc -p 1234` +that will serve documentation to visiting web browsers. :program:`python -m pydoc -p 1234` will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. -:program:`pydoc -n <hostname>` will start the server listening at the given +:program:`python -m pydoc -n <hostname>` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the server responds to. During development this is especially useful if you want to run pydoc from within a container. -:program:`pydoc -b` will start the server and additionally open a web +:program:`python -m pydoc -b` will start the server and additionally open a web browser to a module index page. Each served page has a navigation bar at the top where you can *Get* help on an individual item, *Search* all modules with a keyword in their synopsis line, and go to the *Module index*, *Topics* and From webhook-mailer at python.org Thu Dec 29 10:41:40 2022 From: webhook-mailer at python.org (miss-islington) Date: Thu, 29 Dec 2022 15:41:40 -0000 Subject: [Python-checkins] gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (GH-100601) Message-ID: <mailman.3131.1672328501.3313.python-checkins@python.org> https://github.com/python/cpython/commit/dd520e7bae18549971b900b6dadc69bfc35d4b3f commit: dd520e7bae18549971b900b6dadc69bfc35d4b3f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-29T07:41:34-08:00 summary: gh-100600: Fix "coroutine was never awaited" warning in `test_coroutines` (GH-100601) (cherry picked from commit 76856366d3ece34c3e738f7167329e97bbf52b34) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: M Lib/test/test_coroutines.py diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 10f1a9efbcbd..93ddbf6e8976 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2207,6 +2207,7 @@ async def f(): gen = f() with self.assertWarns(RuntimeWarning): gen.cr_frame.clear() + gen.close() def test_stack_in_coroutine_throw(self): # Regression test for https://github.com/python/cpython/issues/93592 From webhook-mailer at python.org Thu Dec 29 11:15:20 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 29 Dec 2022 16:15:20 -0000 Subject: [Python-checkins] [3.10] gh-100583: Improve the `pydoc` documentation (GH-100590) (#100607) Message-ID: <mailman.3132.1672330520.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2b1cc33db0b413c42393ea3a9b9cdf2c585d804a commit: 2b1cc33db0b413c42393ea3a9b9cdf2c585d804a branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-29T21:45:13+05:30 summary: [3.10] gh-100583: Improve the `pydoc` documentation (GH-100590) (#100607) gh-100583: Improve the `pydoc` documentation (GH-100590) (cherry picked from commit 7223d50b9785bc7b0cd76dcc68d97dabcbade4b6) Co-authored-by: ram vikram singh <ramvikrams243 at gmail.com> files: M Doc/library/pydoc.rst diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 94daf4a58f9c..03e0915bf6d1 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -33,7 +33,7 @@ as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: - pydoc sys + python -m pydoc sys at a shell prompt will display documentation on the :mod:`sys` module, in a style similar to the manual pages shown by the Unix :program:`man` command. The @@ -65,18 +65,18 @@ manner similar to the Unix :program:`man` command. The synopsis line of a module is the first line of its documentation string. You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting web browsers. :program:`pydoc -p 1234` +that will serve documentation to visiting web browsers. :program:`python -m pydoc -p 1234` will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. -:program:`pydoc -n <hostname>` will start the server listening at the given +:program:`python -m pydoc -n <hostname>` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the server responds to. During development this is especially useful if you want to run pydoc from within a container. -:program:`pydoc -b` will start the server and additionally open a web +:program:`python -m pydoc -b` will start the server and additionally open a web browser to a module index page. Each served page has a navigation bar at the top where you can *Get* help on an individual item, *Search* all modules with a keyword in their synopsis line, and go to the *Module index*, *Topics* and From webhook-mailer at python.org Thu Dec 29 17:13:52 2022 From: webhook-mailer at python.org (zware) Date: Thu, 29 Dec 2022 22:13:52 -0000 Subject: [Python-checkins] gh-100540: Remove obsolete Modules/_ctypes/darwin/ dlfcn shim (GH-100541) Message-ID: <mailman.3133.1672352033.3313.python-checkins@python.org> https://github.com/python/cpython/commit/2df82db48506e5a2044a28f147fdb42f662d37b9 commit: 2df82db48506e5a2044a28f147fdb42f662d37b9 branch: main author: Zachary Ware <zach at python.org> committer: zware <zachary.ware at gmail.com> date: 2022-12-29T16:13:28-06:00 summary: gh-100540: Remove obsolete Modules/_ctypes/darwin/ dlfcn shim (GH-100541) As far as I can tell, this hasn't been actually used since Mac OS X 10.2. files: A Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst D Modules/_ctypes/ctypes_dlfcn.h D Modules/_ctypes/darwin/LICENSE D Modules/_ctypes/darwin/README D Modules/_ctypes/darwin/README.ctypes D Modules/_ctypes/darwin/dlfcn.h D Modules/_ctypes/darwin/dlfcn_simple.c M Makefile.pre.in M Modules/_ctypes/_ctypes.c M Modules/_ctypes/callproc.c M PCbuild/_ctypes.vcxproj M PCbuild/_ctypes.vcxproj.filters M Tools/c-analyzer/cpython/_parser.py M configure M configure.ac diff --git a/Makefile.pre.in b/Makefile.pre.in index dd6c3fbd1c64..1f8bd561f61d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2591,7 +2591,7 @@ MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h -MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h $(srcdir)/Modules/_ctypes/darwin/dlfcn.h +MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ diff --git a/Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst b/Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst new file mode 100644 index 000000000000..a42814e1861d --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2022-12-26-14-52-37.gh-issue-100540.kYZLtX.rst @@ -0,0 +1,2 @@ +Removed obsolete ``dlfcn.h`` shim from the ``_ctypes`` extension module, +which has not been necessary since Mac OS X 10.2. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index f69a37709963..4ce6433a2e45 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -120,7 +120,7 @@ bytes(cdata) #define IS_INTRESOURCE(x) (((size_t)(x) >> 16) == 0) #endif #else -#include "ctypes_dlfcn.h" +#include <dlfcn.h> #endif #include "ctypes.h" @@ -768,7 +768,7 @@ CDataType_in_dll(PyObject *type, PyObject *args) return NULL; } #else - address = (void *)ctypes_dlsym(handle, name); + address = (void *)dlsym(handle, name); if (!address) { #ifdef __CYGWIN__ /* dlerror() isn't very helpful on cygwin */ @@ -776,7 +776,7 @@ CDataType_in_dll(PyObject *type, PyObject *args) "symbol '%s' not found", name); #else - PyErr_SetString(PyExc_ValueError, ctypes_dlerror()); + PyErr_SetString(PyExc_ValueError, dlerror()); #endif return NULL; } @@ -3560,7 +3560,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else - address = (PPROC)ctypes_dlsym(handle, name); + address = (PPROC)dlsym(handle, name); if (!address) { #ifdef __CYGWIN__ /* dlerror() isn't very helpful on cygwin */ @@ -3568,7 +3568,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) "function '%s' not found", name); #else - PyErr_SetString(PyExc_AttributeError, ctypes_dlerror()); + PyErr_SetString(PyExc_AttributeError, dlerror()); #endif Py_DECREF(ftuple); return NULL; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 28b7cd406971..1958758dd0cf 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -67,7 +67,7 @@ #include <windows.h> #include <tchar.h> #else -#include "ctypes_dlfcn.h" +#include <dlfcn.h> #endif #ifdef __APPLE__ @@ -1537,10 +1537,10 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlopen", "O", name) < 0) { return NULL; } - handle = ctypes_dlopen(name_str, mode); + handle = dlopen(name_str, mode); Py_XDECREF(name2); if (!handle) { - const char *errmsg = ctypes_dlerror(); + const char *errmsg = dlerror(); if (!errmsg) errmsg = "dlopen() error"; PyErr_SetString(PyExc_OSError, @@ -1558,7 +1558,7 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) return NULL; if (dlclose(handle)) { PyErr_SetString(PyExc_OSError, - ctypes_dlerror()); + dlerror()); return NULL; } Py_RETURN_NONE; @@ -1576,10 +1576,10 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { return NULL; } - ptr = ctypes_dlsym((void*)handle, name); + ptr = dlsym((void*)handle, name); if (!ptr) { PyErr_SetString(PyExc_OSError, - ctypes_dlerror()); + dlerror()); return NULL; } return PyLong_FromVoidPtr(ptr); diff --git a/Modules/_ctypes/ctypes_dlfcn.h b/Modules/_ctypes/ctypes_dlfcn.h deleted file mode 100644 index 54cdde9a4fdb..000000000000 --- a/Modules/_ctypes/ctypes_dlfcn.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef _CTYPES_DLFCN_H_ -#define _CTYPES_DLFCN_H_ - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#ifndef MS_WIN32 - -#include <dlfcn.h> - -#ifndef CTYPES_DARWIN_DLFCN - -#define ctypes_dlsym dlsym -#define ctypes_dlerror dlerror -#define ctypes_dlopen dlopen -#define ctypes_dlclose dlclose -#define ctypes_dladdr dladdr - -#endif /* !CTYPES_DARWIN_DLFCN */ - -#endif /* !MS_WIN32 */ - -#ifdef __cplusplus -} -#endif /* __cplusplus */ -#endif /* _CTYPES_DLFCN_H_ */ diff --git a/Modules/_ctypes/darwin/LICENSE b/Modules/_ctypes/darwin/LICENSE deleted file mode 100644 index 786fb50258eb..000000000000 --- a/Modules/_ctypes/darwin/LICENSE +++ /dev/null @@ -1,31 +0,0 @@ -Copyright (c) 2002 Jorge Acereda <jacereda at users.sourceforge.net> & - Peter O'Gorman <ogorman at users.sourceforge.net> - -Portions may be copyright others, see the AUTHORS file included with this -distribution. - -Maintained by Peter O'Gorman <ogorman at users.sourceforge.net> - -Bug Reports and other queries should go to <ogorman at users.sourceforge.net> - - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - diff --git a/Modules/_ctypes/darwin/README b/Modules/_ctypes/darwin/README deleted file mode 100644 index 4d63f3dfa5eb..000000000000 --- a/Modules/_ctypes/darwin/README +++ /dev/null @@ -1,95 +0,0 @@ -dlcompat for Darwin -========================= - -This is dlcompat, a small library that emulates the dlopen() -interface on top of Darwin's dyld API. - -dlcompat allows loading a ".dylib" library (as long as the RTLD_LOCAL -flag isn't passed to dlopen()). It can be configured to yield a warning -when trying to close it (dynamic libraries cannot currently be unloaded). - -It automatically searches for modules in several directories when no -absolute path is specified and the module is not found in the current -directory. - -The paths searched are those specified in the environment variables -LD_LIBRARY_PATH and DYLD_LIBRARY_PATH plus /lib, /usr/local/lib and -/usr/lib or the path specified in the environment variable -DYLD_FALLBACK_LIBRARY_PATH. - -In the default install the behavior of dlsym is to automatically prepend -an underscore to passed in symbol names, this allows easier porting of -applications which were written specifically for ELF based lifeforms. - -Installation --------------- -Type: - ./configure - make - sudo make install - -This will compile the source file, generate both a static and shared -library called libdl and install it into /usr/local/lib. The header -file dlfcn.h will be installed in /usr/local/include. - -If you want to place the files somewhere else, run - - make clean - ./configure --prefix=<prefix> - make - sudo make install - -where <prefix> is the hierarchy you want to install into, e.g. /usr -for /usr/lib and /usr/include (_NOT_ recommended!). - -To enable debugging output (useful for me), run - - make clean - ./configure --enable-debug - make - sudo make install - -If you want old dlcompat style behavior of not prepending the underscore -on calls to dlsym then type: - - make clean - ./configure --enable-fink - make - sudo make install - -Usage -------- -Software that uses GNU autoconf will likely check for a library called -libdl, that's why I named it that way. For software that doesn't find -the library on its own, you must add a '-ldl' to the appropriate -Makefile (or environment) variable, usually LIBS. - -If you installed dlcompat into a directory other than /usr/local/lib, -you must tell the compiler where to find it. Add '-L<prefix>/lib' to -LDFLAGS (or CFLAGS) and '-I<prefix>/include' to CPPFLAGS (or CFLAGS). - -Notes ------ -If you are writing new software and plan to have Mac OX X compatibility you -should look at the dyld api's in /usr/include/mach-o/dyld.h, rather than -using dlcompat, using the native api's is the supported method of loading -dynamically on Mac OS X, if you want an small example, look at dlfcn_simple.c, -which should help get you started. - -Also note that the functions in dlcompat are not thread safe, and while it is not -POSIX spec compliant, it is about as close to compliance as it is going to get though. - -You can always get the latest version from opendarwin cvs: - - cvs -d :pserver:anonymous at anoncvs.opendarwin.org:/cvs/od login - cvs -z3 -d :pserver:anonymous at anoncvs.opendarwin.org:/cvs/od \ - co -d dlcompat proj/dlcompat - - -It is hoped that this library will be useful, and as bug free as possible, if you find -any bugs please let us know about them so they can be fixed. - -Please send bug reports to Peter O'Gorman <ogorman at users.sourceforge.net> - -Thanks. - diff --git a/Modules/_ctypes/darwin/README.ctypes b/Modules/_ctypes/darwin/README.ctypes deleted file mode 100644 index 8520b01f49da..000000000000 --- a/Modules/_ctypes/darwin/README.ctypes +++ /dev/null @@ -1,11 +0,0 @@ -The files in this directory are taken from -http://www.opendarwin.org/cgi-bin/cvsweb.cgi/~checkout~/proj/dlcompat/ - -The LICENSE in this directory applies to these files. - -Thomas Heller, Jan 2003 - -These files have been modified so they fall back to the system -dlfcn calls if available in libSystem. - -Bob Ippolito, Feb 2006 diff --git a/Modules/_ctypes/darwin/dlfcn.h b/Modules/_ctypes/darwin/dlfcn.h deleted file mode 100644 index a2afc3eeb847..000000000000 --- a/Modules/_ctypes/darwin/dlfcn.h +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright (c) 2002 Jorge Acereda <jacereda at users.sourceforge.net> & - Peter O'Gorman <ogorman at users.sourceforge.net> - -Portions may be copyright others, see the AUTHORS file included with this -distribution. - -Maintained by Peter O'Gorman <ogorman at users.sourceforge.net> - -Bug Reports and other queries should go to <ogorman at users.sourceforge.net> - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ -#ifndef _DLFCN_H_ -#define _DLFCN_H_ - -#include <AvailabilityMacros.h> - -#ifdef __cplusplus -extern "C" { -#endif - - -/* - * Structure filled in by dladdr(). - */ - -typedef struct dl_info { - const char *dli_fname; /* Pathname of shared object */ - void *dli_fbase; /* Base address of shared object */ - const char *dli_sname; /* Name of nearest symbol */ - void *dli_saddr; /* Address of nearest symbol */ -} Dl_info; - - -#if MAC_OS_X_VERSION_MIN_REQUIRED <= MAC_OS_X_VERSION_10_2 -#warning CTYPES_DARWIN_DLFCN -#define CTYPES_DARWIN_DLFCN -extern void * (*ctypes_dlopen)(const char *path, int mode); -extern void * (*ctypes_dlsym)(void * handle, const char *symbol); -extern const char * (*ctypes_dlerror)(void); -extern int (*ctypes_dlclose)(void * handle); -extern int (*ctypes_dladdr)(const void *, Dl_info *); -#else -extern void * dlopen(const char *path, int mode); -extern void * dlsym(void * handle, const char *symbol); -extern const char * dlerror(void); -extern int dlclose(void * handle); -extern int dladdr(const void *, Dl_info *); -#endif - -#define RTLD_LAZY 0x1 -#define RTLD_NOW 0x2 -#define RTLD_LOCAL 0x4 -#define RTLD_GLOBAL 0x8 -#define RTLD_NOLOAD 0x10 -#define RTLD_NODELETE 0x80 - -/* These are from the Mac OS X 10.4 headers */ -#define RTLD_NEXT ((void *) -1) /* Search subsequent objects. */ -#define RTLD_DEFAULT ((void *) -2) /* Use default search algorithm. */ - -#ifdef __cplusplus -} -#endif - -#endif /* _DLFCN_H_ */ diff --git a/Modules/_ctypes/darwin/dlfcn_simple.c b/Modules/_ctypes/darwin/dlfcn_simple.c deleted file mode 100644 index 2b293bb8695b..000000000000 --- a/Modules/_ctypes/darwin/dlfcn_simple.c +++ /dev/null @@ -1,272 +0,0 @@ -/* -Copyright (c) 2002 Peter O'Gorman <ogorman at users.sourceforge.net> - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - - -/* Just to prove that it isn't that hard to add Mac calls to your code :) - This works with pretty much everything, including kde3 xemacs and the gimp, - I'd guess that it'd work in at least 95% of cases, use this as your starting - point, rather than the mess that is dlfcn.c, assuming that your code does not - require ref counting or symbol lookups in dependent libraries -*/ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <stdarg.h> -#include <limits.h> -#include <mach-o/dyld.h> -#include <AvailabilityMacros.h> -#include "dlfcn.h" - -#ifdef CTYPES_DARWIN_DLFCN - -#define ERR_STR_LEN 256 - -#ifndef MAC_OS_X_VERSION_10_3 -#define MAC_OS_X_VERSION_10_3 1030 -#endif - -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3 -#define DARWIN_HAS_DLOPEN -extern void * dlopen(const char *path, int mode) __attribute__((weak_import)); -extern void * dlsym(void * handle, const char *symbol) __attribute__((weak_import)); -extern const char * dlerror(void) __attribute__((weak_import)); -extern int dlclose(void * handle) __attribute__((weak_import)); -extern int dladdr(const void *, Dl_info *) __attribute__((weak_import)); -#endif /* MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3 */ - -#ifndef DARWIN_HAS_DLOPEN -#define dlopen darwin_dlopen -#define dlsym darwin_dlsym -#define dlerror darwin_dlerror -#define dlclose darwin_dlclose -#define dladdr darwin_dladdr -#endif - -void * (*ctypes_dlopen)(const char *path, int mode); -void * (*ctypes_dlsym)(void * handle, const char *symbol); -const char * (*ctypes_dlerror)(void); -int (*ctypes_dlclose)(void * handle); -int (*ctypes_dladdr)(const void *, Dl_info *); - -#if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 -/* Mac OS X 10.3+ has dlopen, so strip all this dead code to avoid warnings */ - -static void *dlsymIntern(void *handle, const char *symbol); - -static const char *error(int setget, const char *str, ...); - -/* Set and get the error string for use by dlerror */ -static const char *error(int setget, const char *str, ...) -{ - static char errstr[ERR_STR_LEN]; - static int err_filled = 0; - const char *retval; - va_list arg; - if (setget == 0) - { - va_start(arg, str); - strncpy(errstr, "dlcompat: ", ERR_STR_LEN); - vsnprintf(errstr + 10, ERR_STR_LEN - 10, str, arg); - va_end(arg); - err_filled = 1; - retval = NULL; - } - else - { - if (!err_filled) - retval = NULL; - else - retval = errstr; - err_filled = 0; - } - return retval; -} - -/* darwin_dlopen */ -static void *darwin_dlopen(const char *path, int mode) -{ - void *module = 0; - NSObjectFileImage ofi = 0; - NSObjectFileImageReturnCode ofirc; - - /* If we got no path, the app wants the global namespace, use -1 as the marker - in this case */ - if (!path) - return (void *)-1; - - /* Create the object file image, works for things linked with the -bundle arg to ld */ - ofirc = NSCreateObjectFileImageFromFile(path, &ofi); - switch (ofirc) - { - case NSObjectFileImageSuccess: - /* It was okay, so use NSLinkModule to link in the image */ - module = NSLinkModule(ofi, path, - NSLINKMODULE_OPTION_RETURN_ON_ERROR - | (mode & RTLD_GLOBAL) ? 0 : NSLINKMODULE_OPTION_PRIVATE - | (mode & RTLD_LAZY) ? 0 : NSLINKMODULE_OPTION_BINDNOW); - NSDestroyObjectFileImage(ofi); - break; - case NSObjectFileImageInappropriateFile: - /* It may have been a dynamic library rather than a bundle, try to load it */ - module = (void *)NSAddImage(path, NSADDIMAGE_OPTION_RETURN_ON_ERROR); - break; - default: - /* God knows what we got */ - error(0, "Can not open \"%s\"", path); - return 0; - } - if (!module) - error(0, "Can not open \"%s\"", path); - return module; - -} - -/* dlsymIntern is used by dlsym to find the symbol */ -static void *dlsymIntern(void *handle, const char *symbol) -{ - NSSymbol nssym = 0; - /* If the handle is -1, if is the app global context */ - if (handle == (void *)-1) - { - /* Global context, use NSLookupAndBindSymbol */ - if (NSIsSymbolNameDefined(symbol)) - { - nssym = NSLookupAndBindSymbol(symbol); - } - - } - /* Now see if the handle is a struch mach_header* or not, use NSLookupSymbol in image - for libraries, and NSLookupSymbolInModule for bundles */ - else - { - /* Check for both possible magic numbers depending on x86/ppc byte order */ - if ((((struct mach_header *)handle)->magic == MH_MAGIC) || - (((struct mach_header *)handle)->magic == MH_CIGAM)) - { - if (NSIsSymbolNameDefinedInImage((struct mach_header *)handle, symbol)) - { - nssym = NSLookupSymbolInImage((struct mach_header *)handle, - symbol, - NSLOOKUPSYMBOLINIMAGE_OPTION_BIND - | NSLOOKUPSYMBOLINIMAGE_OPTION_RETURN_ON_ERROR); - } - - } - else - { - nssym = NSLookupSymbolInModule(handle, symbol); - } - } - if (!nssym) - { - error(0, "Symbol \"%s\" Not found", symbol); - return NULL; - } - return NSAddressOfSymbol(nssym); -} - -static const char *darwin_dlerror(void) -{ - return error(1, (char *)NULL); -} - -static int darwin_dlclose(void *handle) -{ - if ((((struct mach_header *)handle)->magic == MH_MAGIC) || - (((struct mach_header *)handle)->magic == MH_CIGAM)) - { - error(0, "Can't remove dynamic libraries on darwin"); - return 0; - } - if (!NSUnLinkModule(handle, 0)) - { - error(0, "unable to unlink module %s", NSNameOfModule(handle)); - return 1; - } - return 0; -} - - -/* dlsym, prepend the underscore and call dlsymIntern */ -static void *darwin_dlsym(void *handle, const char *symbol) -{ - static char undersym[257]; /* Saves calls to malloc(3) */ - int sym_len = strlen(symbol); - void *value = NULL; - char *malloc_sym = NULL; - - if (sym_len < 256) - { - snprintf(undersym, 256, "_%s", symbol); - value = dlsymIntern(handle, undersym); - } - else - { - malloc_sym = malloc(sym_len + 2); - if (malloc_sym) - { - sprintf(malloc_sym, "_%s", symbol); - value = dlsymIntern(handle, malloc_sym); - free(malloc_sym); - } - else - { - error(0, "Unable to allocate memory"); - } - } - return value; -} - -static int darwin_dladdr(const void *handle, Dl_info *info) { - return 0; -} -#endif /* MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 */ - -#if __GNUC__ < 4 -#pragma CALL_ON_LOAD ctypes_dlfcn_init -#else -static void __attribute__ ((constructor)) ctypes_dlfcn_init(void); -static -#endif -void ctypes_dlfcn_init(void) { - if (dlopen != NULL) { - ctypes_dlsym = dlsym; - ctypes_dlopen = dlopen; - ctypes_dlerror = dlerror; - ctypes_dlclose = dlclose; - ctypes_dladdr = dladdr; - } else { -#if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 - ctypes_dlsym = darwin_dlsym; - ctypes_dlopen = darwin_dlopen; - ctypes_dlerror = darwin_dlerror; - ctypes_dlclose = darwin_dlclose; - ctypes_dladdr = darwin_dladdr; -#endif /* MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_3 */ - } -} - -#endif /* CTYPES_DARWIN_DLFCN */ diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 6ac26f1916c9..253da31e9ce1 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -102,7 +102,6 @@ </ItemDefinitionGroup> <ItemGroup> <ClInclude Include="..\Modules\_ctypes\ctypes.h" /> - <ClInclude Include="..\Modules\_ctypes\ctypes_dlfcn.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_ctypes\_ctypes.c" /> diff --git a/PCbuild/_ctypes.vcxproj.filters b/PCbuild/_ctypes.vcxproj.filters index 118c4f0698cc..a38473e3e81d 100644 --- a/PCbuild/_ctypes.vcxproj.filters +++ b/PCbuild/_ctypes.vcxproj.filters @@ -15,9 +15,6 @@ <ClInclude Include="..\Modules\_ctypes\ctypes.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="..\Modules\_ctypes\ctypes_dlfcn.h"> - <Filter>Header Files</Filter> - </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_ctypes\_ctypes.c"> diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ac1721c061f0..ab1d6257f1b1 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -51,7 +51,6 @@ def clean_lines(text): # @begin=conf@ # OSX -#Modules/_ctypes/darwin/*.c Modules/_scproxy.c # SystemConfiguration/SystemConfiguration.h # Windows diff --git a/configure b/configure index 6afd1e9c367c..946218fd8d85 100755 --- a/configure +++ b/configure @@ -12802,8 +12802,7 @@ if test "x$have_libffi" = xyes; then : case $ac_sys_system in #( Darwin) : - as_fn_append LIBFFI_CFLAGS " -I\$(srcdir)/Modules/_ctypes/darwin" - ctypes_malloc_closure=yes + ctypes_malloc_closure=yes ;; #( sunos5) : as_fn_append LIBFFI_LIBS " -mimpure-text" diff --git a/configure.ac b/configure.ac index 48736649a98e..22028972cb3d 100644 --- a/configure.ac +++ b/configure.ac @@ -3741,7 +3741,6 @@ AS_VAR_IF([have_libffi], [yes], [ AS_CASE([$ac_sys_system], [Darwin], [ dnl when do we need USING_APPLE_OS_LIBFFI? - AS_VAR_APPEND([LIBFFI_CFLAGS], [" -I\$(srcdir)/Modules/_ctypes/darwin"]) ctypes_malloc_closure=yes ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] From webhook-mailer at python.org Thu Dec 29 17:41:45 2022 From: webhook-mailer at python.org (gpshead) Date: Thu, 29 Dec 2022 22:41:45 -0000 Subject: [Python-checkins] gh-100228: Warn from os.fork() if other threads exist. (#100229) Message-ID: <mailman.3134.1672353706.3313.python-checkins@python.org> https://github.com/python/cpython/commit/894f2c3c161933bd820ad322b3b678d89bc2377c commit: 894f2c3c161933bd820ad322b3b678d89bc2377c branch: main author: Gregory P. Smith <greg at krypto.org> committer: gpshead <greg at krypto.org> date: 2022-12-29T14:41:39-08:00 summary: gh-100228: Warn from os.fork() if other threads exist. (#100229) Not comprehensive, best effort warning. There are cases when threads exist on some platforms that this code cannot detect. macOS when API permissions allow and Linux with a readable /proc procfs present are the currently supported cases where a warning should show up reliably. Starting with a DeprecationWarning for now, it is less disruptive than something like RuntimeWarning and most likely to only be seen in people's CI tests - a good place to start with this messaging. files: A Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst M Include/internal/pycore_global_objects_fini_generated.h M Include/internal/pycore_global_strings.h M Include/internal/pycore_runtime_init_generated.h M Include/internal/pycore_unicodeobject_generated.h M Lib/test/fork_wait.py M Lib/test/test_os.py M Lib/test/test_thread.py M Lib/test/test_threading.py M Lib/threading.py M Modules/_testcapimodule.c M Modules/posixmodule.c diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6aba2f19ebde..a5365d53150b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -733,6 +733,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__xor__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); @@ -753,6 +754,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index acb9a4fbb92d..3d9e61e50ecc 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -219,6 +219,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__xor__) STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) + STRUCT_FOR_ID(_active) STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) @@ -239,6 +240,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_is_text_encoding) STRUCT_FOR_ID(_length_) + STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) STRUCT_FOR_ID(_needs_com_addref_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6d1b8702c776..3534b94753e0 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -725,6 +725,7 @@ extern "C" { INIT_ID(__xor__), \ INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ + INIT_ID(_active), \ INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ @@ -745,6 +746,7 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_is_text_encoding), \ INIT_ID(_length_), \ + INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ INIT_ID(_needs_com_addref_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7f407c0141b8..02435071bb2c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -344,6 +344,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_abstract_); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_active); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_annotation); PyUnicode_InternInPlace(&string); string = &_Py_ID(_anonymous_); @@ -384,6 +386,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_length_); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_limbo); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_lock_unlock_module); PyUnicode_InternInPlace(&string); string = &_Py_ID(_loop); diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index ebd07e612e58..c26c7aaaeb43 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -13,6 +13,7 @@ import threading from test import support from test.support import threading_helper +import warnings LONGSLEEP = 2 @@ -63,19 +64,17 @@ def test_wait(self): prefork_lives = self.alive.copy() - if sys.platform in ['unixware7']: - cpid = os.fork1() - else: - cpid = os.fork() - - if cpid == 0: - # Child - time.sleep(LONGSLEEP) - n = 0 - for key in self.alive: - if self.alive[key] != prefork_lives[key]: - n += 1 - os._exit(n) - else: - # Parent - self.wait_impl(cpid, exitcode=0) + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (cpid := os.fork()) == 0: + # Child + time.sleep(LONGSLEEP) + n = 0 + for key in self.alive: + if self.alive[key] != prefork_lives[key]: + n += 1 + os._exit(n) + else: + # Parent + self.wait_impl(cpid, exitcode=0) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e6e25b507de0..58e04dd1348f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4577,6 +4577,34 @@ def test_fork(self): assert_python_ok("-c", code) assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + @unittest.skipUnless(sys.platform in ("linux", "darwin"), + "Only Linux and macOS detect this today.") + def test_fork_warns_when_non_python_thread_exists(self): + code = """if 1: + import os, threading, warnings + from _testcapi import _spawn_pthread_waiter, _end_spawned_pthread + _spawn_pthread_waiter() + try: + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + if os.fork() == 0: + assert not ws, f"unexpected warnings in child: {ws}" + os._exit(0) # child + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + # Waiting allows an error in the child to hit stderr. + exitcode = os.wait()[1] + assert exitcode == 0, f"child exited {exitcode}" + assert threading.active_count() == 1, threading.enumerate() + finally: + _end_spawned_pthread() + """ + _, out, err = assert_python_ok("-c", code, PYTHONOPTIMIZE='0') + self.assertEqual(err.decode("utf-8"), "") + self.assertEqual(out.decode("utf-8"), "") + # Only test if the C version is provided, otherwise TestPEP519 already tested # the pure Python implementation. diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 2ae5e9ccb440..8656fbdd83e9 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -5,6 +5,7 @@ from test.support import threading_helper import _thread as thread import time +import warnings import weakref from test import lock_tests @@ -238,11 +239,13 @@ def test_forkinthread(self): def fork_thread(read_fd, write_fd): nonlocal pid - # fork in a thread - pid = os.fork() - if pid: - # parent process - return + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + # fork in a thread (DANGER, undefined per POSIX) + if (pid := os.fork()): + # parent process + return # child process try: diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 13ba5068ae20..31bf46311a80 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -20,6 +20,7 @@ import signal import textwrap import traceback +import warnings from unittest import mock from test import lock_tests @@ -563,7 +564,7 @@ def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. code = """if 1: - import _thread, threading, os, time + import _thread, threading, os, time, warnings def background_thread(evt): # Creates and registers the _DummyThread instance @@ -575,11 +576,16 @@ def background_thread(evt): _thread.start_new_thread(background_thread, (evt,)) evt.wait() assert threading.active_count() == 2, threading.active_count() - if os.fork() == 0: - assert threading.active_count() == 1, threading.active_count() - os._exit(0) - else: - os.wait() + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + os.wait() """ _, out, err = assert_python_ok("-c", code) self.assertEqual(out, b'') @@ -598,13 +604,15 @@ def test_is_alive_after_fork(self): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - pid = os.fork() - if pid == 0: - os._exit(11 if t.is_alive() else 10) - else: - t.join() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (pid := os.fork()) == 0: + os._exit(11 if t.is_alive() else 10) + else: + t.join() - support.wait_process(pid, exitcode=10) + support.wait_process(pid, exitcode=10) def test_main_thread(self): main = threading.main_thread() @@ -645,21 +653,26 @@ def test_main_thread_after_fork(self): @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork_from_nonmain_thread(self): code = """if 1: - import os, threading, sys + import os, threading, sys, warnings from test import support def func(): - pid = os.fork() - if pid == 0: - main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) - # stdout is fully buffered because not a tty, - # we have to flush before exit. - sys.stdout.flush() - else: - support.wait_process(pid, exitcode=0) + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + support.wait_process(pid, exitcode=0) th = threading.Thread(target=func) th.start() @@ -667,7 +680,7 @@ def func(): """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') - self.assertEqual(err, b"") + self.assertEqual(err.decode('utf-8'), "") self.assertEqual(data, "Thread-1 (func)\nTrue\nTrue\n") def test_main_thread_during_shutdown(self): @@ -1173,15 +1186,18 @@ def do_fork_and_wait(): else: os._exit(50) - # start a bunch of threads that will fork() child processes - threads = [] - for i in range(16): - t = threading.Thread(target=do_fork_and_wait) - threads.append(t) - t.start() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() - for t in threads: - t.join() + for t in threads: + t.join() @support.requires_fork() def test_clear_threads_states_after_fork(self): @@ -1194,18 +1210,22 @@ def test_clear_threads_states_after_fork(self): threads.append(t) t.start() - pid = os.fork() - if pid == 0: - # check that threads states have been cleared - if len(sys._current_frames()) == 1: - os._exit(51) - else: - os._exit(52) - else: - support.wait_process(pid, exitcode=51) - - for t in threads: - t.join() + try: + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(51) + else: + os._exit(52) + else: + support.wait_process(pid, exitcode=51) + finally: + for t in threads: + t.join() class SubinterpThreadingTests(BaseTestCase): diff --git a/Lib/threading.py b/Lib/threading.py index 723bd58bf5a6..df273870fa42 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1490,6 +1490,8 @@ def active_count(): enumerate(). """ + # NOTE: if the logic in here ever changes, update Modules/posixmodule.c + # warn_about_fork_with_threads() to match. with _active_limbo_lock: return len(_active) + len(_limbo) diff --git a/Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst b/Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst new file mode 100644 index 000000000000..ca6e4a2a1f0d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-13-17-29-09.gh-issue-100228.bgtzMV.rst @@ -0,0 +1,5 @@ +A :exc:`DeprecationWarning` may be raised when :func:`os.fork()` or +:func:`os.forkpty()` is called from multi-threaded processes. Forking +with threads is unsafe and can cause deadlocks, crashes and subtle +problems. Lack of a warning does not indicate that the fork call was +actually safe, as Python may not be aware of all threads. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c32fdb5f5fbe..c777c3efc492 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -25,6 +25,9 @@ #include "structmember.h" // for offsetof(), T_OBJECT #include <float.h> // FLT_MAX #include <signal.h> +#ifndef MS_WINDOWS +#include <unistd.h> +#endif #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> // W_STOPCODE @@ -871,6 +874,46 @@ test_thread_state(PyObject *self, PyObject *args) Py_RETURN_NONE; } +#ifndef MS_WINDOWS +static PyThread_type_lock wait_done = NULL; + +static void wait_for_lock(void *unused) { + PyThread_acquire_lock(wait_done, 1); + PyThread_release_lock(wait_done); + PyThread_free_lock(wait_done); + wait_done = NULL; +} + +// These can be used to test things that care about the existence of another +// thread that the threading module doesn't know about. + +static PyObject * +spawn_pthread_waiter(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (wait_done) { + PyErr_SetString(PyExc_RuntimeError, "thread already running"); + return NULL; + } + wait_done = PyThread_allocate_lock(); + if (wait_done == NULL) + return PyErr_NoMemory(); + PyThread_acquire_lock(wait_done, 1); + PyThread_start_new_thread(wait_for_lock, NULL); + Py_RETURN_NONE; +} + +static PyObject * +end_spawned_pthread(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (!wait_done) { + PyErr_SetString(PyExc_RuntimeError, "call _spawn_pthread_waiter 1st"); + return NULL; + } + PyThread_release_lock(wait_done); + Py_RETURN_NONE; +} +#endif // not MS_WINDOWS + /* test Py_AddPendingCalls using threads */ static int _pending_callback(void *arg) { @@ -3207,6 +3250,10 @@ static PyMethodDef TestMethods[] = { {"test_get_type_name", test_get_type_name, METH_NOARGS}, {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, +#ifndef MS_WINDOWS + {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, + {"_end_spawned_pthread", end_spawned_pthread, METH_NOARGS}, +#endif {"_pending_threadfunc", pending_threadfunc, METH_VARARGS}, #ifdef HAVE_GETTIMEOFDAY {"profile_int", profile_int, METH_NOARGS}, diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 607d40b59d96..1d9a33ab7bc9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -72,6 +72,8 @@ */ #if defined(__APPLE__) +#include <mach/mach.h> + #if defined(__has_builtin) #if __has_builtin(__builtin_available) #define HAVE_BUILTIN_AVAILABLE 1 @@ -6745,6 +6747,104 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, } #endif /* HAVE_FORK */ +// Common code to raise a warning if we detect there is more than one thread +// running in the process. Best effort, silent if unable to count threads. +// Constraint: Quick. Never overcounts. Never leaves an error set. +// +// This code might do an import, thus acquiring the import lock, which +// PyOS_BeforeFork() also does. As this should only be called from +// the parent process, it is in the same thread so that works. +static void warn_about_fork_with_threads(const char* name) { + // TODO: Consider making an `os` module API to return the current number + // of threads in the process. That'd presumably use this platform code but + // raise an error rather than using the inaccurate fallback. + Py_ssize_t num_python_threads = 0; +#if defined(__APPLE__) && defined(HAVE_GETPID) + mach_port_t macos_self = mach_task_self(); + mach_port_t macos_task; + if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { + thread_array_t macos_threads; + mach_msg_type_number_t macos_n_threads; + if (task_threads(macos_task, &macos_threads, + &macos_n_threads) == KERN_SUCCESS) { + num_python_threads = macos_n_threads; + } + } +#elif defined(__linux__) + // Linux /proc/self/stat 20th field is the number of threads. + FILE* proc_stat = fopen("/proc/self/stat", "r"); + if (proc_stat) { + size_t n; + // Size chosen arbitrarily. ~60% more bytes than a 20th column index + // observed on the author's workstation. + char stat_line[160]; + n = fread(&stat_line, 1, 159, proc_stat); + stat_line[n] = '\0'; + fclose(proc_stat); + + char *saveptr = NULL; + char *field = strtok_r(stat_line, " ", &saveptr); + unsigned int idx; + for (idx = 19; idx && field; --idx) { + field = strtok_r(NULL, " ", &saveptr); + } + if (idx == 0 && field) { // found the 20th field + num_python_threads = atoi(field); // 0 on error + } + } +#endif + if (num_python_threads <= 0) { + // Fall back to just the number our threading module knows about. + // An incomplete view of the world, but better than nothing. + PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); + if (!threading) { + PyErr_Clear(); + return; + } + PyObject *threading_active = + PyObject_GetAttr(threading, &_Py_ID(_active)); + if (!threading_active) { + PyErr_Clear(); + Py_DECREF(threading); + return; + } + PyObject *threading_limbo = + PyObject_GetAttr(threading, &_Py_ID(_limbo)); + if (!threading_limbo) { + PyErr_Clear(); + Py_DECREF(threading); + Py_DECREF(threading_active); + return; + } + Py_DECREF(threading); + // Duplicating what threading.active_count() does but without holding + // threading._active_limbo_lock so our count could be inaccurate if + // these dicts are mid-update from another thread. Not a big deal. + // Worst case if someone replaced threading._active or threading._limbo + // with non-dicts, we get -1 from *Length() below and undercount. + // Nobody should, but we're best effort so we clear errors and move on. + num_python_threads = (PyMapping_Length(threading_active) + + PyMapping_Length(threading_limbo)); + PyErr_Clear(); + Py_DECREF(threading_active); + Py_DECREF(threading_limbo); + } + if (num_python_threads > 1) { + PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, +#ifdef HAVE_GETPID + "This process (pid=%d) is multi-threaded, " +#else + "This process is multi-threaded, " +#endif + "use of %s() may lead to deadlocks in the child.", +#ifdef HAVE_GETPID + getpid(), +#endif + name); + PyErr_Clear(); + } +} #ifdef HAVE_FORK1 /*[clinic input] @@ -6771,6 +6871,7 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + warn_about_fork_with_threads("fork1"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } @@ -6810,6 +6911,7 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + warn_about_fork_with_threads("fork"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } @@ -7479,6 +7581,7 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + warn_about_fork_with_threads("forkpty"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); } From webhook-mailer at python.org Fri Dec 30 06:55:31 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Fri, 30 Dec 2022 11:55:31 -0000 Subject: [Python-checkins] gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (#99434) Message-ID: <mailman.3135.1672401333.3313.python-checkins@python.org> https://github.com/python/cpython/commit/79c10b7da84f52999dc483fc62c8e758ad3eff23 commit: 79c10b7da84f52999dc483fc62c8e758ad3eff23 branch: main author: Nikita Sobolev <mail at sobolevn.me> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-30T17:25:01+05:30 summary: gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (#99434) files: A Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst M Lib/doctest.py M Lib/test/doctest_lineno.py diff --git a/Lib/doctest.py b/Lib/doctest.py index b2ef2ce63672..dafad505ef0e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -956,7 +956,8 @@ def _from_module(self, module, object): return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): diff --git a/Lib/test/doctest_lineno.py b/Lib/test/doctest_lineno.py index be198513a150..729a68aceaa9 100644 --- a/Lib/test/doctest_lineno.py +++ b/Lib/test/doctest_lineno.py @@ -48,3 +48,6 @@ def method_with_doctest(self): >>> MethodWrapper.method_with_doctest.__name__ 'method_with_doctest' """ + +# https://github.com/python/cpython/issues/99433 +str_wrapper = object().__str__ diff --git a/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst new file mode 100644 index 000000000000..8e13a9a66a7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` failure on :class:`types.MethodWrapperType` in modules. From webhook-mailer at python.org Fri Dec 30 07:19:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 12:19:00 -0000 Subject: [Python-checkins] gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (GH-99434) Message-ID: <mailman.3136.1672402742.3313.python-checkins@python.org> https://github.com/python/cpython/commit/c88a83e7d81fbf394bbdebe0f453bb64bdf33ba6 commit: c88a83e7d81fbf394bbdebe0f453bb64bdf33ba6 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T04:18:54-08:00 summary: gh-99433: Fix `doctest` failure on `types.MethodWrapperType` (GH-99434) (cherry picked from commit 79c10b7da84f52999dc483fc62c8e758ad3eff23) Co-authored-by: Nikita Sobolev <mail at sobolevn.me> files: A Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst M Lib/doctest.py M Lib/test/doctest_lineno.py diff --git a/Lib/doctest.py b/Lib/doctest.py index b2ef2ce63672..dafad505ef0e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -956,7 +956,8 @@ def _from_module(self, module, object): return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): diff --git a/Lib/test/doctest_lineno.py b/Lib/test/doctest_lineno.py index be198513a150..729a68aceaa9 100644 --- a/Lib/test/doctest_lineno.py +++ b/Lib/test/doctest_lineno.py @@ -48,3 +48,6 @@ def method_with_doctest(self): >>> MethodWrapper.method_with_doctest.__name__ 'method_with_doctest' """ + +# https://github.com/python/cpython/issues/99433 +str_wrapper = object().__str__ diff --git a/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst new file mode 100644 index 000000000000..8e13a9a66a7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-13-15-32-19.gh-issue-99433.Ys6y0A.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` failure on :class:`types.MethodWrapperType` in modules. From webhook-mailer at python.org Fri Dec 30 11:35:31 2022 From: webhook-mailer at python.org (JelleZijlstra) Date: Fri, 30 Dec 2022 16:35:31 -0000 Subject: [Python-checkins] gh-100616: Document 'attr' parameter for window.vline() in curses module (#24961) Message-ID: <mailman.3137.1672418133.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f4fcfdf8c593611f98b9358cc0c5604c15306465 commit: f4fcfdf8c593611f98b9358cc0c5604c15306465 branch: main author: mathieui <mathieui at users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra at gmail.com> date: 2022-12-30T08:35:04-08:00 summary: gh-100616: Document 'attr' parameter for window.vline() in curses module (#24961) Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst M Doc/library/curses.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index f9f94b22f69e..f50b51c3780e 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1297,11 +1297,11 @@ the following methods and attributes: :meth:`refresh`. -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch*. + character *ch* with attributes *attr*. Constants diff --git a/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst new file mode 100644 index 000000000000..97a7022c94b4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst @@ -0,0 +1,2 @@ +Document existing ``attr`` parameter to :func:`curses.window.vline` function +in :mod:`curses`. From webhook-mailer at python.org Fri Dec 30 12:06:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 17:06:00 -0000 Subject: [Python-checkins] gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) Message-ID: <mailman.3138.1672419961.3313.python-checkins@python.org> https://github.com/python/cpython/commit/297465a4db0aff45e57cf5bfef4f3ab4d9538de0 commit: 297465a4db0aff45e57cf5bfef4f3ab4d9538de0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T09:05:49-08:00 summary: gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) (cherry picked from commit f4fcfdf8c593611f98b9358cc0c5604c15306465) Co-authored-by: mathieui <mathieui at users.noreply.github.com> Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst M Doc/library/curses.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 9b2c3fbd8e4e..56311033d631 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1300,11 +1300,11 @@ the following methods and attributes: :meth:`refresh`. -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch*. + character *ch* with attributes *attr*. Constants diff --git a/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst new file mode 100644 index 000000000000..97a7022c94b4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst @@ -0,0 +1,2 @@ +Document existing ``attr`` parameter to :func:`curses.window.vline` function +in :mod:`curses`. From webhook-mailer at python.org Fri Dec 30 12:06:00 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 17:06:00 -0000 Subject: [Python-checkins] gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) Message-ID: <mailman.3139.1672419961.3313.python-checkins@python.org> https://github.com/python/cpython/commit/3ab7b61b416b3ce4a01b0566a8d2bec3e3d0e3d4 commit: 3ab7b61b416b3ce4a01b0566a8d2bec3e3d0e3d4 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T09:05:40-08:00 summary: gh-100616: Document 'attr' parameter for window.vline() in curses module (GH-24961) (cherry picked from commit f4fcfdf8c593611f98b9358cc0c5604c15306465) Co-authored-by: mathieui <mathieui at users.noreply.github.com> Co-authored-by: Stanley <46876382+slateny at users.noreply.github.com> files: A Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst M Doc/library/curses.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 55ab06018d70..5d74b73e2e16 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1314,11 +1314,11 @@ the following methods and attributes: :meth:`refresh`. -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch*. + character *ch* with attributes *attr*. Constants diff --git a/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst new file mode 100644 index 000000000000..97a7022c94b4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-12-30-00-42-23.gh-issue-100616.eu80ij.rst @@ -0,0 +1,2 @@ +Document existing ``attr`` parameter to :func:`curses.window.vline` function +in :mod:`curses`. From webhook-mailer at python.org Fri Dec 30 16:21:38 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 21:21:38 -0000 Subject: [Python-checkins] gh-95778: add doc missing in some places (GH-100627) Message-ID: <mailman.3140.1672435299.3313.python-checkins@python.org> https://github.com/python/cpython/commit/46521826cb1883e29e4640f94089dd92c57efc5b commit: 46521826cb1883e29e4640f94089dd92c57efc5b branch: main author: ?ric <earaujo at caravan.coop> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T13:21:15-08:00 summary: gh-95778: add doc missing in some places (GH-100627) files: M Misc/python.man M Python/initconfig.c diff --git a/Misc/python.man b/Misc/python.man index 1705eeb0c9c1..bf7cf767d164 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -358,6 +358,10 @@ Set implementation-specific option. The following options are available: -X frozen_modules=[on|off]: whether or not frozen modules should be used. The default is "on" (or "off" if you are running a local build). + -X int_max_str_digits=number: limit the size of int<->str conversions. + This helps avoid denial of service attacks when parsing untrusted data. + The default is sys.int_info.default_max_str_digits. 0 disables. + .TP .B \-x Skip the first line of the source. This is intended for a DOS @@ -531,6 +535,11 @@ values. The integer must be a decimal number in the range [0,4294967295]. Specifying the value 0 will disable hash randomization. +.IP PYTHONINTMAXSTRDIGITS +Limit the maximum digit characters in an int value +when converting from a string and when converting an int back to a str. +A value of 0 disables the limit. Conversions to or from bases 2, 4, 8, +16, and 32 are never limited. .IP PYTHONMALLOC Set the Python memory allocators and/or install debug hooks. The available memory allocators are diff --git a/Python/initconfig.c b/Python/initconfig.c index d05099cd9977..d7b2dc4a2974 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -180,6 +180,8 @@ static const char usage_envvars[] = "PYTHONDEBUG : enable parser debug mode (-d)\n" "PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" "PYTHONINSPECT : inspect interactively after running script (-i)\n" +"PYTHONINTMAXSTRDIGITS : limit max digit characters in an int value\n" +" (-X int_max_str_digits=number)\n" "PYTHONNOUSERSITE : disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" "PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" From webhook-mailer at python.org Fri Dec 30 16:51:11 2022 From: webhook-mailer at python.org (miss-islington) Date: Fri, 30 Dec 2022 21:51:11 -0000 Subject: [Python-checkins] gh-95778: add doc missing in some places (GH-100627) Message-ID: <mailman.3141.1672437072.3313.python-checkins@python.org> https://github.com/python/cpython/commit/b1e314ab9f8c3a2b53c7179674811f9c79328ce7 commit: b1e314ab9f8c3a2b53c7179674811f9c79328ce7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T13:51:06-08:00 summary: gh-95778: add doc missing in some places (GH-100627) (cherry picked from commit 46521826cb1883e29e4640f94089dd92c57efc5b) Co-authored-by: ?ric <earaujo at caravan.coop> files: M Misc/python.man M Python/initconfig.c diff --git a/Misc/python.man b/Misc/python.man index c979f5dff3ca..07a08315d8f3 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -358,6 +358,10 @@ Set implementation-specific option. The following options are available: -X frozen_modules=[on|off]: whether or not frozen modules should be used. The default is "on" (or "off" if you are running a local build). + -X int_max_str_digits=number: limit the size of int<->str conversions. + This helps avoid denial of service attacks when parsing untrusted data. + The default is sys.int_info.default_max_str_digits. 0 disables. + .TP .B \-x Skip the first line of the source. This is intended for a DOS @@ -531,6 +535,11 @@ values. The integer must be a decimal number in the range [0,4294967295]. Specifying the value 0 will disable hash randomization. +.IP PYTHONINTMAXSTRDIGITS +Limit the maximum digit characters in an int value +when converting from a string and when converting an int back to a str. +A value of 0 disables the limit. Conversions to or from bases 2, 4, 8, +16, and 32 are never limited. .IP PYTHONMALLOC Set the Python memory allocators and/or install debug hooks. The available memory allocators are diff --git a/Python/initconfig.c b/Python/initconfig.c index 0ccca265314d..d81cbaff7ec9 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -168,6 +168,8 @@ static const char usage_envvars[] = "PYTHONDEBUG : enable parser debug mode (-d)\n" "PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" "PYTHONINSPECT : inspect interactively after running script (-i)\n" +"PYTHONINTMAXSTRDIGITS : limit max digit characters in an int value\n" +" (-X int_max_str_digits=number)\n" "PYTHONNOUSERSITE : disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" "PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" From webhook-mailer at python.org Fri Dec 30 21:36:43 2022 From: webhook-mailer at python.org (rhettinger) Date: Sat, 31 Dec 2022 02:36:43 -0000 Subject: [Python-checkins] Improve comments in itertools uniquification recipes (GH-100631) Message-ID: <mailman.3142.1672454204.3313.python-checkins@python.org> https://github.com/python/cpython/commit/4ebaae8aedd91d2cffa5e89123f3539ddcbdfd5b commit: 4ebaae8aedd91d2cffa5e89123f3539ddcbdfd5b branch: main author: Raymond Hettinger <rhettinger at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-30T20:36:38-06:00 summary: Improve comments in itertools uniquification recipes (GH-100631) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5d6efabcef60..4ad679dfccdb 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1019,15 +1019,14 @@ which incur interpreter overhead. def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D + # unique_everseen('ABBcCAD', str.lower) --> A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): seen.add(element) yield element - # Note: The steps shown above are intended to demonstrate - # filterfalse(). For order preserving deduplication, - # a better solution is: + # For order preserving deduplication, + # a faster but non-lazy solution is: # yield from dict.fromkeys(iterable) else: for element in iterable: @@ -1035,11 +1034,15 @@ which incur interpreter overhead. if k not in seen: seen.add(k) yield element + # For use cases that allow the last matching element to be returned, + # a faster but non-lazy solution is: + # t1, t2 = tee(iterable) + # yield from dict(zip(map(key, t1), t2)).values() def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBCcAD', str.lower) --> A B C A D + # unique_justseen('ABBcCAD', str.lower) --> A B c A D return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def iter_except(func, exception, first=None): @@ -1371,15 +1374,17 @@ which incur interpreter overhead. >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'D'] + >>> list(unique_everseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'A', 'D'] + >>> list(unique_justseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) From webhook-mailer at python.org Fri Dec 30 23:23:44 2022 From: webhook-mailer at python.org (rhettinger) Date: Sat, 31 Dec 2022 04:23:44 -0000 Subject: [Python-checkins] [3.11] Improve comments in itertools uniquification recipes (GH-100631) (GH-100632) Message-ID: <mailman.3143.1672460626.3313.python-checkins@python.org> https://github.com/python/cpython/commit/18006309eac07e5f42232fd4ba016640aeac5b1a commit: 18006309eac07e5f42232fd4ba016640aeac5b1a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: rhettinger <rhettinger at users.noreply.github.com> date: 2022-12-30T22:23:39-06:00 summary: [3.11] Improve comments in itertools uniquification recipes (GH-100631) (GH-100632) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 93bc1f792f54..fbbc96c60aba 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -989,15 +989,14 @@ which incur interpreter overhead. def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D + # unique_everseen('ABBcCAD', str.lower) --> A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): seen.add(element) yield element - # Note: The steps shown above are intended to demonstrate - # filterfalse(). For order preserving deduplication, - # a better solution is: + # For order preserving deduplication, + # a faster but non-lazy solution is: # yield from dict.fromkeys(iterable) else: for element in iterable: @@ -1005,11 +1004,15 @@ which incur interpreter overhead. if k not in seen: seen.add(k) yield element + # For use cases that allow the last matching element to be returned, + # a faster but non-lazy solution is: + # t1, t2 = tee(iterable) + # yield from dict(zip(map(key, t1), t2)).values() def unique_justseen(iterable, key=None): "List unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBCcAD', str.lower) --> A B C A D + # unique_justseen('ABBcCAD', str.lower) --> A B c A D return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def iter_except(func, exception, first=None): @@ -1365,15 +1368,17 @@ which incur interpreter overhead. >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'D'] + >>> list(unique_everseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) ['A', 'B', 'C', 'A', 'D'] + >>> list(unique_justseen('ABBcCAD', str.lower)) + ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) From webhook-mailer at python.org Sat Dec 31 01:24:13 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 31 Dec 2022 06:24:13 -0000 Subject: [Python-checkins] GH-85979: Clarify specification of `object.__await__` (#22320) Message-ID: <mailman.3144.1672467854.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f59c7f8edd5ba5f6c1954383542a2292bcf51d91 commit: f59c7f8edd5ba5f6c1954383542a2292bcf51d91 branch: main author: Paolo Lammens <lammenspaolo at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-31T11:54:04+05:30 summary: GH-85979: Clarify specification of `object.__await__` (#22320) Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index fd682fcff020..1d2ddf3507ae 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2950,6 +2950,14 @@ are awaitable. :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + .. note:: + + The language doesn't place any restriction on the type or value of the + objects yielded by the iterator returned by ``__await__``, as this is + specific to the implementation of the asynchronous execution framework + (e.g. :mod:`asyncio`) that will be managing the :term:`awaitable` object. + + .. versionadded:: 3.5 .. seealso:: :pep:`492` for additional information about awaitable objects. From webhook-mailer at python.org Sat Dec 31 01:31:48 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 06:31:48 -0000 Subject: [Python-checkins] GH-85979: Clarify specification of `object.__await__` (GH-22320) Message-ID: <mailman.3145.1672468310.3313.python-checkins@python.org> https://github.com/python/cpython/commit/504aa92b70d3902f373dc5a6ca0b3ddc1ad233d5 commit: 504aa92b70d3902f373dc5a6ca0b3ddc1ad233d5 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T22:31:41-08:00 summary: GH-85979: Clarify specification of `object.__await__` (GH-22320) (cherry picked from commit f59c7f8edd5ba5f6c1954383542a2292bcf51d91) Co-authored-by: Paolo Lammens <lammenspaolo at gmail.com> Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index eafb6ff15998..6aed328a9fdb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2879,6 +2879,14 @@ are awaitable. :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + .. note:: + + The language doesn't place any restriction on the type or value of the + objects yielded by the iterator returned by ``__await__``, as this is + specific to the implementation of the asynchronous execution framework + (e.g. :mod:`asyncio`) that will be managing the :term:`awaitable` object. + + .. versionadded:: 3.5 .. seealso:: :pep:`492` for additional information about awaitable objects. From webhook-mailer at python.org Sat Dec 31 01:34:18 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 06:34:18 -0000 Subject: [Python-checkins] GH-85979: Clarify specification of `object.__await__` (GH-22320) Message-ID: <mailman.3146.1672468459.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f9ddbc0de5ca0689f87f18819c0bf4b7f1271726 commit: f9ddbc0de5ca0689f87f18819c0bf4b7f1271726 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-30T22:34:12-08:00 summary: GH-85979: Clarify specification of `object.__await__` (GH-22320) (cherry picked from commit f59c7f8edd5ba5f6c1954383542a2292bcf51d91) Co-authored-by: Paolo Lammens <lammenspaolo at gmail.com> Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 92cd2f8f8074..afd4a5477ffb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2950,6 +2950,14 @@ are awaitable. :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + .. note:: + + The language doesn't place any restriction on the type or value of the + objects yielded by the iterator returned by ``__await__``, as this is + specific to the implementation of the asynchronous execution framework + (e.g. :mod:`asyncio`) that will be managing the :term:`awaitable` object. + + .. versionadded:: 3.5 .. seealso:: :pep:`492` for additional information about awaitable objects. From webhook-mailer at python.org Sat Dec 31 04:15:35 2022 From: webhook-mailer at python.org (methane) Date: Sat, 31 Dec 2022 09:15:35 -0000 Subject: [Python-checkins] gh-94808: Improve coverage of dictresize (GH-100619) Message-ID: <mailman.3147.1672478136.3313.python-checkins@python.org> https://github.com/python/cpython/commit/636e9dd23f88c701eecf91156835fe0fc8b1feb6 commit: 636e9dd23f88c701eecf91156835fe0fc8b1feb6 branch: main author: tqxia <44689929+tqxia at users.noreply.github.com> committer: methane <songofacandy at gmail.com> date: 2022-12-31T18:15:30+09:00 summary: gh-94808: Improve coverage of dictresize (GH-100619) files: M Lib/test/test_dict.py diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 5b8baaf9e6e2..79638340059f 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1094,6 +1094,21 @@ def __init__(self, order): d.update(o.__dict__) self.assertEqual(list(d), ["c", "b", "a"]) + @support.cpython_only + def test_splittable_to_generic_combinedtable(self): + """split table must be correctly resized and converted to generic combined table""" + class C: + pass + + a = C() + a.x = 1 + d = a.__dict__ + before_resize = sys.getsizeof(d) + d[2] = 2 # split table is resized to a generic combined table + + self.assertGreater(sys.getsizeof(d), before_resize) + self.assertEqual(list(d), ['x', 2]) + def test_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): data = {1:"a", 2:"b", 3:"c"} From webhook-mailer at python.org Sat Dec 31 05:23:38 2022 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 31 Dec 2022 10:23:38 -0000 Subject: [Python-checkins] gh-100633 Tutorial: Fix dataclasses import (#100638) Message-ID: <mailman.3148.1672482219.3313.python-checkins@python.org> https://github.com/python/cpython/commit/98308dbeb110198ebe28bdb7720d3671b3e7f57b commit: 98308dbeb110198ebe28bdb7720d3671b3e7f57b branch: main author: Owain Davies <116417456+OTheDev at users.noreply.github.com> committer: AlexWaygood <Alex.Waygood at Gmail.com> date: 2022-12-31T10:23:18Z summary: gh-100633 Tutorial: Fix dataclasses import (#100638) import dataclass not dataclasses from dataclasses files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index a206ba371976..116801177a3a 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -741,7 +741,7 @@ Sometimes it is useful to have a data type similar to the Pascal "record" or C "struct", bundling together a few named data items. The idiomatic approach is to use :mod:`dataclasses` for this purpose:: - from dataclasses import dataclasses + from dataclasses import dataclass @dataclass class Employee: From webhook-mailer at python.org Sat Dec 31 07:28:48 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 12:28:48 -0000 Subject: [Python-checkins] gh-100633 Tutorial: Fix dataclasses import (GH-100638) Message-ID: <mailman.3149.1672489728.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f7ad4ffa563fbe14d6b0020c45720bebde73de62 commit: f7ad4ffa563fbe14d6b0020c45720bebde73de62 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T04:28:36-08:00 summary: gh-100633 Tutorial: Fix dataclasses import (GH-100638) import dataclass not dataclasses from dataclasses (cherry picked from commit 98308dbeb110198ebe28bdb7720d3671b3e7f57b) Co-authored-by: Owain Davies <116417456+OTheDev at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 5abb767cb0a8..30450c70f3e8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -740,7 +740,7 @@ Sometimes it is useful to have a data type similar to the Pascal "record" or C "struct", bundling together a few named data items. The idiomatic approach is to use :mod:`dataclasses` for this purpose:: - from dataclasses import dataclasses + from dataclasses import dataclass @dataclass class Employee: From webhook-mailer at python.org Sat Dec 31 07:29:05 2022 From: webhook-mailer at python.org (miss-islington) Date: Sat, 31 Dec 2022 12:29:05 -0000 Subject: [Python-checkins] gh-100633 Tutorial: Fix dataclasses import (GH-100638) Message-ID: <mailman.3150.1672489746.3313.python-checkins@python.org> https://github.com/python/cpython/commit/f80ba44f7d01b153b12eb7573568f9b1f33654ee commit: f80ba44f7d01b153b12eb7573568f9b1f33654ee branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T04:28:59-08:00 summary: gh-100633 Tutorial: Fix dataclasses import (GH-100638) import dataclass not dataclasses from dataclasses (cherry picked from commit 98308dbeb110198ebe28bdb7720d3671b3e7f57b) Co-authored-by: Owain Davies <116417456+OTheDev at users.noreply.github.com> files: M Doc/tutorial/classes.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 5abb767cb0a8..30450c70f3e8 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -740,7 +740,7 @@ Sometimes it is useful to have a data type similar to the Pascal "record" or C "struct", bundling together a few named data items. The idiomatic approach is to use :mod:`dataclasses` for this purpose:: - from dataclasses import dataclasses + from dataclasses import dataclass @dataclass class Employee: From webhook-mailer at python.org Sat Dec 31 11:58:04 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 31 Dec 2022 16:58:04 -0000 Subject: [Python-checkins] GH-87002: fix caching documentation in `struct` module (#24164) Message-ID: <mailman.3151.1672505885.3313.python-checkins@python.org> https://github.com/python/cpython/commit/ac7a0a9f9e9a749ebcdd9d1ac21f73449d279958 commit: ac7a0a9f9e9a749ebcdd9d1ac21f73449d279958 branch: main author: Sandeep Subramanian <sandeepsubramanian94 at gmail.com> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-31T22:27:58+05:30 summary: GH-87002: fix caching documentation in `struct` module (#24164) files: M Doc/library/struct.rst diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 50d70731f775..830ba69e294d 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -548,9 +548,9 @@ The :mod:`struct` module also defines the following type: .. note:: The compiled versions of the most recent format strings passed to - :class:`Struct` and the module-level functions are cached, so programs - that use only a few format strings needn't worry about reusing a single - :class:`Struct` instance. + the module-level functions are cached, so programs that use only a few + format strings needn't worry about reusing a single :class:`Struct` + instance. Compiled Struct objects support the following methods and attributes: From webhook-mailer at python.org Sat Dec 31 12:10:37 2022 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 31 Dec 2022 17:10:37 -0000 Subject: [Python-checkins] Fix `pydtrace.d` path comment in `Include/pydtrace.h` (#28539) Message-ID: <mailman.3152.1672506638.3313.python-checkins@python.org> https://github.com/python/cpython/commit/baf234971f20e5e6c7438f46889e9c8f172cc25e commit: baf234971f20e5e6c7438f46889e9c8f172cc25e branch: main author: Mark Hansen <mark at markhansen.co.nz> committer: kumaraditya303 <59607654+kumaraditya303 at users.noreply.github.com> date: 2022-12-31T22:40:25+05:30 summary: Fix `pydtrace.d` path comment in `Include/pydtrace.h` (#28539) files: M Include/pydtrace.h diff --git a/Include/pydtrace.h b/Include/pydtrace.h index 75f8e7f70979..e197d3669453 100644 --- a/Include/pydtrace.h +++ b/Include/pydtrace.h @@ -12,7 +12,7 @@ extern "C" { /* pydtrace_probes.h, on systems with DTrace, is auto-generated to include `PyDTrace_{PROBE}` and `PyDTrace_{PROBE}_ENABLED()` macros for every probe - defined in pydtrace_provider.d. + defined in pydtrace.d. Calling these functions must be guarded by a `PyDTrace_{PROBE}_ENABLED()` check to minimize performance impact when probing is off. For example: From webhook-mailer at python.org Sat Dec 31 19:01:49 2022 From: webhook-mailer at python.org (terryjreedy) Date: Sun, 01 Jan 2023 00:01:49 -0000 Subject: [Python-checkins] IDLE - fix module browser test (#100647) Message-ID: <mailman.3153.1672531311.3313.python-checkins@python.org> https://github.com/python/cpython/commit/1f6c87ca7b9351b2e5c5363504796fce0554c9b8 commit: 1f6c87ca7b9351b2e5c5363504796fce0554c9b8 branch: main author: Terry Jan Reedy <tjreedy at udel.edu> committer: terryjreedy <tjreedy at udel.edu> date: 2022-12-31T19:01:44-05:00 summary: IDLE - fix module browser test (#100647) files: M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 343d50a6e37b..6cfea3888cd6 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -170,8 +170,7 @@ def test_ondoubleclick(self, fopen): with mock.patch('os.path.exists', return_value=True): mbt.OnDoubleClick() - fopen.assert_called() - fopen.called_with(fname) + fopen.assert_called_once_with(fname) class ChildBrowserTreeItemTest(unittest.TestCase): From webhook-mailer at python.org Sat Dec 31 19:20:59 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 01 Jan 2023 00:20:59 -0000 Subject: [Python-checkins] IDLE - fix module browser test (GH-100647) Message-ID: <mailman.3154.1672532459.3313.python-checkins@python.org> https://github.com/python/cpython/commit/0d57f10e40a4c4053169fe743882118e2bf3dee8 commit: 0d57f10e40a4c4053169fe743882118e2bf3dee8 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T16:20:53-08:00 summary: IDLE - fix module browser test (GH-100647) (cherry picked from commit 1f6c87ca7b9351b2e5c5363504796fce0554c9b8) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 343d50a6e37b..6cfea3888cd6 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -170,8 +170,7 @@ def test_ondoubleclick(self, fopen): with mock.patch('os.path.exists', return_value=True): mbt.OnDoubleClick() - fopen.assert_called() - fopen.called_with(fname) + fopen.assert_called_once_with(fname) class ChildBrowserTreeItemTest(unittest.TestCase): From webhook-mailer at python.org Sat Dec 31 19:31:58 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 01 Jan 2023 00:31:58 -0000 Subject: [Python-checkins] IDLE - fix module browser test (GH-100647) Message-ID: <mailman.3155.1672533120.3313.python-checkins@python.org> https://github.com/python/cpython/commit/83f85539a1559de285d2767c791235876ccda2d3 commit: 83f85539a1559de285d2767c791235876ccda2d3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T16:31:53-08:00 summary: IDLE - fix module browser test (GH-100647) (cherry picked from commit 1f6c87ca7b9351b2e5c5363504796fce0554c9b8) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 343d50a6e37b..6cfea3888cd6 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -170,8 +170,7 @@ def test_ondoubleclick(self, fopen): with mock.patch('os.path.exists', return_value=True): mbt.OnDoubleClick() - fopen.assert_called() - fopen.called_with(fname) + fopen.assert_called_once_with(fname) class ChildBrowserTreeItemTest(unittest.TestCase): From webhook-mailer at python.org Sat Dec 31 19:59:52 2022 From: webhook-mailer at python.org (hauntsaninja) Date: Sun, 01 Jan 2023 00:59:52 -0000 Subject: [Python-checkins] gh-100546: Remove incorrect positional-only marker from eval (#100547) Message-ID: <mailman.3156.1672534793.3313.python-checkins@python.org> https://github.com/python/cpython/commit/71159a8e078bda0c9a39c6cd0980b7ba238dc582 commit: 71159a8e078bda0c9a39c6cd0980b7ba238dc582 branch: main author: Shantanu <12621235+hauntsaninja at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2022-12-31T17:59:31-07:00 summary: gh-100546: Remove incorrect positional-only marker from eval (#100547) All the arguments are positional-only. The current status after #99476 seems to be to not use positional-only markers in documentation, hence I've simply removed it. files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index f00a072e6f41..77ebdd0367c3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -513,7 +513,7 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, /, globals=None, locals=None) +.. function:: eval(expression, globals=None, locals=None) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping From webhook-mailer at python.org Sat Dec 31 20:06:37 2022 From: webhook-mailer at python.org (miss-islington) Date: Sun, 01 Jan 2023 01:06:37 -0000 Subject: [Python-checkins] gh-100546: Remove incorrect positional-only marker from eval (GH-100547) Message-ID: <mailman.3157.1672535198.3313.python-checkins@python.org> https://github.com/python/cpython/commit/7ed71278768f2646af0a74caef6648c385d9f334 commit: 7ed71278768f2646af0a74caef6648c385d9f334 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: miss-islington <31488909+miss-islington at users.noreply.github.com> date: 2022-12-31T17:06:31-08:00 summary: gh-100546: Remove incorrect positional-only marker from eval (GH-100547) All the arguments are positional-only. The current status after GH-99476 seems to be to not use positional-only markers in documentation, hence I've simply removed it. (cherry picked from commit 71159a8e078bda0c9a39c6cd0980b7ba238dc582) Co-authored-by: Shantanu <12621235+hauntsaninja at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c71ebd235cf3..e9e3177a99c7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -512,7 +512,7 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, /, globals=None, locals=None) +.. function:: eval(expression, globals=None, locals=None) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping