[Python-checkins] [3.8] bpo-37170: Fix the cast on error in PyLong_AsUnsignedLongLongMask() (GH-13860) (GH-13891)

[Victor Stinner]

https://github.com/python/cpython/commit/dd492d9c352eb0fa2bc48ea9acc47e47a7fab8a0
commit: dd492d9c352eb0fa2bc48ea9acc47e47a7fab8a0
branch: 3.8
author: Zackery Spytz <zspytz at gmail.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-06-07T16:22:58+02:00
summary:

[3.8] bpo-37170: Fix the cast on error in PyLong_AsUnsignedLongLongMask() (GH-13860) (GH-13891)

(cherry picked from commit dc2476500d91082f0c907772c83a044bf49af279)

Co-authored-by: Zackery Spytz <zspytz at gmail.com>

files:
A Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst
M Doc/c-api/long.rst
M Modules/_testcapimodule.c
M Objects/longobject.c

diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index 6be29f9cd32e..fdaefafe21ba 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -288,7 +288,8 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    If the value of *obj* is out of range for an :c:type:`unsigned long`,
    return the reduction of that value modulo ``ULONG_MAX + 1``.
 
-   Returns ``-1`` on error.  Use :c:func:`PyErr_Occurred` to disambiguate.
+   Returns ``(unsigned long)-1`` on error.  Use :c:func:`PyErr_Occurred` to
+   disambiguate.
 
    .. versionchanged:: 3.8
       Use :meth:`__index__` if available.
@@ -307,7 +308,8 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    If the value of *obj* is out of range for an :c:type:`unsigned long long`,
    return the reduction of that value modulo ``PY_ULLONG_MAX + 1``.
 
-   Returns ``-1`` on error.  Use :c:func:`PyErr_Occurred` to disambiguate.
+   Returns ``(unsigned long long)-1`` on error.  Use :c:func:`PyErr_Occurred`
+   to disambiguate.
 
    .. versionchanged:: 3.8
       Use :meth:`__index__` if available.
diff --git a/Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst b/Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst
new file mode 100644
index 000000000000..7a35c9583d69
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst	
@@ -0,0 +1 @@
+Fix the cast on error in :c:func:`PyLong_AsUnsignedLongLongMask()`.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index f059b4df11b7..c1ae237f2e6c 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -825,6 +825,26 @@ test_long_as_size_t(PyObject *self, PyObject *Py_UNUSED(ignored))
     return Py_None;
 }
 
+static PyObject *
+test_long_as_unsigned_long_long_mask(PyObject *self,
+                                     PyObject *Py_UNUSED(ignored))
+{
+    unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL);
+
+    if (res != (unsigned long long)-1 || !PyErr_Occurred()) {
+        return raiseTestError("test_long_as_unsigned_long_long_mask",
+                              "PyLong_AsUnsignedLongLongMask(NULL) didn't "
+                              "complain");
+    }
+    if (!PyErr_ExceptionMatches(PyExc_SystemError)) {
+        return raiseTestError("test_long_as_unsigned_long_long_mask",
+                              "PyLong_AsUnsignedLongLongMask(NULL) raised "
+                              "something other than SystemError");
+    }
+    PyErr_Clear();
+    Py_RETURN_NONE;
+}
+
 /* Test the PyLong_AsDouble API. At present this just tests that
    non-integer arguments are handled correctly.
  */
@@ -5070,6 +5090,8 @@ static PyMethodDef TestMethods[] = {
     {"test_long_and_overflow",  test_long_and_overflow,          METH_NOARGS},
     {"test_long_as_double",     test_long_as_double,             METH_NOARGS},
     {"test_long_as_size_t",     test_long_as_size_t,             METH_NOARGS},
+    {"test_long_as_unsigned_long_long_mask",
+        test_long_as_unsigned_long_long_mask, METH_NOARGS},
     {"test_long_numbits",       test_long_numbits,               METH_NOARGS},
     {"test_k_code",             test_k_code,                     METH_NOARGS},
     {"test_empty_argparse",     test_empty_argparse,             METH_NOARGS},
diff --git a/Objects/longobject.c b/Objects/longobject.c
index a1103f697c73..50ea2a4e54a4 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -1376,7 +1376,7 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv)
 
     if (vv == NULL || !PyLong_Check(vv)) {
         PyErr_BadInternalCall();
-        return (unsigned long) -1;
+        return (unsigned long long) -1;
     }
     v = (PyLongObject *)vv;
     switch(Py_SIZE(v)) {
@@ -1404,7 +1404,7 @@ PyLong_AsUnsignedLongLongMask(PyObject *op)
 
     if (op == NULL) {
         PyErr_BadInternalCall();
-        return (unsigned long)-1;
+        return (unsigned long long)-1;
     }
 
     if (PyLong_Check(op)) {