[Python-checkins] gh-98393: os module reject bytes-like, only accept bytes (#98394)
vstinner
webhook-mailer at python.org
Tue Oct 18 11:52:51 EDT 2022
https://github.com/python/cpython/commit/db03c8066a6b12fa23618f9add0eb795936726b4
commit: db03c8066a6b12fa23618f9add0eb795936726b4
branch: main
author: Victor Stinner <vstinner at python.org>
committer: vstinner <vstinner at python.org>
date: 2022-10-18T17:52:31+02:00
summary:
gh-98393: os module reject bytes-like, only accept bytes (#98394)
The os module and the PyUnicode_FSDecoder() function no longer accept
bytes-like paths, like bytearray and memoryview types: only the exact
bytes type is accepted for bytes strings.
files:
A Misc/NEWS.d/next/C API/2022-10-18-16-16-27.gh-issue-98393.55u4BF.rst
A Misc/NEWS.d/next/Library/2022-10-18-15-41-37.gh-issue-98393.vhPu4L.rst
M Doc/whatsnew/3.12.rst
M Lib/test/test_compile.py
M Lib/test/test_os.py
M Lib/test/test_posix.py
M Lib/test/test_symtable.py
M Modules/posixmodule.c
M Objects/unicodeobject.c
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index ad0270ae19b8..525efc405c85 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -515,6 +515,11 @@ Changes in the Python API
in Python 3.9.
(Contributed by Victor Stinner in :gh:`94352`.)
+* The :mod:`os` module no longer accepts bytes-like paths, like
+ :class:`bytearray` and :class:`memoryview` types: only the exact
+ :class:`bytes` type is accepted for bytes strings.
+ (Contributed by Victor Stinner in :gh:`98393`.)
+
Build Changes
=============
@@ -631,6 +636,11 @@ Porting to Python 3.12
to traverse and clear their instance's dictionaries.
To clear weakrefs, call :c:func:`PyObject_ClearWeakRefs`, as before.
+* The :c:func:`PyUnicode_FSDecoder` function no longer accepts bytes-like
+ paths, like :class:`bytearray` and :class:`memoryview` types: only the exact
+ :class:`bytes` type is accepted for bytes strings.
+ (Contributed by Victor Stinner in :gh:`98393`.)
+
Deprecated
----------
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 8bf8470ff16f..f5c0c7647832 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -494,9 +494,8 @@ def test_compile_filename(self):
code = compile('pass', filename, 'exec')
self.assertEqual(code.co_filename, 'file.py')
for filename in bytearray(b'file.py'), memoryview(b'file.py'):
- with self.assertWarns(DeprecationWarning):
- code = compile('pass', filename, 'exec')
- self.assertEqual(code.co_filename, 'file.py')
+ with self.assertRaises(TypeError):
+ compile('pass', filename, 'exec')
self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec')
@support.cpython_only
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 2dafb78ebe0e..10dbb2bd15df 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -3860,18 +3860,18 @@ def test_oserror_filename(self):
for filenames, func, *func_args in funcs:
for name in filenames:
- try:
- if isinstance(name, (str, bytes)):
+ if not isinstance(name, (str, bytes)):
+ with self.assertRaises(TypeError):
func(name, *func_args)
- else:
- with self.assertWarnsRegex(DeprecationWarning, 'should be'):
- func(name, *func_args)
- except OSError as err:
- self.assertIs(err.filename, name, str(func))
- except UnicodeDecodeError:
- pass
else:
- self.fail("No exception thrown by {}".format(func))
+ try:
+ func(name, *func_args)
+ except OSError as err:
+ self.assertIs(err.filename, name, str(func))
+ except UnicodeDecodeError:
+ pass
+ else:
+ self.fail("No exception thrown by {}".format(func))
class CPUCountTests(unittest.TestCase):
def test_cpu_count(self):
@@ -4350,16 +4350,8 @@ def test_bytes_like(self):
for cls in bytearray, memoryview:
path_bytes = cls(os.fsencode(self.path))
- with self.assertWarns(DeprecationWarning):
- entries = list(os.scandir(path_bytes))
- self.assertEqual(len(entries), 1, entries)
- entry = entries[0]
-
- self.assertEqual(entry.name, b'file.txt')
- self.assertEqual(entry.path,
- os.fsencode(os.path.join(self.path, 'file.txt')))
- self.assertIs(type(entry.name), bytes)
- self.assertIs(type(entry.path), bytes)
+ with self.assertRaises(TypeError):
+ list(os.scandir(path_bytes))
@unittest.skipUnless(os.listdir in os.supports_fd,
'fd support for listdir required for this test.')
diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py
index e643d8e5a4ce..be561afc4cff 100644
--- a/Lib/test/test_posix.py
+++ b/Lib/test/test_posix.py
@@ -634,7 +634,7 @@ def test_stat(self):
self.assertTrue(posix.stat(os_helper.TESTFN))
self.assertTrue(posix.stat(os.fsencode(os_helper.TESTFN)))
- self.assertWarnsRegex(DeprecationWarning,
+ self.assertRaisesRegex(TypeError,
'should be string, bytes, os.PathLike or integer, not',
posix.stat, bytearray(os.fsencode(os_helper.TESTFN)))
self.assertRaisesRegex(TypeError,
@@ -841,11 +841,8 @@ def test_listdir_bytes(self):
def test_listdir_bytes_like(self):
for cls in bytearray, memoryview:
- with self.assertWarns(DeprecationWarning):
- names = posix.listdir(cls(b'.'))
- self.assertIn(os.fsencode(os_helper.TESTFN), names)
- for name in names:
- self.assertIs(type(name), bytes)
+ with self.assertRaises(TypeError):
+ posix.listdir(cls(b'.'))
@unittest.skipUnless(posix.listdir in os.supports_fd,
"test needs fd support for posix.listdir()")
diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py
index 819354e4eee9..25714aecda3a 100644
--- a/Lib/test/test_symtable.py
+++ b/Lib/test/test_symtable.py
@@ -222,10 +222,9 @@ def checkfilename(brokencode, offset):
checkfilename("def f(x): foo)(", 14) # parse-time
checkfilename("def f(x): global x", 11) # symtable-build-time
symtable.symtable("pass", b"spam", "exec")
- with self.assertWarns(DeprecationWarning), \
- self.assertRaises(TypeError):
+ with self.assertRaises(TypeError):
symtable.symtable("pass", bytearray(b"spam"), "exec")
- with self.assertWarns(DeprecationWarning):
+ with self.assertRaises(TypeError):
symtable.symtable("pass", memoryview(b"spam"), "exec")
with self.assertRaises(TypeError):
symtable.symtable("pass", list(b"spam"), "exec")
diff --git a/Misc/NEWS.d/next/C API/2022-10-18-16-16-27.gh-issue-98393.55u4BF.rst b/Misc/NEWS.d/next/C API/2022-10-18-16-16-27.gh-issue-98393.55u4BF.rst
new file mode 100644
index 000000000000..658f587348a3
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2022-10-18-16-16-27.gh-issue-98393.55u4BF.rst
@@ -0,0 +1,3 @@
+The :c:func:`PyUnicode_FSDecoder` function no longer accepts bytes-like
+paths, like :class:`bytearray` and :class:`memoryview` types: only the exact
+:class:`bytes` type is accepted for bytes strings. Patch by Victor Stinner.
diff --git a/Misc/NEWS.d/next/Library/2022-10-18-15-41-37.gh-issue-98393.vhPu4L.rst b/Misc/NEWS.d/next/Library/2022-10-18-15-41-37.gh-issue-98393.vhPu4L.rst
new file mode 100644
index 000000000000..54e45bb900b2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-10-18-15-41-37.gh-issue-98393.vhPu4L.rst
@@ -0,0 +1,3 @@
+The :mod:`os` module no longer accepts bytes-like paths, like
+:class:`bytearray` and :class:`memoryview` types: only the exact
+:class:`bytes` type is accepted for bytes strings. Patch by Victor Stinner.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 1e556fc904b7..39198cb7ddcd 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -1120,7 +1120,7 @@ path_converter(PyObject *o, void *p)
path_t *path = (path_t *)p;
PyObject *bytes = NULL;
Py_ssize_t length = 0;
- int is_index, is_buffer, is_bytes, is_unicode;
+ int is_index, is_bytes, is_unicode;
const char *narrow;
#ifdef MS_WINDOWS
PyObject *wo = NULL;
@@ -1158,11 +1158,10 @@ path_converter(PyObject *o, void *p)
/* Only call this here so that we don't treat the return value of
os.fspath() as an fd or buffer. */
is_index = path->allow_fd && PyIndex_Check(o);
- is_buffer = PyObject_CheckBuffer(o);
is_bytes = PyBytes_Check(o);
is_unicode = PyUnicode_Check(o);
- if (!is_index && !is_buffer && !is_unicode && !is_bytes) {
+ if (!is_index && !is_unicode && !is_bytes) {
/* Inline PyOS_FSPath() for better error messages. */
PyObject *func, *res;
@@ -1225,27 +1224,6 @@ path_converter(PyObject *o, void *p)
bytes = o;
Py_INCREF(bytes);
}
- else if (is_buffer) {
- /* XXX Replace PyObject_CheckBuffer with PyBytes_Check in other code
- after removing support of non-bytes buffer objects. */
- if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
- "%s%s%s should be %s, not %.200s",
- path->function_name ? path->function_name : "",
- path->function_name ? ": " : "",
- path->argument_name ? path->argument_name : "path",
- path->allow_fd && path->nullable ? "string, bytes, os.PathLike, "
- "integer or None" :
- path->allow_fd ? "string, bytes, os.PathLike or integer" :
- path->nullable ? "string, bytes, os.PathLike or None" :
- "string, bytes or os.PathLike",
- _PyType_Name(Py_TYPE(o)))) {
- goto error_exit;
- }
- bytes = PyBytes_FromObject(o);
- if (!bytes) {
- goto error_exit;
- }
- }
else if (is_index) {
if (!_fd_converter(o, &path->fd)) {
goto error_exit;
@@ -4126,8 +4104,8 @@ _posix_listdir(path_t *path, PyObject *list)
const char *name;
if (path->narrow) {
name = path->narrow;
- /* only return bytes if they specified a bytes-like object */
- return_str = !PyObject_CheckBuffer(path->object);
+ /* only return bytes if they specified a bytes object */
+ return_str = !PyBytes_Check(path->object);
}
else {
name = ".";
@@ -14049,7 +14027,7 @@ DirEntry_from_posix_info(PyObject *module, path_t *path, const char *name,
goto error;
}
- if (!path->narrow || !PyObject_CheckBuffer(path->object)) {
+ if (!path->narrow || !PyBytes_Check(path->object)) {
entry->name = PyUnicode_DecodeFSDefaultAndSize(name, name_len);
if (joined_path)
entry->path = PyUnicode_DecodeFSDefault(joined_path);
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 5b737f1b596e..d090915146f8 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -3619,48 +3619,25 @@ PyUnicode_FSConverter(PyObject* arg, void* addr)
int
PyUnicode_FSDecoder(PyObject* arg, void* addr)
{
- int is_buffer = 0;
- PyObject *path = NULL;
- PyObject *output = NULL;
if (arg == NULL) {
Py_DECREF(*(PyObject**)addr);
*(PyObject**)addr = NULL;
return 1;
}
- is_buffer = PyObject_CheckBuffer(arg);
- if (!is_buffer) {
- path = PyOS_FSPath(arg);
- if (path == NULL) {
- return 0;
- }
- }
- else {
- path = arg;
- Py_INCREF(arg);
+ PyObject *path = PyOS_FSPath(arg);
+ if (path == NULL) {
+ return 0;
}
+ PyObject *output = NULL;
if (PyUnicode_Check(path)) {
output = path;
}
- else if (PyBytes_Check(path) || is_buffer) {
- PyObject *path_bytes = NULL;
-
- if (!PyBytes_Check(path) &&
- PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
- "path should be string, bytes, or os.PathLike, not %.200s",
- Py_TYPE(arg)->tp_name)) {
- Py_DECREF(path);
- return 0;
- }
- path_bytes = PyBytes_FromObject(path);
+ else if (PyBytes_Check(path)) {
+ output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path),
+ PyBytes_GET_SIZE(path));
Py_DECREF(path);
- if (!path_bytes) {
- return 0;
- }
- output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path_bytes),
- PyBytes_GET_SIZE(path_bytes));
- Py_DECREF(path_bytes);
if (!output) {
return 0;
}
@@ -3672,6 +3649,7 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr)
Py_DECREF(path);
return 0;
}
+
if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output),
PyUnicode_GET_LENGTH(output), 0, 1) >= 0) {
PyErr_SetString(PyExc_ValueError, "embedded null character");
More information about the Python-checkins
mailing list