[Python-checkins] bpo-37001: Makes symtable.symtable have parity with compile for input (#13483)
Dino Viehland
webhook-mailer at python.org
Tue May 28 19:21:22 EDT 2019
https://github.com/python/cpython/commit/415406999d7c09af9f3dcacfb4578b9e97b2ce77
commit: 415406999d7c09af9f3dcacfb4578b9e97b2ce77
branch: master
author: Dino Viehland <dinoviehland at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-05-28T16:21:17-07:00
summary:
bpo-37001: Makes symtable.symtable have parity with compile for input (#13483)
* Makes symtable.symtable have parity for accepted datatypes
for source code as compile()
* Add NEWS blurb
files:
A Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst
M Include/pythonrun.h
M Lib/test/test_symtable.py
M Modules/clinic/symtablemodule.c.h
M Modules/symtablemodule.c
M Python/bltinmodule.c
M Python/pythonrun.c
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index e83846add981..196355cb8f40 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -119,10 +119,23 @@ PyAPI_FUNC(struct symtable *) Py_SymtableString(
const char *filename, /* decoded from the filesystem encoding */
int start);
#ifndef Py_LIMITED_API
+PyAPI_FUNC(const char *) _Py_SourceAsString(
+ PyObject *cmd,
+ const char *funcname,
+ const char *what,
+ PyCompilerFlags *cf,
+ PyObject **cmd_copy);
+
PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
const char *str,
PyObject *filename,
int start);
+
+PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags(
+ const char *str,
+ PyObject *filename,
+ int start,
+ PyCompilerFlags *flags);
#endif
PyAPI_FUNC(void) PyErr_Print(void);
diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py
index 0a1cb8d5b432..bea2ce120ca9 100644
--- a/Lib/test/test_symtable.py
+++ b/Lib/test/test_symtable.py
@@ -215,6 +215,15 @@ def test_single(self):
def test_exec(self):
symbols = symtable.symtable("def f(x): return x", "?", "exec")
+ def test_bytes(self):
+ top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec")
+ self.assertIsNotNone(find_block(top, "Mine"))
+
+ code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n'
+
+ top = symtable.symtable(code, "?", "exec")
+ self.assertIsNotNone(find_block(top, "\u017d"))
+
if __name__ == '__main__':
unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst
new file mode 100644
index 000000000000..5bcd7a9976c5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst
@@ -0,0 +1,2 @@
+:func:`symtable.symtable` now accepts the same input types for source code as the
+built-in :func:`compile` function. Patch by Dino Viehland.
diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h
index 73e340bd462a..7d8b0ad300c2 100644
--- a/Modules/clinic/symtablemodule.c.h
+++ b/Modules/clinic/symtablemodule.c.h
@@ -3,7 +3,7 @@ preserve
[clinic start generated code]*/
PyDoc_STRVAR(_symtable_symtable__doc__,
-"symtable($module, str, filename, startstr, /)\n"
+"symtable($module, source, filename, startstr, /)\n"
"--\n"
"\n"
"Return symbol and scope dictionaries used internally by compiler.");
@@ -12,33 +12,21 @@ PyDoc_STRVAR(_symtable_symtable__doc__,
{"symtable", (PyCFunction)(void(*)(void))_symtable_symtable, METH_FASTCALL, _symtable_symtable__doc__},
static PyObject *
-_symtable_symtable_impl(PyObject *module, const char *str,
+_symtable_symtable_impl(PyObject *module, PyObject *source,
PyObject *filename, const char *startstr);
static PyObject *
_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
- const char *str;
+ PyObject *source;
PyObject *filename;
const char *startstr;
if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) {
goto exit;
}
- if (!PyUnicode_Check(args[0])) {
- _PyArg_BadArgument("symtable", 1, "str", args[0]);
- goto exit;
- }
- Py_ssize_t str_length;
- str = PyUnicode_AsUTF8AndSize(args[0], &str_length);
- if (str == NULL) {
- goto exit;
- }
- if (strlen(str) != (size_t)str_length) {
- PyErr_SetString(PyExc_ValueError, "embedded null character");
- goto exit;
- }
+ source = args[0];
if (!PyUnicode_FSDecoder(args[1], &filename)) {
goto exit;
}
@@ -55,9 +43,9 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
PyErr_SetString(PyExc_ValueError, "embedded null character");
goto exit;
}
- return_value = _symtable_symtable_impl(module, str, filename, startstr);
+ return_value = _symtable_symtable_impl(module, source, filename, startstr);
exit:
return return_value;
}
-/*[clinic end generated code: output=be1cca59de019984 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=de655625eee705f4 input=a9049054013a1b77]*/
diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c
index e8d2f5b582be..d66cb44f69bd 100644
--- a/Modules/symtablemodule.c
+++ b/Modules/symtablemodule.c
@@ -14,7 +14,7 @@ module _symtable
/*[clinic input]
_symtable.symtable
- str: str
+ source: object
filename: object(converter='PyUnicode_FSDecoder')
startstr: str
/
@@ -23,13 +23,23 @@ Return symbol and scope dictionaries used internally by compiler.
[clinic start generated code]*/
static PyObject *
-_symtable_symtable_impl(PyObject *module, const char *str,
+_symtable_symtable_impl(PyObject *module, PyObject *source,
PyObject *filename, const char *startstr)
-/*[clinic end generated code: output=914b369c9b785956 input=6c615e84d5f408e3]*/
+/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/
{
struct symtable *st;
PyObject *t;
int start;
+ PyCompilerFlags cf;
+ PyObject *source_copy = NULL;
+
+ cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+ cf.cf_feature_version = PY_MINOR_VERSION;
+
+ const char *str = _Py_SourceAsString(source, "symtable", "string or bytes", &cf, &source_copy);
+ if (str == NULL) {
+ return NULL;
+ }
if (strcmp(startstr, "exec") == 0)
start = Py_file_input;
@@ -41,12 +51,15 @@ _symtable_symtable_impl(PyObject *module, const char *str,
PyErr_SetString(PyExc_ValueError,
"symtable() arg 3 must be 'exec' or 'eval' or 'single'");
Py_DECREF(filename);
+ Py_XDECREF(source_copy);
return NULL;
}
- st = Py_SymtableStringObject(str, filename, start);
+ st = _Py_SymtableStringObjectFlags(str, filename, start, &cf);
Py_DECREF(filename);
- if (st == NULL)
+ Py_XDECREF(source_copy);
+ if (st == NULL) {
return NULL;
+ }
t = (PyObject *)st->st_top;
Py_INCREF(t);
PyMem_Free((void *)st->st_future);
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 5d5808530e11..065ad95c95b1 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -687,55 +687,6 @@ builtin_chr_impl(PyObject *module, int i)
}
-static const char *
-source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
-{
- const char *str;
- Py_ssize_t size;
- Py_buffer view;
-
- *cmd_copy = NULL;
- if (PyUnicode_Check(cmd)) {
- cf->cf_flags |= PyCF_IGNORE_COOKIE;
- str = PyUnicode_AsUTF8AndSize(cmd, &size);
- if (str == NULL)
- return NULL;
- }
- else if (PyBytes_Check(cmd)) {
- str = PyBytes_AS_STRING(cmd);
- size = PyBytes_GET_SIZE(cmd);
- }
- else if (PyByteArray_Check(cmd)) {
- str = PyByteArray_AS_STRING(cmd);
- size = PyByteArray_GET_SIZE(cmd);
- }
- else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
- /* Copy to NUL-terminated buffer. */
- *cmd_copy = PyBytes_FromStringAndSize(
- (const char *)view.buf, view.len);
- PyBuffer_Release(&view);
- if (*cmd_copy == NULL) {
- return NULL;
- }
- str = PyBytes_AS_STRING(*cmd_copy);
- size = PyBytes_GET_SIZE(*cmd_copy);
- }
- else {
- PyErr_Format(PyExc_TypeError,
- "%s() arg 1 must be a %s object",
- funcname, what);
- return NULL;
- }
-
- if (strlen(str) != (size_t)size) {
- PyErr_SetString(PyExc_ValueError,
- "source code string cannot contain null bytes");
- Py_CLEAR(*cmd_copy);
- return NULL;
- }
- return str;
-}
-
/*[clinic input]
compile as builtin_compile
@@ -855,7 +806,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
goto finally;
}
- str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy);
+ str = _Py_SourceAsString(source, "compile", "string, bytes or AST", &cf, &source_copy);
if (str == NULL)
goto error;
@@ -991,7 +942,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals,
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
cf.cf_feature_version = PY_MINOR_VERSION;
- str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
+ str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy);
if (str == NULL)
return NULL;
@@ -1083,7 +1034,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals,
PyCompilerFlags cf;
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
cf.cf_feature_version = PY_MINOR_VERSION;
- str = source_as_string(source, "exec",
+ str = _Py_SourceAsString(source, "exec",
"string, bytes or code", &cf,
&source_copy);
if (str == NULL)
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index ace9f2f9874e..784c15bb4b22 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -1231,21 +1231,77 @@ PyCompileString(const char *str, const char *filename, int start)
return Py_CompileStringFlags(str, filename, start, NULL);
}
+const char *
+_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
+{
+ const char *str;
+ Py_ssize_t size;
+ Py_buffer view;
+
+ *cmd_copy = NULL;
+ if (PyUnicode_Check(cmd)) {
+ cf->cf_flags |= PyCF_IGNORE_COOKIE;
+ str = PyUnicode_AsUTF8AndSize(cmd, &size);
+ if (str == NULL)
+ return NULL;
+ }
+ else if (PyBytes_Check(cmd)) {
+ str = PyBytes_AS_STRING(cmd);
+ size = PyBytes_GET_SIZE(cmd);
+ }
+ else if (PyByteArray_Check(cmd)) {
+ str = PyByteArray_AS_STRING(cmd);
+ size = PyByteArray_GET_SIZE(cmd);
+ }
+ else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
+ /* Copy to NUL-terminated buffer. */
+ *cmd_copy = PyBytes_FromStringAndSize(
+ (const char *)view.buf, view.len);
+ PyBuffer_Release(&view);
+ if (*cmd_copy == NULL) {
+ return NULL;
+ }
+ str = PyBytes_AS_STRING(*cmd_copy);
+ size = PyBytes_GET_SIZE(*cmd_copy);
+ }
+ else {
+ PyErr_Format(PyExc_TypeError,
+ "%s() arg 1 must be a %s object",
+ funcname, what);
+ return NULL;
+ }
+
+ if (strlen(str) != (size_t)size) {
+ PyErr_SetString(PyExc_ValueError,
+ "source code string cannot contain null bytes");
+ Py_CLEAR(*cmd_copy);
+ return NULL;
+ }
+ return str;
+}
+
struct symtable *
Py_SymtableStringObject(const char *str, PyObject *filename, int start)
+{
+ PyCompilerFlags flags;
+
+ flags.cf_flags = 0;
+ flags.cf_feature_version = PY_MINOR_VERSION;
+ return _Py_SymtableStringObjectFlags(str, filename, start, &flags);
+}
+
+struct symtable *
+_Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags)
{
struct symtable *st;
mod_ty mod;
- PyCompilerFlags flags;
PyArena *arena;
arena = PyArena_New();
if (arena == NULL)
return NULL;
- flags.cf_flags = 0;
- flags.cf_feature_version = PY_MINOR_VERSION;
- mod = PyParser_ASTFromStringObject(str, filename, start, &flags, arena);
+ mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
More information about the Python-checkins
mailing list