[Python-checkins] gh-90110: Fix the c-analyzer Tool (#102483)
ericsnowcurrently
webhook-mailer at python.org
Mon Mar 6 21:40:16 EST 2023
https://github.com/python/cpython/commit/8606697f49dc58ff7e18147401ac65a09c38cf57
commit: 8606697f49dc58ff7e18147401ac65a09c38cf57
branch: main
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: ericsnowcurrently <ericsnowcurrently at gmail.com>
date: 2023-03-06T19:40:09-07:00
summary:
gh-90110: Fix the c-analyzer Tool (#102483)
Some incompatible changes had gone in, and the "ignore" lists weren't properly undated. This change fixes that. It's necessary prior to enabling test_check_c_globals, which I hope to do soon.
Note that this does include moving last_resort_memory_error to PyInterpreterState.
https://github.com/python/cpython/issues/90110
files:
M Include/internal/pycore_global_objects.h
M Include/internal/pycore_intrinsics.h
M Include/internal/pycore_runtime_init.h
M Objects/exceptions.c
M Python/intrinsics.c
M Tools/c-analyzer/c_parser/parser/_common.py
M Tools/c-analyzer/c_parser/parser/_func_body.py
M Tools/c-analyzer/c_parser/preprocessor/gcc.py
M Tools/c-analyzer/cpython/_parser.py
M Tools/c-analyzer/cpython/globals-to-fix.tsv
M Tools/c-analyzer/cpython/ignored.tsv
diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h
index 30c7c4e3bbd0..9957da1fc5f2 100644
--- a/Include/internal/pycore_global_objects.h
+++ b/Include/internal/pycore_global_objects.h
@@ -86,6 +86,7 @@ struct _Py_interp_static_objects {
// hamt_empty is here instead of global because of its weakreflist.
_PyGC_Head_UNUSED _hamt_empty_gc_not_used;
PyHamtObject hamt_empty;
+ PyBaseExceptionObject last_resort_memory_error;
} singletons;
};
diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h
index deac145fff76..46a52740eb8a 100644
--- a/Include/internal/pycore_intrinsics.h
+++ b/Include/internal/pycore_intrinsics.h
@@ -21,6 +21,6 @@
typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value);
typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2);
-extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[];
-extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[];
+extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[];
+extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[];
diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index b54adf04761d..a2cc7c87c2f3 100644
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -14,6 +14,9 @@ extern "C" {
#include "pycore_obmalloc_init.h"
+extern PyTypeObject _PyExc_MemoryError;
+
+
/* The static initializers defined here should only be used
in the runtime init code (in pystate.c and pylifecycle.c). */
@@ -120,6 +123,9 @@ extern "C" {
.ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \
.h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \
}, \
+ .last_resort_memory_error = { \
+ _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \
+ }, \
}, \
}, \
._initial_thread = _PyThreadState_INIT, \
diff --git a/Objects/exceptions.c b/Objects/exceptions.c
index 976f84dbf63c..a473cbdfeda7 100644
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -3207,8 +3207,6 @@ SimpleExtendsException(PyExc_Exception, ReferenceError,
#define MEMERRORS_SAVE 16
-static PyBaseExceptionObject last_resort_memory_error;
-
static PyObject *
get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
{
@@ -3216,7 +3214,9 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
struct _Py_exc_state *state = get_exc_state();
if (state->memerrors_freelist == NULL) {
if (!allow_allocation) {
- return Py_NewRef(&last_resort_memory_error);
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ return Py_NewRef(
+ &_Py_INTERP_SINGLETON(interp, last_resort_memory_error));
}
PyObject *result = BaseException_new((PyTypeObject *)PyExc_MemoryError, args, kwds);
return result;
@@ -3239,8 +3239,6 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
return (PyObject *)self;
}
-static PyBaseExceptionObject last_resort_memory_error;
-
static PyObject *
MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
@@ -3325,7 +3323,7 @@ free_preallocated_memerrors(struct _Py_exc_state *state)
}
-static PyTypeObject _PyExc_MemoryError = {
+PyTypeObject _PyExc_MemoryError = {
PyVarObject_HEAD_INIT(NULL, 0)
"MemoryError",
sizeof(PyBaseExceptionObject),
@@ -3339,9 +3337,6 @@ static PyTypeObject _PyExc_MemoryError = {
};
PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError;
-static PyBaseExceptionObject last_resort_memory_error = {
- _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError)
-};
/*
* BufferError extends Exception
diff --git a/Python/intrinsics.c b/Python/intrinsics.c
index 9e90ef32130f..cca29d859902 100644
--- a/Python/intrinsics.c
+++ b/Python/intrinsics.c
@@ -199,7 +199,7 @@ list_to_tuple(PyThreadState* unused, PyObject *v)
return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v));
}
-instrinsic_func1
+const instrinsic_func1
_PyIntrinsics_UnaryFunctions[] = {
[0] = no_intrinsic,
[INTRINSIC_PRINT] = print_expr,
@@ -221,7 +221,7 @@ prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs)
return _PyExc_PrepReraiseStar(orig, excs);
}
-instrinsic_func2
+const instrinsic_func2
_PyIntrinsics_BinaryFunctions[] = {
[INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star,
};
diff --git a/Tools/c-analyzer/c_parser/parser/_common.py b/Tools/c-analyzer/c_parser/parser/_common.py
index d468d5442a93..2eacace2c001 100644
--- a/Tools/c-analyzer/c_parser/parser/_common.py
+++ b/Tools/c-analyzer/c_parser/parser/_common.py
@@ -7,13 +7,25 @@
)
-def log_match(group, m):
+def log_match(group, m, depth_before=None, depth_after=None):
from . import _logger
- text = m.group(0)
- if text.startswith(('(', ')')) or text.endswith(('(', ')')):
- _logger.debug(f'matched <{group}> ({text!r})')
+
+ if m is not None:
+ text = m.group(0)
+ if text.startswith(('(', ')')) or text.endswith(('(', ')')):
+ _logger.debug(f'matched <{group}> ({text!r})')
+ else:
+ _logger.debug(f'matched <{group}> ({text})')
+
+ elif depth_before is not None or depth_after is not None:
+ if depth_before is None:
+ depth_before = '???'
+ elif depth_after is None:
+ depth_after = '???'
+ _logger.log(1, f'depth: %s -> %s', depth_before, depth_after)
+
else:
- _logger.debug(f'matched <{group}> ({text})')
+ raise NotImplementedError('this should not have been hit')
#############################
diff --git a/Tools/c-analyzer/c_parser/parser/_func_body.py b/Tools/c-analyzer/c_parser/parser/_func_body.py
index 42fd459e111d..25f2f5807ae8 100644
--- a/Tools/c-analyzer/c_parser/parser/_func_body.py
+++ b/Tools/c-analyzer/c_parser/parser/_func_body.py
@@ -65,11 +65,11 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
) = m.groups()
if empty:
- log_match('', m)
+ log_match('', m, depth)
resolve(None, None, None, text)
yield None, text
elif inline_kind:
- log_match('', m)
+ log_match('', m, depth)
kind = inline_kind
name = inline_name or anon_name('inline-')
data = [] # members
@@ -92,7 +92,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
# XXX Should "parent" really be None for inline type decls?
yield resolve(kind, data, name, text, None), text
elif block_close:
- log_match('', m)
+ log_match('', m, depth)
depth -= 1
resolve(None, None, None, text)
# XXX This isn't great. Calling resolve() should have
@@ -101,13 +101,13 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
# needs to be fixed.
yield None, text
elif compound_bare:
- log_match('', m)
+ log_match('', m, depth)
yield resolve('statement', compound_bare, None, text, parent), text
elif compound_labeled:
- log_match('', m)
+ log_match('', m, depth)
yield resolve('statement', compound_labeled, None, text, parent), text
elif compound_paren:
- log_match('', m)
+ log_match('', m, depth)
try:
pos = match_paren(text)
except ValueError:
@@ -132,7 +132,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
}
yield resolve('statement', data, None, text, parent), text
elif block_open:
- log_match('', m)
+ log_match('', m, depth)
depth += 1
if block_leading:
# An inline block: the last evaluated expression is used
@@ -144,10 +144,10 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
resolve(None, None, None, text)
yield None, text
elif simple_ending:
- log_match('', m)
+ log_match('', m, depth)
yield resolve('statement', simple_stmt, None, text, parent), text
elif var_ending:
- log_match('', m)
+ log_match('', m, depth)
kind = 'variable'
_, name, vartype = parse_var_decl(decl)
data = {
@@ -220,7 +220,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth):
remainder = srcinfo.text[m.end():]
if inline_kind:
- log_match('func inline', m)
+ log_match('func inline', m, depth, depth)
kind = inline_kind
name = inline_name or anon_name('inline-')
# Immediately emit a forward declaration.
@@ -249,7 +249,7 @@ def parse_body(source):
yield parse_body, depth
elif static_decl:
- log_match('local variable', m)
+ log_match('local variable', m, depth, depth)
_, name, data = parse_var_decl(static_decl)
yield srcinfo.resolve('variable', data, name, parent=func), depth
@@ -266,10 +266,13 @@ def parse_body(source):
else:
log_match('func other', m)
if block_open:
+ log_match('func other', None, depth, depth + 1)
depth += 1
elif block_close:
+ log_match('func other', None, depth, depth - 1)
depth -= 1
elif stmt_end:
+ log_match('func other', None, depth, depth)
pass
else:
# This should be unreachable.
diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py
index 770802253792..7ef1a8afc3b1 100644
--- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py
+++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py
@@ -29,7 +29,7 @@
[^()]*
)*
) # <args>
- ( [)] [)] )? # <closed>
+ ( [)] [)] ) # <closed>
''', re.VERBOSE)
POST_ARGS = (
@@ -156,6 +156,7 @@ def _iter_top_include_lines(lines, topfile, cwd,
if name != 'pragma':
raise Exception(line)
else:
+ line = re.sub(r'__inline__', 'inline', line)
if not raw:
line, partial = _strip_directives(line, partial=partial)
yield _common.SourceLine(
diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py
index ab1d6257f1b1..e7764165d36c 100644
--- a/Tools/c-analyzer/cpython/_parser.py
+++ b/Tools/c-analyzer/cpython/_parser.py
@@ -105,9 +105,14 @@ def clean_lines(text):
* ./Include
* ./Include/internal
+Modules/_decimal/**/*.c Modules/_decimal/libmpdec
+Modules/_hacl/*.c Modules/_hacl/include
+Modules/_hacl/*.h Modules/_hacl/include
Modules/_tkinter.c /usr/include/tcl8.6
+Modules/md5module.c Modules/_hacl/include
+Modules/sha1module.c Modules/_hacl/include
+Modules/sha2module.c Modules/_hacl/include
Modules/tkappinit.c /usr/include/tcl
-Modules/_decimal/**/*.c Modules/_decimal/libmpdec
Objects/stringlib/*.h Objects
# @end=tsv@
@@ -173,6 +178,7 @@ def clean_lines(text):
Modules/_functoolsmodule.c Py_BUILD_CORE 1
Modules/_heapqmodule.c Py_BUILD_CORE 1
Modules/_io/*.c Py_BUILD_CORE 1
+Modules/_io/*.h Py_BUILD_CORE 1
Modules/_localemodule.c Py_BUILD_CORE 1
Modules/_operator.c Py_BUILD_CORE 1
Modules/_posixsubprocess.c Py_BUILD_CORE 1
@@ -296,6 +302,7 @@ def clean_lines(text):
# First match wins.
_abs('Modules/_ctypes/ctypes.h'): (5_000, 500),
_abs('Modules/_datetimemodule.c'): (20_000, 300),
+ _abs('Modules/_hacl/*.c'): (200_000, 500),
_abs('Modules/posixmodule.c'): (20_000, 500),
_abs('Modules/termios.c'): (10_000, 800),
_abs('Modules/_testcapimodule.c'): (20_000, 400),
diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index 2e28c50c6ff6..57b8542fb464 100644
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -539,7 +539,7 @@ Modules/_tkinter.c - command_mutex -
Modules/_tkinter.c - HeadFHCD -
Modules/_tkinter.c - stdin_ready -
Modules/_tkinter.c - event_tstate -
-Modules/_xxsubinterpretersmodule.c - _globals -
+Modules/_xxinterpchannelsmodule.c - _globals -
Modules/readline.c - completer_word_break_characters -
Modules/readline.c - _history_length -
Modules/readline.c - should_auto_add_history -
diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv
index 849e20a1b0f4..700ddf285183 100644
--- a/Tools/c-analyzer/cpython/ignored.tsv
+++ b/Tools/c-analyzer/cpython/ignored.tsv
@@ -206,6 +206,10 @@ Modules/_decimal/_decimal.c - invalid_signals_err -
Modules/_decimal/_decimal.c - signal_map -
Modules/_decimal/_decimal.c - ssize_constants -
Modules/_elementtree.c - ExpatMemoryHandler -
+Modules/_hashopenssl.c - py_hashes -
+Modules/_hacl/Hacl_Hash_SHA1.c - _h0 -
+Modules/_hacl/Hacl_Hash_MD5.c - _h0 -
+Modules/_hacl/Hacl_Hash_MD5.c - _t -
Modules/_io/_iomodule.c - static_types -
Modules/_io/textio.c - encodefuncs -
Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type -
@@ -218,9 +222,10 @@ Modules/_sre/sre_targets.h - sre_targets -
Modules/_sre.c pattern_repr flag_names -
Modules/_struct.c - bigendian_table -
Modules/_struct.c - lilendian_table -
+Modules/_struct.c - native_table -
Modules/_tkinter.c - state_key -
-Modules/_xxsubinterpretersmodule.c - _channelid_end_recv -
-Modules/_xxsubinterpretersmodule.c - _channelid_end_send -
+Modules/_xxinterpchannelsmodule.c - _channelid_end_recv -
+Modules/_xxinterpchannelsmodule.c - _channelid_end_send -
Modules/_zoneinfo.c - DAYS_BEFORE_MONTH -
Modules/_zoneinfo.c - DAYS_IN_MONTH -
Modules/arraymodule.c - descriptors -
@@ -332,12 +337,15 @@ Python/frozen.c - _PyImport_FrozenTest -
Python/getopt.c - longopts -
Python/import.c - _PyImport_Inittab -
Python/import.c - _PySys_ImplCacheTag -
+Python/intrinsics.c - _PyIntrinsics_UnaryFunctions -
+Python/intrinsics.c - _PyIntrinsics_BinaryFunctions -
Python/opcode_targets.h - opcode_targets -
Python/perf_trampoline.c - _Py_perfmap_callbacks -
Python/pyhash.c - PyHash_Func -
Python/pylifecycle.c - _C_LOCALE_WARNING -
Python/pylifecycle.c - _PyOS_mystrnicmp_hack -
Python/pylifecycle.c - _TARGET_LOCALES -
+Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF -
Python/pystate.c - initial -
Python/specialize.c - adaptive_opcodes -
Python/specialize.c - cache_requirements -
@@ -392,8 +400,23 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides -
Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets -
Modules/_testbuffer.c ndarray_push kwlist -
Modules/_testbuffer.c staticarray_init kwlist -
+Modules/_testcapi/code.c get_code_extra_index key -
+Modules/_testcapi/datetime.c - test_run_counter -
+Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type -
Modules/_testcapi/heaptype.c - _testcapimodule -
+Modules/_testcapi/mem.c - FmData -
+Modules/_testcapi/mem.c - FmHook -
+Modules/_testcapi/structmember.c - test_structmembersType_OldAPI -
Modules/_testcapi/unicode.c - _testcapimodule -
+Modules/_testcapi/watchers.c - g_dict_watch_events -
+Modules/_testcapi/watchers.c - g_dict_watchers_installed -
+Modules/_testcapi/watchers.c - g_type_modified_events -
+Modules/_testcapi/watchers.c - g_type_watchers_installed -
+Modules/_testcapi/watchers.c - num_code_object_created_events -
+Modules/_testcapi/watchers.c - num_code_object_destroyed_events -
+Modules/_testcapi/watchers.c - pyfunc_watchers -
+Modules/_testcapi/watchers.c - func_watcher_ids -
+Modules/_testcapi/watchers.c - func_watcher_callbacks -
Modules/_testcapimodule.c - ContainerNoGC_members -
Modules/_testcapimodule.c - ContainerNoGC_type -
Modules/_testcapimodule.c - FmData -
@@ -460,11 +483,13 @@ Modules/_testcapimodule.c - meth_static_methods -
Modules/_testcapimodule.c - ml -
Modules/_testcapimodule.c - str1 -
Modules/_testcapimodule.c - str2 -
+Modules/_testcapimodule.c - test_c_thread -
Modules/_testcapimodule.c - test_members -
Modules/_testcapimodule.c - test_run_counter -
Modules/_testcapimodule.c - test_structmembersType -
Modules/_testcapimodule.c - thread_done -
Modules/_testcapimodule.c - x -
+Modules/_testcapimodule.c - wait_done -
Modules/_testcapimodule.c getargs_keyword_only keywords -
Modules/_testcapimodule.c getargs_keywords keywords -
Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords -
@@ -526,6 +551,7 @@ Modules/_testmultiphase.c - slots_exec_unreported_exception -
Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots -
Modules/_testmultiphase.c - testexport_methods -
Modules/_testmultiphase.c - uninitialized_def -
+Modules/_testsinglephase.c - global_state -
Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule -
Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods -
Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG -
More information about the Python-checkins
mailing list