gh-90110: Fix the c-analyzer Tool (#102483)

Some incompatible changes had gone in, and the "ignore" lists weren't properly undated. This change fixes that. It's necessary prior to enabling test_check_c_globals, which I hope to do soon.

Note that this does include moving last_resort_memory_error to PyInterpreterState.

https://github.com/python/cpython/issues/90110

Include/internal/pycore_global_objects.h

@@ -86,6 +86,7 @@ struct _Py_interp_static_objects {
         // hamt_empty is here instead of global because of its weakreflist.
         _PyGC_Head_UNUSED _hamt_empty_gc_not_used;
         PyHamtObject hamt_empty;
+        PyBaseExceptionObject last_resort_memory_error;
     } singletons;
 };
 

Include/internal/pycore_intrinsics.h

@@ -21,6 +21,6 @@
 typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value);
 typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2);
 
-extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[];
+extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[];
-extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[];
+extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[];
 

Include/internal/pycore_runtime_init.h

@@ -14,6 +14,9 @@ extern "C" {
 #include "pycore_obmalloc_init.h"
 
 
+extern PyTypeObject _PyExc_MemoryError;
+
+
 /* The static initializers defined here should only be used
    in the runtime init code (in pystate.c and pylifecycle.c). */
 
@@ -120,6 +123,9 @@ extern "C" {
                     .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \
                     .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \
                 }, \
+                .last_resort_memory_error = { \
+                    _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \
+                }, \
             }, \
         }, \
         ._initial_thread = _PyThreadState_INIT, \

Objects/exceptions.c

@@ -3207,8 +3207,6 @@ SimpleExtendsException(PyExc_Exception, ReferenceError,
 
 #define MEMERRORS_SAVE 16
 
-static PyBaseExceptionObject last_resort_memory_error;
-
 static PyObject *
 get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
 {
@@ -3216,7 +3214,9 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
     struct _Py_exc_state *state = get_exc_state();
     if (state->memerrors_freelist == NULL) {
         if (!allow_allocation) {
-            return Py_NewRef(&last_resort_memory_error);
+            PyInterpreterState *interp = _PyInterpreterState_GET();
+            return Py_NewRef(
+                &_Py_INTERP_SINGLETON(interp, last_resort_memory_error));
         }
         PyObject *result = BaseException_new((PyTypeObject *)PyExc_MemoryError, args, kwds);
         return result;
@@ -3239,8 +3239,6 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
     return (PyObject *)self;
 }
 
-static PyBaseExceptionObject last_resort_memory_error;
-
 static PyObject *
 MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
@@ -3325,7 +3323,7 @@ free_preallocated_memerrors(struct _Py_exc_state *state)
 }
 
 
-static PyTypeObject _PyExc_MemoryError = {
+PyTypeObject _PyExc_MemoryError = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "MemoryError",
     sizeof(PyBaseExceptionObject),
@@ -3339,9 +3337,6 @@ static PyTypeObject _PyExc_MemoryError = {
 };
 PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError;
 
-static PyBaseExceptionObject last_resort_memory_error = {
-    _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError)
-};
 
 /*
  *    BufferError extends Exception

Python/intrinsics.c

@@ -199,7 +199,7 @@ list_to_tuple(PyThreadState* unused, PyObject *v)
     return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v));
 }
 
-instrinsic_func1
+const instrinsic_func1
 _PyIntrinsics_UnaryFunctions[] = {
     [0] = no_intrinsic,
     [INTRINSIC_PRINT] = print_expr,
@@ -221,7 +221,7 @@ prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs)
     return _PyExc_PrepReraiseStar(orig, excs);
 }
 
-instrinsic_func2
+const instrinsic_func2
 _PyIntrinsics_BinaryFunctions[] = {
     [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star,
 };

Tools/c-analyzer/c_parser/parser/_common.py

@@ -7,14 +7,26 @@ from ._regexes import (
 )
 
 
-def log_match(group, m):
+def log_match(group, m, depth_before=None, depth_after=None):
     from . import _logger
+
+    if m is not None:
         text = m.group(0)
         if text.startswith(('(', ')')) or text.endswith(('(', ')')):
             _logger.debug(f'matched <{group}> ({text!r})')
         else:
             _logger.debug(f'matched <{group}> ({text})')
 
+    elif depth_before is not None or depth_after is not None:
+        if depth_before is None:
+            depth_before = '???'
+        elif depth_after is None:
+            depth_after = '???'
+        _logger.log(1, f'depth: %s -> %s', depth_before, depth_after)
+
+    else:
+        raise NotImplementedError('this should not have been hit')
+
 
 #############################
 # regex utils

Tools/c-analyzer/c_parser/parser/_func_body.py

@@ -65,11 +65,11 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
          ) = m.groups()
 
         if empty:
-            log_match('', m)
+            log_match('', m, depth)
             resolve(None, None, None, text)
             yield None, text
         elif inline_kind:
-            log_match('', m)
+            log_match('', m, depth)
             kind = inline_kind
             name = inline_name or anon_name('inline-')
             data = []  # members
@@ -92,7 +92,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
             # XXX Should "parent" really be None for inline type decls?
             yield resolve(kind, data, name, text, None), text
         elif block_close:
-            log_match('', m)
+            log_match('', m, depth)
             depth -= 1
             resolve(None, None, None, text)
             # XXX This isn't great.  Calling resolve() should have
@@ -101,13 +101,13 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
             # needs to be fixed.
             yield None, text
         elif compound_bare:
-            log_match('', m)
+            log_match('', m, depth)
             yield resolve('statement', compound_bare, None, text, parent), text
         elif compound_labeled:
-            log_match('', m)
+            log_match('', m, depth)
             yield resolve('statement', compound_labeled, None, text, parent), text
         elif compound_paren:
-            log_match('', m)
+            log_match('', m, depth)
             try:
                 pos = match_paren(text)
             except ValueError:
@@ -132,7 +132,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
                     }
                 yield resolve('statement', data, None, text, parent), text
         elif block_open:
-            log_match('', m)
+            log_match('', m, depth)
             depth += 1
             if block_leading:
                 # An inline block: the last evaluated expression is used
@@ -144,10 +144,10 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
                 resolve(None, None, None, text)
                 yield None, text
         elif simple_ending:
-            log_match('', m)
+            log_match('', m, depth)
             yield resolve('statement', simple_stmt, None, text, parent), text
         elif var_ending:
-            log_match('', m)
+            log_match('', m, depth)
             kind = 'variable'
             _, name, vartype = parse_var_decl(decl)
             data = {
@@ -220,7 +220,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth):
     remainder = srcinfo.text[m.end():]
 
     if inline_kind:
-        log_match('func inline', m)
+        log_match('func inline', m, depth, depth)
         kind = inline_kind
         name = inline_name or anon_name('inline-')
         # Immediately emit a forward declaration.
@@ -249,7 +249,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth):
         yield parse_body, depth
 
     elif static_decl:
-        log_match('local variable', m)
+        log_match('local variable', m, depth, depth)
         _, name, data = parse_var_decl(static_decl)
 
         yield srcinfo.resolve('variable', data, name, parent=func), depth
@@ -266,10 +266,13 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth):
     else:
         log_match('func other', m)
         if block_open:
+            log_match('func other', None, depth, depth + 1)
             depth += 1
         elif block_close:
+            log_match('func other', None, depth, depth - 1)
             depth -= 1
         elif stmt_end:
+            log_match('func other', None, depth, depth)
             pass
         else:
             # This should be unreachable.

Tools/c-analyzer/c_parser/preprocessor/gcc.py

@@ -29,7 +29,7 @@ COMPILER_DIRECTIVE_RE = re.compile(r'''
             [^()]*
          )*
      )  # <args>
-    ( [)] [)] )?  # <closed>
+    ( [)] [)] )  # <closed>
 ''', re.VERBOSE)
 
 POST_ARGS = (
@@ -156,6 +156,7 @@ def _iter_top_include_lines(lines, topfile, cwd,
                 if name != 'pragma':
                     raise Exception(line)
             else:
+                line = re.sub(r'__inline__', 'inline', line)
                 if not raw:
                     line, partial = _strip_directives(line, partial=partial)
                 yield _common.SourceLine(

Tools/c-analyzer/cpython/_parser.py

@@ -105,9 +105,14 @@ glob	dirname
 *	./Include
 *	./Include/internal
 
-Modules/_tkinter.c	/usr/include/tcl8.6
-Modules/tkappinit.c	/usr/include/tcl
 Modules/_decimal/**/*.c	Modules/_decimal/libmpdec
+Modules/_hacl/*.c	Modules/_hacl/include
+Modules/_hacl/*.h	Modules/_hacl/include
+Modules/_tkinter.c	/usr/include/tcl8.6
+Modules/md5module.c	Modules/_hacl/include
+Modules/sha1module.c	Modules/_hacl/include
+Modules/sha2module.c	Modules/_hacl/include
+Modules/tkappinit.c	/usr/include/tcl
 Objects/stringlib/*.h	Objects
 
 # @end=tsv@
@@ -173,6 +178,7 @@ Modules/_datetimemodule.c	Py_BUILD_CORE	1
 Modules/_functoolsmodule.c	Py_BUILD_CORE	1
 Modules/_heapqmodule.c	Py_BUILD_CORE	1
 Modules/_io/*.c	Py_BUILD_CORE	1
+Modules/_io/*.h	Py_BUILD_CORE	1
 Modules/_localemodule.c	Py_BUILD_CORE	1
 Modules/_operator.c	Py_BUILD_CORE	1
 Modules/_posixsubprocess.c	Py_BUILD_CORE	1
@@ -296,6 +302,7 @@ MAX_SIZES = {
     # First match wins.
     _abs('Modules/_ctypes/ctypes.h'): (5_000, 500),
     _abs('Modules/_datetimemodule.c'): (20_000, 300),
+    _abs('Modules/_hacl/*.c'): (200_000, 500),
     _abs('Modules/posixmodule.c'): (20_000, 500),
     _abs('Modules/termios.c'): (10_000, 800),
     _abs('Modules/_testcapimodule.c'): (20_000, 400),

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -539,7 +539,7 @@ Modules/_tkinter.c	-	command_mutex	-
 Modules/_tkinter.c	-	HeadFHCD	-
 Modules/_tkinter.c	-	stdin_ready	-
 Modules/_tkinter.c	-	event_tstate	-
-Modules/_xxsubinterpretersmodule.c	-	_globals	-
+Modules/_xxinterpchannelsmodule.c	-	_globals	-
 Modules/readline.c	-	completer_word_break_characters	-
 Modules/readline.c	-	_history_length	-
 Modules/readline.c	-	should_auto_add_history	-

Tools/c-analyzer/cpython/ignored.tsv

@@ -206,6 +206,10 @@ Modules/_decimal/_decimal.c	-	invalid_signals_err	-
 Modules/_decimal/_decimal.c	-	signal_map	-
 Modules/_decimal/_decimal.c	-	ssize_constants	-
 Modules/_elementtree.c	-	ExpatMemoryHandler	-
+Modules/_hashopenssl.c	-	py_hashes	-
+Modules/_hacl/Hacl_Hash_SHA1.c	-	_h0	-
+Modules/_hacl/Hacl_Hash_MD5.c	-	_h0	-
+Modules/_hacl/Hacl_Hash_MD5.c	-	_t	-
 Modules/_io/_iomodule.c	-	static_types	-
 Modules/_io/textio.c	-	encodefuncs	-
 Modules/_io/winconsoleio.c	-	_PyWindowsConsoleIO_Type	-
@@ -218,9 +222,10 @@ Modules/_sre/sre_targets.h	-	sre_targets	-
 Modules/_sre.c	pattern_repr	flag_names	-
 Modules/_struct.c	-	bigendian_table	-
 Modules/_struct.c	-	lilendian_table	-
+Modules/_struct.c	-	native_table	-
 Modules/_tkinter.c	-	state_key	-
-Modules/_xxsubinterpretersmodule.c	-	_channelid_end_recv	-
+Modules/_xxinterpchannelsmodule.c	-	_channelid_end_recv	-
-Modules/_xxsubinterpretersmodule.c	-	_channelid_end_send	-
+Modules/_xxinterpchannelsmodule.c	-	_channelid_end_send	-
 Modules/_zoneinfo.c	-	DAYS_BEFORE_MONTH	-
 Modules/_zoneinfo.c	-	DAYS_IN_MONTH	-
 Modules/arraymodule.c	-	descriptors	-
@@ -332,12 +337,15 @@ Python/frozen.c	-	_PyImport_FrozenTest	-
 Python/getopt.c	-	longopts	-
 Python/import.c	-	_PyImport_Inittab	-
 Python/import.c	-	_PySys_ImplCacheTag	-
+Python/intrinsics.c	-	_PyIntrinsics_UnaryFunctions	-
+Python/intrinsics.c	-	_PyIntrinsics_BinaryFunctions	-
 Python/opcode_targets.h	-	opcode_targets	-
 Python/perf_trampoline.c	-	_Py_perfmap_callbacks	-
 Python/pyhash.c	-	PyHash_Func	-
 Python/pylifecycle.c	-	_C_LOCALE_WARNING	-
 Python/pylifecycle.c	-	_PyOS_mystrnicmp_hack	-
 Python/pylifecycle.c	-	_TARGET_LOCALES	-
+Python/pylifecycle.c	-	INTERPRETER_TRAMPOLINE_CODEDEF	-
 Python/pystate.c	-	initial	-
 Python/specialize.c	-	adaptive_opcodes	-
 Python/specialize.c	-	cache_requirements	-
@@ -392,8 +400,23 @@ Modules/_testbuffer.c	ndarray_memoryview_from_buffer	strides	-
 Modules/_testbuffer.c	ndarray_memoryview_from_buffer	suboffsets	-
 Modules/_testbuffer.c	ndarray_push	kwlist	-
 Modules/_testbuffer.c	staticarray_init	kwlist	-
+Modules/_testcapi/code.c	get_code_extra_index	key	-
+Modules/_testcapi/datetime.c	-	test_run_counter	-
+Modules/_testcapi/exceptions.c	-	PyRecursingInfinitelyError_Type	-
 Modules/_testcapi/heaptype.c	-	_testcapimodule	-
+Modules/_testcapi/mem.c	-	FmData	-
+Modules/_testcapi/mem.c	-	FmHook	-
+Modules/_testcapi/structmember.c	-	test_structmembersType_OldAPI	-
 Modules/_testcapi/unicode.c	-	_testcapimodule	-
+Modules/_testcapi/watchers.c	-	g_dict_watch_events	-
+Modules/_testcapi/watchers.c	-	g_dict_watchers_installed	-
+Modules/_testcapi/watchers.c	-	g_type_modified_events	-
+Modules/_testcapi/watchers.c	-	g_type_watchers_installed	-
+Modules/_testcapi/watchers.c	-	num_code_object_created_events	-
+Modules/_testcapi/watchers.c	-	num_code_object_destroyed_events	-
+Modules/_testcapi/watchers.c	-	pyfunc_watchers	-
+Modules/_testcapi/watchers.c	-	func_watcher_ids	-
+Modules/_testcapi/watchers.c	-	func_watcher_callbacks	-
 Modules/_testcapimodule.c	-	ContainerNoGC_members	-
 Modules/_testcapimodule.c	-	ContainerNoGC_type	-
 Modules/_testcapimodule.c	-	FmData	-
@@ -460,11 +483,13 @@ Modules/_testcapimodule.c	-	meth_static_methods	-
 Modules/_testcapimodule.c	-	ml	-
 Modules/_testcapimodule.c	-	str1	-
 Modules/_testcapimodule.c	-	str2	-
+Modules/_testcapimodule.c	-	test_c_thread	-
 Modules/_testcapimodule.c	-	test_members	-
 Modules/_testcapimodule.c	-	test_run_counter	-
 Modules/_testcapimodule.c	-	test_structmembersType	-
 Modules/_testcapimodule.c	-	thread_done	-
 Modules/_testcapimodule.c	-	x	-
+Modules/_testcapimodule.c	-	wait_done	-
 Modules/_testcapimodule.c	getargs_keyword_only	keywords	-
 Modules/_testcapimodule.c	getargs_keywords	keywords	-
 Modules/_testcapimodule.c	getargs_positional_only_and_keywords	keywords	-
@@ -526,6 +551,7 @@ Modules/_testmultiphase.c	-	slots_exec_unreported_exception	-
 Modules/_testmultiphase.c	-	slots_nonmodule_with_exec_slots	-
 Modules/_testmultiphase.c	-	testexport_methods	-
 Modules/_testmultiphase.c	-	uninitialized_def	-
+Modules/_testsinglephase.c	-	global_state	-
 Modules/_xxtestfuzz/_xxtestfuzz.c	-	_fuzzmodule	-
 Modules/_xxtestfuzz/_xxtestfuzz.c	-	module_methods	-
 Modules/_xxtestfuzz/fuzzer.c	-	SRE_FLAG_DEBUG	-