bpo-36876: Small adjustments to the C-analyzer tool. (GH-23045)

This is a little bit of clean-up, small fixes, and additional helpers prior to building an updated & accurate list of globals to eliminate.

Tools/c-analyzer/cpython/__main__.py

@@ -31,6 +31,9 @@ def _resolve_filenames(filenames):
     return resolved
 
 
+#######################################
+# the formats
+
 def fmt_summary(analysis):
     # XXX Support sorting and grouping.
     supported = []
@@ -179,7 +182,7 @@ def cmd_data(datacmd, **kwargs):
                 analyze_resolved=_analyzer.analyze_resolved,
             )
             return _analyzer.Analysis.from_results(results)
-    else:
+    else:  # check
         known = _analyzer.read_known()
         def analyze(files, **kwargs):
             return _analyzer.iter_decls(files, **kwargs)

Tools/c-analyzer/cpython/_analyzer.py

@@ -11,9 +11,14 @@ from c_parser.info import (
     Struct,
     Member,
     FIXED_TYPE,
+)
+from c_parser.match import (
     is_type_decl,
     is_pots,
     is_funcptr,
+)
+from c_analyzer.match import (
+    is_system_type,
     is_process_global,
     is_fixed_type,
     is_immutable,
@@ -246,7 +251,7 @@ def _check_typespec(decl, typedecl, types, knowntypes):
     # Fall back to default known types.
     if is_pots(typespec):
         return None
-    elif _info.is_system_type(typespec):
+    elif is_system_type(typespec):
         return None
     elif is_funcptr(decl.vartype):
         return None

Tools/c-analyzer/cpython/_parser.py

@@ -46,10 +46,14 @@ def clean_lines(text):
 GLOBS = [
     'Include/*.h',
     'Include/internal/*.h',
+    'Modules/**/*.h',
     'Modules/**/*.c',
+    'Objects/**/*.h',
     'Objects/**/*.c',
+    'Python/**/*.h',
+    'Parser/**/*.c',
+    'Python/**/*.h',
     'Parser/**/*.c',
-    'Python/**/*.c',
 ]
 
 EXCLUDED = clean_lines('''
@@ -67,11 +71,24 @@ Modules/_scproxy.c                # SystemConfiguration/SystemConfiguration.h
 Modules/_winapi.c               # windows.h
 Modules/overlapped.c            # winsock.h
 Python/dynload_win.c            # windows.h
+Modules/expat/winconfig.h
+Python/thread_nt.h
 
 # other OS-dependent
 Python/dynload_dl.c             # dl.h
 Python/dynload_hpux.c           # dl.h
 Python/dynload_aix.c            # sys/ldr.h
+Python/thread_pthread.h
+
+# only huge constants (safe but parsing is slow)
+Modules/_ssl_data.h
+Modules/unicodedata_db.h
+Modules/unicodename_db.h
+Modules/cjkcodecs/mappings_*.h
+Objects/unicodetype_db.h
+Python/importlib.h
+Python/importlib_external.h
+Python/importlib_zipimport.h
 
 # @end=conf@
 ''')
@@ -80,6 +97,17 @@ Python/dynload_aix.c            # sys/ldr.h
 EXCLUDED += clean_lines('''
 # The tool should be able to parse these...
 
+Modules/hashlib.h
+Objects/stringlib/codecs.h
+Objects/stringlib/count.h
+Objects/stringlib/ctype.h
+Objects/stringlib/fastsearch.h
+Objects/stringlib/find.h
+Objects/stringlib/find_max_char.h
+Objects/stringlib/partition.h
+Objects/stringlib/replace.h
+Objects/stringlib/split.h
+
 Modules/_dbmmodule.c
 Modules/cjkcodecs/_codecs_*.c
 Modules/expat/xmlrole.c
@@ -134,6 +162,9 @@ Modules/_datetimemodule.c	Py_BUILD_CORE	1
 Modules/_ctypes/cfield.c	Py_BUILD_CORE	1
 Modules/_heapqmodule.c	Py_BUILD_CORE	1
 Modules/_posixsubprocess.c	Py_BUILD_CORE	1
+Objects/stringlib/codecs.h	Py_BUILD_CORE	1
+Python/ceval_gil.h	Py_BUILD_CORE	1
+Python/condvar.h	Py_BUILD_CORE	1
 
 Modules/_json.c	Py_BUILD_CORE_BUILTIN	1
 Modules/_pickle.c	Py_BUILD_CORE_BUILTIN	1
@@ -177,6 +208,12 @@ Python/Python-ast.c	PyMODINIT_FUNC	PyObject*
 Python/import.c	PyMODINIT_FUNC	PyObject*
 Modules/_testcapimodule.c	PyAPI_FUNC(RTYPE)	RTYPE
 Python/getargs.c	PyAPI_FUNC(RTYPE)	RTYPE
+Objects/stringlib/unicode_format.h	Py_LOCAL_INLINE(type)	static inline type
+
+# implied include of pymacro.h
+*/clinic/*.c.h	PyDoc_VAR(name)	static const char name[]
+*/clinic/*.c.h	PyDoc_STR(str)	str
+*/clinic/*.c.h	PyDoc_STRVAR(name,str)	PyDoc_VAR(name) = PyDoc_STR(str)
 
 # implied include of exports.h
 #Modules/_io/bytesio.c	Py_EXPORTED_SYMBOL	/* */
@@ -212,6 +249,11 @@ Modules/expat/xmlparse.c	HAVE_EXPAT_CONFIG_H	1
 Modules/expat/xmlparse.c	XML_POOR_ENTROPY	1
 Modules/_dbmmodule.c	HAVE_GDBM_DASH_NDBM_H	1
 
+# others
+Modules/sre_lib.h	LOCAL(type)	static inline type
+Modules/sre_lib.h	SRE(F)	sre_ucs2_##F
+Objects/stringlib/codecs.h	STRINGLIB_IS_UNICODE	1
+
 # @end=tsv@
 ''')[1:]