From 8606697f49dc58ff7e18147401ac65a09c38cf57 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Mar 2023 19:40:09 -0700 Subject: [PATCH] gh-90110: Fix the c-analyzer Tool (#102483) Some incompatible changes had gone in, and the "ignore" lists weren't properly undated. This change fixes that. It's necessary prior to enabling test_check_c_globals, which I hope to do soon. Note that this does include moving last_resort_memory_error to PyInterpreterState. https://github.com/python/cpython/issues/90110 --- Include/internal/pycore_global_objects.h | 1 + Include/internal/pycore_intrinsics.h | 4 +-- Include/internal/pycore_runtime_init.h | 6 ++++ Objects/exceptions.c | 13 +++----- Python/intrinsics.c | 4 +-- Tools/c-analyzer/c_parser/parser/_common.py | 22 ++++++++++---- .../c-analyzer/c_parser/parser/_func_body.py | 25 +++++++++------- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 3 +- Tools/c-analyzer/cpython/_parser.py | 11 +++++-- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 30 +++++++++++++++++-- 11 files changed, 86 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 30c7c4e3bbd..9957da1fc5f 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -86,6 +86,7 @@ struct _Py_interp_static_objects { // hamt_empty is here instead of global because of its weakreflist. _PyGC_Head_UNUSED _hamt_empty_gc_not_used; PyHamtObject hamt_empty; + PyBaseExceptionObject last_resort_memory_error; } singletons; }; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index deac145fff7..46a52740eb8 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -21,6 +21,6 @@ typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); -extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; -extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; +extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b54adf04761..a2cc7c87c2f 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -14,6 +14,9 @@ extern "C" { #include "pycore_obmalloc_init.h" +extern PyTypeObject _PyExc_MemoryError; + + /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ @@ -120,6 +123,9 @@ extern "C" { .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ + .last_resort_memory_error = { \ + _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \ + }, \ }, \ }, \ ._initial_thread = _PyThreadState_INIT, \ diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 976f84dbf63..a473cbdfeda 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3207,8 +3207,6 @@ SimpleExtendsException(PyExc_Exception, ReferenceError, #define MEMERRORS_SAVE 16 -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) { @@ -3216,7 +3214,9 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) struct _Py_exc_state *state = get_exc_state(); if (state->memerrors_freelist == NULL) { if (!allow_allocation) { - return Py_NewRef(&last_resort_memory_error); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return Py_NewRef( + &_Py_INTERP_SINGLETON(interp, last_resort_memory_error)); } PyObject *result = BaseException_new((PyTypeObject *)PyExc_MemoryError, args, kwds); return result; @@ -3239,8 +3239,6 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) return (PyObject *)self; } -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -3325,7 +3323,7 @@ free_preallocated_memerrors(struct _Py_exc_state *state) } -static PyTypeObject _PyExc_MemoryError = { +PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), @@ -3339,9 +3337,6 @@ static PyTypeObject _PyExc_MemoryError = { }; PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; -static PyBaseExceptionObject last_resort_memory_error = { - _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError) -}; /* * BufferError extends Exception diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 9e90ef32130..cca29d85990 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -199,7 +199,7 @@ list_to_tuple(PyThreadState* unused, PyObject *v) return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); } -instrinsic_func1 +const instrinsic_func1 _PyIntrinsics_UnaryFunctions[] = { [0] = no_intrinsic, [INTRINSIC_PRINT] = print_expr, @@ -221,7 +221,7 @@ prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs) return _PyExc_PrepReraiseStar(orig, excs); } -instrinsic_func2 +const instrinsic_func2 _PyIntrinsics_BinaryFunctions[] = { [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star, }; diff --git a/Tools/c-analyzer/c_parser/parser/_common.py b/Tools/c-analyzer/c_parser/parser/_common.py index d468d5442a9..2eacace2c00 100644 --- a/Tools/c-analyzer/c_parser/parser/_common.py +++ b/Tools/c-analyzer/c_parser/parser/_common.py @@ -7,13 +7,25 @@ from ._regexes import ( ) -def log_match(group, m): +def log_match(group, m, depth_before=None, depth_after=None): from . import _logger - text = m.group(0) - if text.startswith(('(', ')')) or text.endswith(('(', ')')): - _logger.debug(f'matched <{group}> ({text!r})') + + if m is not None: + text = m.group(0) + if text.startswith(('(', ')')) or text.endswith(('(', ')')): + _logger.debug(f'matched <{group}> ({text!r})') + else: + _logger.debug(f'matched <{group}> ({text})') + + elif depth_before is not None or depth_after is not None: + if depth_before is None: + depth_before = '???' + elif depth_after is None: + depth_after = '???' + _logger.log(1, f'depth: %s -> %s', depth_before, depth_after) + else: - _logger.debug(f'matched <{group}> ({text})') + raise NotImplementedError('this should not have been hit') ############################# diff --git a/Tools/c-analyzer/c_parser/parser/_func_body.py b/Tools/c-analyzer/c_parser/parser/_func_body.py index 42fd459e111..25f2f5807ae 100644 --- a/Tools/c-analyzer/c_parser/parser/_func_body.py +++ b/Tools/c-analyzer/c_parser/parser/_func_body.py @@ -65,11 +65,11 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): ) = m.groups() if empty: - log_match('', m) + log_match('', m, depth) resolve(None, None, None, text) yield None, text elif inline_kind: - log_match('', m) + log_match('', m, depth) kind = inline_kind name = inline_name or anon_name('inline-') data = [] # members @@ -92,7 +92,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # XXX Should "parent" really be None for inline type decls? yield resolve(kind, data, name, text, None), text elif block_close: - log_match('', m) + log_match('', m, depth) depth -= 1 resolve(None, None, None, text) # XXX This isn't great. Calling resolve() should have @@ -101,13 +101,13 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # needs to be fixed. yield None, text elif compound_bare: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_bare, None, text, parent), text elif compound_labeled: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_labeled, None, text, parent), text elif compound_paren: - log_match('', m) + log_match('', m, depth) try: pos = match_paren(text) except ValueError: @@ -132,7 +132,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): } yield resolve('statement', data, None, text, parent), text elif block_open: - log_match('', m) + log_match('', m, depth) depth += 1 if block_leading: # An inline block: the last evaluated expression is used @@ -144,10 +144,10 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): resolve(None, None, None, text) yield None, text elif simple_ending: - log_match('', m) + log_match('', m, depth) yield resolve('statement', simple_stmt, None, text, parent), text elif var_ending: - log_match('', m) + log_match('', m, depth) kind = 'variable' _, name, vartype = parse_var_decl(decl) data = { @@ -220,7 +220,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth): remainder = srcinfo.text[m.end():] if inline_kind: - log_match('func inline', m) + log_match('func inline', m, depth, depth) kind = inline_kind name = inline_name or anon_name('inline-') # Immediately emit a forward declaration. @@ -249,7 +249,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth): yield parse_body, depth elif static_decl: - log_match('local variable', m) + log_match('local variable', m, depth, depth) _, name, data = parse_var_decl(static_decl) yield srcinfo.resolve('variable', data, name, parent=func), depth @@ -266,10 +266,13 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth): else: log_match('func other', m) if block_open: + log_match('func other', None, depth, depth + 1) depth += 1 elif block_close: + log_match('func other', None, depth, depth - 1) depth -= 1 elif stmt_end: + log_match('func other', None, depth, depth) pass else: # This should be unreachable. diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 77080225379..7ef1a8afc3b 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -29,7 +29,7 @@ COMPILER_DIRECTIVE_RE = re.compile(r''' [^()]* )* ) # - ( [)] [)] )? # + ( [)] [)] ) # ''', re.VERBOSE) POST_ARGS = ( @@ -156,6 +156,7 @@ def _iter_top_include_lines(lines, topfile, cwd, if name != 'pragma': raise Exception(line) else: + line = re.sub(r'__inline__', 'inline', line) if not raw: line, partial = _strip_directives(line, partial=partial) yield _common.SourceLine( diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ab1d6257f1b..e7764165d36 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -105,9 +105,14 @@ glob dirname * ./Include * ./Include/internal -Modules/_tkinter.c /usr/include/tcl8.6 -Modules/tkappinit.c /usr/include/tcl Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_hacl/*.c Modules/_hacl/include +Modules/_hacl/*.h Modules/_hacl/include +Modules/_tkinter.c /usr/include/tcl8.6 +Modules/md5module.c Modules/_hacl/include +Modules/sha1module.c Modules/_hacl/include +Modules/sha2module.c Modules/_hacl/include +Modules/tkappinit.c /usr/include/tcl Objects/stringlib/*.h Objects # @end=tsv@ @@ -173,6 +178,7 @@ Modules/_datetimemodule.c Py_BUILD_CORE 1 Modules/_functoolsmodule.c Py_BUILD_CORE 1 Modules/_heapqmodule.c Py_BUILD_CORE 1 Modules/_io/*.c Py_BUILD_CORE 1 +Modules/_io/*.h Py_BUILD_CORE 1 Modules/_localemodule.c Py_BUILD_CORE 1 Modules/_operator.c Py_BUILD_CORE 1 Modules/_posixsubprocess.c Py_BUILD_CORE 1 @@ -296,6 +302,7 @@ MAX_SIZES = { # First match wins. _abs('Modules/_ctypes/ctypes.h'): (5_000, 500), _abs('Modules/_datetimemodule.c'): (20_000, 300), + _abs('Modules/_hacl/*.c'): (200_000, 500), _abs('Modules/posixmodule.c'): (20_000, 500), _abs('Modules/termios.c'): (10_000, 800), _abs('Modules/_testcapimodule.c'): (20_000, 400), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 2e28c50c6ff..57b8542fb46 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -539,7 +539,7 @@ Modules/_tkinter.c - command_mutex - Modules/_tkinter.c - HeadFHCD - Modules/_tkinter.c - stdin_ready - Modules/_tkinter.c - event_tstate - -Modules/_xxsubinterpretersmodule.c - _globals - +Modules/_xxinterpchannelsmodule.c - _globals - Modules/readline.c - completer_word_break_characters - Modules/readline.c - _history_length - Modules/readline.c - should_auto_add_history - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 849e20a1b0f..700ddf28518 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -206,6 +206,10 @@ Modules/_decimal/_decimal.c - invalid_signals_err - Modules/_decimal/_decimal.c - signal_map - Modules/_decimal/_decimal.c - ssize_constants - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_hashopenssl.c - py_hashes - +Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _t - Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - @@ -218,9 +222,10 @@ Modules/_sre/sre_targets.h - sre_targets - Modules/_sre.c pattern_repr flag_names - Modules/_struct.c - bigendian_table - Modules/_struct.c - lilendian_table - +Modules/_struct.c - native_table - Modules/_tkinter.c - state_key - -Modules/_xxsubinterpretersmodule.c - _channelid_end_recv - -Modules/_xxsubinterpretersmodule.c - _channelid_end_send - +Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - +Modules/_xxinterpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - Modules/arraymodule.c - descriptors - @@ -332,12 +337,15 @@ Python/frozen.c - _PyImport_FrozenTest - Python/getopt.c - longopts - Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - +Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - +Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - Python/pylifecycle.c - _TARGET_LOCALES - +Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF - Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - @@ -392,8 +400,23 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/code.c get_code_extra_index key - +Modules/_testcapi/datetime.c - test_run_counter - +Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - +Modules/_testcapi/mem.c - FmData - +Modules/_testcapi/mem.c - FmHook - +Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/unicode.c - _testcapimodule - +Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watchers_installed - +Modules/_testcapi/watchers.c - g_type_modified_events - +Modules/_testcapi/watchers.c - g_type_watchers_installed - +Modules/_testcapi/watchers.c - num_code_object_created_events - +Modules/_testcapi/watchers.c - num_code_object_destroyed_events - +Modules/_testcapi/watchers.c - pyfunc_watchers - +Modules/_testcapi/watchers.c - func_watcher_ids - +Modules/_testcapi/watchers.c - func_watcher_callbacks - Modules/_testcapimodule.c - ContainerNoGC_members - Modules/_testcapimodule.c - ContainerNoGC_type - Modules/_testcapimodule.c - FmData - @@ -460,11 +483,13 @@ Modules/_testcapimodule.c - meth_static_methods - Modules/_testcapimodule.c - ml - Modules/_testcapimodule.c - str1 - Modules/_testcapimodule.c - str2 - +Modules/_testcapimodule.c - test_c_thread - Modules/_testcapimodule.c - test_members - Modules/_testcapimodule.c - test_run_counter - Modules/_testcapimodule.c - test_structmembersType - Modules/_testcapimodule.c - thread_done - Modules/_testcapimodule.c - x - +Modules/_testcapimodule.c - wait_done - Modules/_testcapimodule.c getargs_keyword_only keywords - Modules/_testcapimodule.c getargs_keywords keywords - Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - @@ -526,6 +551,7 @@ Modules/_testmultiphase.c - slots_exec_unreported_exception - Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - Modules/_testmultiphase.c - testexport_methods - Modules/_testmultiphase.c - uninitialized_def - +Modules/_testsinglephase.c - global_state - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG -