gh-124064: Make warning emitting compiler options opt-in (#124070)

Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
This commit is contained in:
Nate Ohlson 2024-09-16 12:22:30 -05:00 committed by GitHub
parent 05235e3c16
commit 44052b5f18
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 809 additions and 115 deletions

View File

@ -35,7 +35,7 @@ jobs:
path: config.cache
key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
- name: Install Homebrew dependencies
run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make
- name: Configure CPython
run: |
GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
@ -44,14 +44,27 @@ jobs:
--config-cache \
--with-pydebug \
--enable-slower-safety \
--enable-safety \
${{ inputs.free-threading && '--disable-gil' || '' }} \
--prefix=/opt/python-dev \
--with-openssl="$(brew --prefix openssl@3.0)"
- name: Build CPython
run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
run: gmake -j8
- name: Build CPython for compiler warning check
if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
- name: Display build info
run: make pythoninfo
- name: Check compiler warnings
run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
run: >-
python3 Tools/build/check_warnings.py
--compiler-output-file-path=compiler_output_macos.txt
--warning-ignore-file-path=Tools/build/.warningignore_macos
--compiler-output-type=clang
--fail-on-regression
--fail-on-improvement
--path-prefix="./"
- name: Tests
run: make test

View File

@ -67,20 +67,33 @@ jobs:
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: >-
../cpython-ro-srcdir/configure
CFLAGS="-fdiagnostics-format=json"
--config-cache
--with-pydebug
--enable-slower-safety
--enable-safety
--with-openssl=$OPENSSL_DIR
${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
- name: Build CPython out-of-tree
if: ${{ inputs.free-threading }}
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
run: make -j4
- name: Build CPython out-of-tree (for compiler warning check)
if: ${{ !inputs.free-threading}}
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt
- name: Display build info
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
- name: Check compiler warnings
run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
if: ${{ !inputs.free-threading }}
run: >-
python Tools/build/check_warnings.py
--compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt
--warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
--compiler-output-type=gcc
--fail-on-regression
--fail-on-improvement
--path-prefix="../cpython-ro-srcdir/"
- name: Remount sources writable for tests
# some tests write to srcdir, lack of pyc files slows down testing
run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw

View File

@ -0,0 +1 @@
Enable compiler options that warn of potential security vulnerabilities.

View File

@ -3,3 +3,230 @@
# Keep lines sorted lexicographically to help avoid merge conflicts.
# Format example:
# /path/to/file (number of warnings in file)
Include/internal/mimalloc/mimalloc/internal.h 4
Include/internal/pycore_backoff.h 1
Include/internal/pycore_dict.h 2
Include/internal/pycore_gc.h 1
Include/internal/pycore_long.h 2
Include/internal/pycore_object.h 4
Modules/_asynciomodule.c 3
Modules/_bisectmodule.c 2
Modules/_bz2module.c 5
Modules/_collectionsmodule.c 2
Modules/_csv.c 3
Modules/_ctypes/_ctypes.c 37
Modules/_ctypes/_ctypes_test_generated.c.h 141
Modules/_ctypes/callbacks.c 6
Modules/_ctypes/callproc.c 15
Modules/_ctypes/cfield.c 56
Modules/_ctypes/malloc_closure.c 3
Modules/_ctypes/stgdict.c 17
Modules/_cursesmodule.c 24
Modules/_datetimemodule.c 28
Modules/_dbmmodule.c 8
Modules/_decimal/_decimal.c 15
Modules/_elementtree.c 42
Modules/_functoolsmodule.c 6
Modules/_gdbmmodule.c 5
Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24
Modules/_hashopenssl.c 16
Modules/_interpchannelsmodule.c 1
Modules/_interpqueuesmodule.c 1
Modules/_io/_iomodule.c 1
Modules/_io/bufferedio.c 4
Modules/_io/bytesio.c 11
Modules/_io/fileio.c 9
Modules/_io/stringio.c 8
Modules/_io/textio.c 11
Modules/_json.c 19
Modules/_localemodule.c 3
Modules/_lzmamodule.c 10
Modules/_multiprocessing/semaphore.c 2
Modules/_operator.c 5
Modules/_pickle.c 71
Modules/_posixsubprocess.c 8
Modules/_queuemodule.c 4
Modules/_randommodule.c 3
Modules/_scproxy.c 3
Modules/_sqlite/connection.c 4
Modules/_sqlite/cursor.c 3
Modules/_sqlite/module.c 2
Modules/_sre/sre.c 18
Modules/_sre/sre_lib.h 62
Modules/_ssl.c 29
Modules/_struct.c 1
Modules/_testbuffer.c 22
Modules/_testcapi/heaptype.c 1
Modules/_testcapi/long.c 2
Modules/_testcapi/mem.c 2
Modules/_testcapi/monitoring.c 3
Modules/_testcapi/pyatomic.c 1
Modules/_testcapi/unicode.c 2
Modules/_testcapi/vectorcall.c 3
Modules/_testcapi/watchers.c 3
Modules/_testcapimodule.c 3
Modules/_testclinic.c 14
Modules/_testexternalinspection.c 8
Modules/_testinternalcapi.c 8
Modules/_testinternalcapi/pytime.c 8
Modules/_testinternalcapi/test_critical_sections.c 1
Modules/_testinternalcapi/test_lock.c 2
Modules/_testlimitedcapi/heaptype_relative.c 4
Modules/_testlimitedcapi/object.c 2
Modules/_testlimitedcapi/unicode.c 2
Modules/_threadmodule.c 2
Modules/_tkinter.c 6
Modules/_xxtestfuzz/_xxtestfuzz.c 1
Modules/_xxtestfuzz/fuzzer.c 11
Modules/_zoneinfo.c 14
Modules/arraymodule.c 32
Modules/atexitmodule.c 1
Modules/binascii.c 206
Modules/blake2module.c 6
Modules/cjkcodecs/_codecs_cn.c 1
Modules/cjkcodecs/_codecs_iso2022.c 2
Modules/cjkcodecs/_codecs_jp.c 14
Modules/cjkcodecs/_codecs_kr.c 3
Modules/cjkcodecs/cjkcodecs.h 1
Modules/cjkcodecs/multibytecodec.c 2
Modules/clinic/_testclinic.c.h 1
Modules/clinic/arraymodule.c.h 1
Modules/clinic/unicodedata.c.h 10
Modules/cmathmodule.c 1
Modules/expat/siphash.h 8
Modules/expat/xmlparse.c 45
Modules/expat/xmltok.c 17
Modules/expat/xmltok_impl.c 34
Modules/faulthandler.c 3
Modules/fcntlmodule.c 1
Modules/getpath.c 7
Modules/grpmodule.c 4
Modules/itertoolsmodule.c 7
Modules/main.c 2
Modules/mathmodule.c 15
Modules/mmapmodule.c 20
Modules/posixmodule.c 67
Modules/pwdmodule.c 4
Modules/pyexpat.c 20
Modules/readline.c 1
Modules/resource.c 3
Modules/rotatingtree.c 1
Modules/selectmodule.c 6
Modules/sha3module.c 4
Modules/signalmodule.c 1
Modules/socketmodule.c 44
Modules/syslogmodule.c 3
Modules/timemodule.c 4
Modules/unicodedata.c 28
Modules/unicodedata_db.h 1
Modules/xxsubtype.c 2
Modules/zlibmodule.c 16
Objects/abstract.c 2
Objects/bytearrayobject.c 34
Objects/bytes_methods.c 9
Objects/bytesobject.c 35
Objects/call.c 13
Objects/classobject.c 4
Objects/codeobject.c 15
Objects/descrobject.c 2
Objects/dictobject.c 28
Objects/fileobject.c 3
Objects/floatobject.c 30
Objects/frameobject.c 19
Objects/funcobject.c 1
Objects/genobject.c 5
Objects/listobject.c 43
Objects/longobject.c 46
Objects/memoryobject.c 6
Objects/methodobject.c 1
Objects/mimalloc/alloc.c 6
Objects/mimalloc/arena.c 6
Objects/mimalloc/heap.c 1
Objects/mimalloc/init.c 2
Objects/mimalloc/options.c 1
Objects/mimalloc/os.c 4
Objects/mimalloc/page-queue.c 2
Objects/mimalloc/page.c 1
Objects/mimalloc/prim/osx/../unix/prim.c 2
Objects/mimalloc/random.c 1
Objects/mimalloc/segment.c 11
Objects/mimalloc/stats.c 1
Objects/moduleobject.c 2
Objects/object.c 1
Objects/obmalloc.c 6
Objects/odictobject.c 3
Objects/rangeobject.c 10
Objects/setobject.c 13
Objects/sliceobject.c 4
Objects/stringlib/codecs.h 26
Objects/stringlib/eq.h 1
Objects/stringlib/fastsearch.h 14
Objects/stringlib/join.h 1
Objects/stringlib/replace.h 4
Objects/stringlib/repr.h 21
Objects/stringlib/transmogrify.h 5
Objects/structseq.c 14
Objects/tupleobject.c 10
Objects/typeobject.c 17
Objects/unicodectype.c 7
Objects/unicodeobject.c 113
Parser/action_helpers.c 4
Parser/lexer/buffer.c 1
Parser/lexer/lexer.c 12
Parser/parser.c 116
Parser/pegen.c 7
Parser/string_parser.c 7
Parser/tokenizer/file_tokenizer.c 8
Parser/tokenizer/helpers.c 7
Parser/tokenizer/readline_tokenizer.c 3
Programs/_freeze_module.c 1
Python/Python-ast.c 15
Python/asdl.c 3
Python/assemble.c 7
Python/ast_opt.c 7
Python/bltinmodule.c 9
Python/bootstrap_hash.c 4
Python/ceval.c 8
Python/ceval_gil.c 2
Python/codecs.c 32
Python/codegen.c 6
Python/compile.c 2
Python/context.c 1
Python/crossinterp.c 2
Python/crossinterp_data_lookup.h 1
Python/dtoa.c 34
Python/errors.c 1
Python/fileutils.c 7
Python/flowgraph.c 8
Python/formatter_unicode.c 7
Python/frame.c 4
Python/gc.c 8
Python/generated_cases.c.h 35
Python/getargs.c 11
Python/import.c 5
Python/initconfig.c 11
Python/instrumentation.c 31
Python/intrinsics.c 1
Python/legacy_tracing.c 3
Python/lock.c 4
Python/marshal.c 11
Python/modsupport.c 3
Python/mystrtoul.c 4
Python/pathconfig.c 1
Python/preconfig.c 2
Python/pyarena.c 1
Python/pyhash.c 2
Python/pylifecycle.c 7
Python/pystate.c 6
Python/pystrhex.c 19
Python/pystrtod.c 3
Python/qsbr.c 2
Python/specialize.c 10
Python/suggestions.c 12
Python/symtable.c 18
Python/sysmodule.c 2
Python/thread_pthread.h 1
Python/traceback.c 6
Python/tracemalloc.c 6

View File

@ -3,3 +3,263 @@
# Keep lines sorted lexicographically to help avoid merge conflicts.
# Format example:
# /path/to/file (number of warnings in file)
/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/evp.h 2
/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/ssl.h 4
/usr/include/tcl8.6/tclTomMathDecls.h 1
Include/cpython/bytearrayobject.h 1
Include/cpython/bytesobject.h 3
Include/cpython/dictobject.h 2
Include/cpython/listobject.h 1
Include/cpython/pyctype.h 2
Include/cpython/tupleobject.h 1
Include/cpython/unicodeobject.h 7
Include/internal/mimalloc/mimalloc/internal.h 4
Include/internal/mimalloc/mimalloc/types.h 2
Include/internal/pycore_asdl.h 1
Include/internal/pycore_backoff.h 3
Include/internal/pycore_blocks_output_buffer.h 1
Include/internal/pycore_dict.h 2
Include/internal/pycore_gc.h 1
Include/internal/pycore_gc.h 1
Include/internal/pycore_interp.h 1
Include/internal/pycore_list.h 1
Include/internal/pycore_long.h 3
Include/internal/pycore_object.h 4
Include/internal/pycore_obmalloc.h 1
Include/internal/pycore_pymath.h 1
Include/internal/pycore_runtime_init.h 1
Include/longobject.h 1
Include/object.h 4
Include/opcode_ids.h 1
Include/pymacro.h 4
Include/pymath.h 1
Include/pymem.h 2
Include/pyport.h 2
Modules/_asynciomodule.c 3
Modules/_bisectmodule.c 4
Modules/_bz2module.c 5
Modules/_collectionsmodule.c 2
Modules/_csv.c 2
Modules/_ctypes/_ctypes.c 53
Modules/_ctypes/_ctypes_test.c 7
Modules/_ctypes/_ctypes_test_generated.c.h 2
Modules/_ctypes/callbacks.c 3
Modules/_ctypes/callproc.c 13
Modules/_ctypes/cfield.c 33
Modules/_ctypes/stgdict.c 17
Modules/_cursesmodule.c 27
Modules/_datetimemodule.c 38
Modules/_datetimemodule.c 38
Modules/_dbmmodule.c 7
Modules/_decimal/_decimal.c 19
Modules/_elementtree.c 37
Modules/_functoolsmodule.c 6
Modules/_gdbmmodule.c 4
Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4
Modules/_hashopenssl.c 13
Modules/_io/_iomodule.c 1
Modules/_io/bufferedio.c 15
Modules/_io/bytesio.c 14
Modules/_io/fileio.c 9
Modules/_io/stringio.c 8
Modules/_io/textio.c 17
Modules/_json.c 19
Modules/_localemodule.c 2
Modules/_lsprof.c 5
Modules/_lzmamodule.c 6
Modules/_multiprocessing/posixshmem.c 1
Modules/_multiprocessing/semaphore.c 1
Modules/_operator.c 5
Modules/_pickle.c 73
Modules/_posixsubprocess.c 11
Modules/_queuemodule.c 4
Modules/_randommodule.c 3
Modules/_sqlite/connection.c 5
Modules/_sqlite/cursor.c 3
Modules/_sqlite/module.c 2
Modules/_sre/sre.c 14
Modules/_sre/sre_lib.h 25
Modules/_ssl.c 26
Modules/_struct.c 3
Modules/_testbuffer.c 27
Modules/_testcapi/bytes.c 1
Modules/_testcapi/heaptype.c 1
Modules/_testcapi/long.c 2
Modules/_testcapi/mem.c 2
Modules/_testcapi/monitoring.c 3
Modules/_testcapi/pyatomic.c 4
Modules/_testcapi/pyatomic.c 4
Modules/_testcapi/unicode.c 1
Modules/_testcapi/vectorcall.c 3
Modules/_testcapi/watchers.c 3
Modules/_testcapimodule.c 1
Modules/_testclinic.c 14
Modules/_testclinic.c 14
Modules/_testexternalinspection.c 7
Modules/_testinternalcapi.c 10
Modules/_testinternalcapi/test_critical_sections.c 1
Modules/_testinternalcapi/test_lock.c 4
Modules/_testlimitedcapi/heaptype_relative.c 3
Modules/_testlimitedcapi/object.c 2
Modules/_testlimitedcapi/unicode.c 1
Modules/_testmultiphase.c 1
Modules/_tkinter.c 8
Modules/_xxtestfuzz/_xxtestfuzz.c 1
Modules/_xxtestfuzz/fuzzer.c 13
Modules/_zoneinfo.c 17
Modules/arraymodule.c 48
Modules/binascii.c 208
Modules/blake2module.c 8
Modules/cjkcodecs/_codecs_iso2022.c 1
Modules/cjkcodecs/_codecs_jp.c 17
Modules/cjkcodecs/_codecs_kr.c 7
Modules/cjkcodecs/alg_jisx0201.h 2
Modules/cjkcodecs/cjkcodecs.h 1
Modules/cjkcodecs/multibytecodec.c 12
Modules/expat/pyexpatns.h 3
Modules/expat/siphash.h 1
Modules/expat/xmlparse.c 43
Modules/expat/xmltok.c 15
Modules/expat/xmltok.c 15
Modules/expat/xmltok_impl.c 8
Modules/faulthandler.c 5
Modules/fcntlmodule.c 6
Modules/getpath.c 7
Modules/grpmodule.c 4
Modules/itertoolsmodule.c 4
Modules/main.c 2
Modules/mathmodule.c 14
Modules/mmapmodule.c 22
Modules/mmapmodule.c 22
Modules/posixmodule.c 79
Modules/pwdmodule.c 4
Modules/pyexpat.c 10
Modules/readline.c 1
Modules/resource.c 4
Modules/rotatingtree.c 2
Modules/selectmodule.c 1
Modules/sha3module.c 4
Modules/signalmodule.c 3
Modules/socketmodule.c 75
Modules/syslogmodule.c 3
Modules/termios.c 1
Modules/timemodule.c 10
Modules/unicodedata.c 24
Modules/unicodedata_db.h 1
Modules/zlibmodule.c 24
Objects/abstract.c 6
Objects/bytearrayobject.c 42
Objects/bytes_methods.c 4
Objects/bytesobject.c 45
Objects/call.c 12
Objects/classobject.c 4
Objects/codeobject.c 19
Objects/descrobject.c 2
Objects/dictobject.c 31
Objects/fileobject.c 3
Objects/floatobject.c 10
Objects/frameobject.c 16
Objects/funcobject.c 1
Objects/genobject.c 3
Objects/listobject.c 38
Objects/longobject.c 47
Objects/memoryobject.c 12
Objects/methodobject.c 1
Objects/mimalloc/alloc.c 6
Objects/mimalloc/arena.c 6
Objects/mimalloc/heap.c 2
Objects/mimalloc/init.c 2
Objects/mimalloc/options.c 4
Objects/mimalloc/os.c 4
Objects/mimalloc/page-queue.c 2
Objects/mimalloc/page.c 2
Objects/mimalloc/prim/unix/prim.c 6
Objects/mimalloc/random.c 1
Objects/mimalloc/segment.c 11
Objects/mimalloc/stats.c 5
Objects/moduleobject.c 4
Objects/object.c 1
Objects/obmalloc.c 6
Objects/odictobject.c 6
Objects/rangeobject.c 10
Objects/setobject.c 13
Objects/sliceobject.c 2
Objects/stringlib/codecs.h 12
Objects/stringlib/eq.h 1
Objects/stringlib/fastsearch.h 8
Objects/stringlib/join.h 3
Objects/stringlib/replace.h 4
Objects/stringlib/repr.h 21
Objects/stringlib/transmogrify.h 26
Objects/structseq.c 10
Objects/tupleobject.c 8
Objects/typeobject.c 38
Objects/unicodectype.c 7
Objects/unicodeobject.c 135
Parser/action_helpers.c 3
Parser/lexer/buffer.c 1
Parser/lexer/lexer.c 14
Parser/parser.c 116
Parser/pegen.c 8
Parser/string_parser.c 7
Parser/tokenizer/file_tokenizer.c 9
Parser/tokenizer/helpers.c 7
Parser/tokenizer/readline_tokenizer.c 4
Python/assemble.c 11
Python/ast_opt.c 5
Python/bltinmodule.c 8
Python/bootstrap_hash.c 7
Python/ceval.c 8
Python/ceval_gil.c 2
Python/codecs.c 28
Python/codegen.c 6
Python/compile.c 2
Python/context.c 1
Python/crossinterp.c 2
Python/crossinterp_data_lookup.h 1
Python/dtoa.c 30
Python/errors.c 1
Python/fileutils.c 11
Python/flowgraph.c 7
Python/formatter_unicode.c 6
Python/frame.c 3
Python/gc.c 9
Python/gc.c 9
Python/generated_cases.c.h 27
Python/generated_cases.c.h 27
Python/getargs.c 7
Python/hashtable.c 1
Python/import.c 6
Python/import.c 7
Python/initconfig.c 11
Python/instrumentation.c 43
Python/intrinsics.c 1
Python/legacy_tracing.c 3
Python/lock.c 4
Python/marshal.c 16
Python/modsupport.c 3
Python/mystrtoul.c 4
Python/pathconfig.c 1
Python/perf_jit_trampoline.c 32
Python/perf_trampoline.c 12
Python/preconfig.c 2
Python/pyarena.c 1
Python/pyhash.c 4
Python/pylifecycle.c 3
Python/pystate.c 4
Python/pystrhex.c 15
Python/pystrtod.c 12
Python/pytime.c 2
Python/qsbr.c 2
Python/specialize.c 9
Python/suggestions.c 12
Python/symtable.c 15
Python/sysmodule.c 2
Python/thread.c 1
Python/thread_pthread.h 6
Python/traceback.c 6
Python/tracemalloc.c 6

View File

@ -1,105 +1,68 @@
"""
Parses compiler output with -fdiagnostics-format=json and checks that warnings
Parses compiler output from Clang or GCC and checks that warnings
exist only in files that are expected to have warnings.
"""
import argparse
from collections import defaultdict
import json
import re
import sys
from pathlib import Path
from typing import NamedTuple
class FileWarnings(NamedTuple):
name: str
count: int
def extract_warnings_from_compiler_output_clang(
def extract_warnings_from_compiler_output(
compiler_output: str,
compiler_output_type: str,
path_prefix: str = "",
) -> list[dict]:
"""
Extracts warnings from the compiler output when using clang
Extracts warnings from the compiler output based on compiler
output type. Removes path prefix from file paths if provided.
Compatible with GCC and Clang compiler output.
"""
# Regex to find warnings in the compiler output
clang_warning_regex = re.compile(
# Choose pattern and compile regex for particular compiler output
if compiler_output_type == "gcc":
regex_pattern = (
r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
r"(?P<message>.*?)(?: (?P<option>\[-[^\]]+\]))?$"
)
elif compiler_output_type == "clang":
regex_pattern = (
r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
r"(?P<message>.*) (?P<option>\[-[^\]]+\])$"
)
compiled_regex = re.compile(regex_pattern)
compiler_warnings = []
for line in compiler_output.splitlines():
if match := clang_warning_regex.match(line):
for i, line in enumerate(compiler_output.splitlines(), start=1):
if match := compiled_regex.match(line):
try:
compiler_warnings.append(
{
"file": match.group("file"),
"file": match.group("file").removeprefix(path_prefix),
"line": match.group("line"),
"column": match.group("column"),
"message": match.group("message"),
"option": match.group("option").lstrip("[").rstrip("]"),
}
)
return compiler_warnings
def extract_warnings_from_compiler_output_json(
compiler_output: str,
) -> list[dict]:
"""
Extracts warnings from the compiler output when using
-fdiagnostics-format=json.
Compiler output as a whole is not a valid json document,
but includes many json objects and may include other output
that is not json.
"""
# Regex to find json arrays at the top level of the file
# in the compiler output
json_arrays = re.findall(r"\[(?:[^[\]]|\[[^]]*])*]", compiler_output)
compiler_warnings = []
for array in json_arrays:
try:
json_data = json.loads(array)
json_objects_in_array = [entry for entry in json_data]
warning_list = [
entry
for entry in json_objects_in_array
if entry.get("kind") == "warning"
]
for warning in warning_list:
locations = warning["locations"]
for location in locations:
for key in ["caret", "start", "end"]:
if key in location:
compiler_warnings.append(
{
# Remove leading current directory if present
"file": location[key]["file"].lstrip("./"),
"line": location[key]["line"],
"column": location[key]["column"],
"message": warning["message"],
"option": warning["option"],
}
)
# Found a caret, start, or end in location so
# break out completely to address next warning
break
else:
continue
break
except json.JSONDecodeError:
continue # Skip malformed JSON
except:
print(f"Error parsing compiler output. Unable to extract warning on line {i}:\n{line}")
sys.exit(1)
return compiler_warnings
def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]:
"""
Returns a dictionary where the key is the file and the data is the warnings
in that file. Does not include duplicate warnings for a file from list of
provided warnings.
Returns a dictionary where the key is the file and the data is the
warnings in that file. Does not include duplicate warnings for a
file from list of provided warnings.
"""
warnings_by_file = defaultdict(list)
warnings_added = set()
@ -124,22 +87,28 @@ def get_unexpected_warnings(
are associated with a file that is not found in the list of files
with expected warnings
"""
unexpected_warnings = []
unexpected_warnings = {}
for file in files_with_warnings.keys():
found_file_in_ignore_list = False
for ignore_file in files_with_expected_warnings:
if file == ignore_file.name:
if len(files_with_warnings[file]) > ignore_file.count:
unexpected_warnings.extend(files_with_warnings[file])
unexpected_warnings[file] = (files_with_warnings[file], ignore_file.count)
found_file_in_ignore_list = True
break
if not found_file_in_ignore_list:
unexpected_warnings.extend(files_with_warnings[file])
unexpected_warnings[file] = (files_with_warnings[file], 0)
if unexpected_warnings:
print("Unexpected warnings:")
for warning in unexpected_warnings:
for file in unexpected_warnings:
print(
f"{file} expected {unexpected_warnings[file][1]} warnings,"
f" found {len(unexpected_warnings[file][0])}"
)
for warning in unexpected_warnings[file][0]:
print(warning)
return 1
return 0
@ -156,14 +125,14 @@ def get_unexpected_improvements(
unexpected_improvements = []
for file in files_with_expected_warnings:
if file.name not in files_with_warnings.keys():
unexpected_improvements.append(file)
unexpected_improvements.append((file.name, file.count, 0))
elif len(files_with_warnings[file.name]) < file.count:
unexpected_improvements.append(file)
unexpected_improvements.append((file.name, file.count, len(files_with_warnings[file.name])))
if unexpected_improvements:
print("Unexpected improvements:")
for file in unexpected_improvements:
print(file.name)
print(f"{file[0]} expected {file[1]} warnings, found {file[2]}")
return 1
return 0
@ -204,8 +173,15 @@ def main(argv: list[str] | None = None) -> int:
"--compiler-output-type",
type=str,
required=True,
choices=["json", "clang"],
help="Type of compiler output file (json or clang)",
choices=["gcc", "clang"],
help="Type of compiler output file (GCC or Clang)",
)
parser.add_argument(
"-p",
"--path-prefix",
type=str,
help="Path prefix to remove from the start of file paths"
" in compiler output",
)
args = parser.parse_args(argv)
@ -241,7 +217,9 @@ def main(argv: list[str] | None = None) -> int:
# where the first element is the file name and the second element
# is the number of warnings expected in that file
files_with_expected_warnings = {
FileWarnings(file.strip().split()[0], int(file.strip().split()[1]))
FileWarnings(
file.strip().split()[0], int(file.strip().split()[1])
)
for file in clean_files
if file.strip() and not file.startswith("#")
}
@ -249,13 +227,10 @@ def main(argv: list[str] | None = None) -> int:
with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f:
compiler_output_file_contents = f.read()
if args.compiler_output_type == "json":
warnings = extract_warnings_from_compiler_output_json(
compiler_output_file_contents
)
elif args.compiler_output_type == "clang":
warnings = extract_warnings_from_compiler_output_clang(
compiler_output_file_contents
warnings = extract_warnings_from_compiler_output(
compiler_output_file_contents,
args.compiler_output_type,
args.path_prefix
)
files_with_warnings = get_warnings_by_file(warnings)
@ -272,6 +247,11 @@ def main(argv: list[str] | None = None) -> int:
if args.fail_on_improvement:
exit_code |= status
print(
"For information about this tool and its configuration"
" visit https://devguide.python.org/development-tools/warnings/"
)
return exit_code

215
configure generated vendored
View File

@ -1829,7 +1829,7 @@ Optional Features:
(default is no)
--enable-bolt enable usage of the llvm-bolt post-link optimizer
(default is no)
--disable-safety disable usage of the security compiler options with
--enable-safety enable usage of the security compiler options with
no performance overhead
--enable-slower-safety enable usage of the security compiler options with
performance overhead
@ -9671,25 +9671,25 @@ fi
# Enable flags that warn and protect for potential security vulnerabilities.
# These flags should be enabled by default for all builds.
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-safety" >&5
printf %s "checking for --disable-safety... " >&6; }
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-safety" >&5
printf %s "checking for --enable-safety... " >&6; }
# Check whether --enable-safety was given.
if test ${enable_safety+y}
then :
enableval=$enable_safety; if test "x$enable_safety" = xyes
enableval=$enable_safety; if test "x$disable_safety" = xyes
then :
disable_safety=no
enable_safety=no
else $as_nop
disable_safety=yes
enable_safety=yes
fi
else $as_nop
disable_safety=no
enable_safety=no
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_safety" >&5
printf "%s\n" "$disable_safety" >&6; }
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_safety" >&5
printf "%s\n" "$enable_safety" >&6; }
if test "$disable_safety" = "no"
if test "$enable_safety" = "yes"
then
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fstack-protector-strong" >&5
printf %s "checking whether C compiler accepts -fstack-protector-strong... " >&6; }
@ -9767,6 +9767,201 @@ then :
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5
printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;}
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wconversion" >&5
printf %s "checking whether C compiler accepts -Wconversion... " >&6; }
if test ${ax_cv_check_cflags__Werror__Wconversion+y}
then :
printf %s "(cached) " >&6
else $as_nop
ax_check_save_flags=$CFLAGS
CFLAGS="$CFLAGS -Werror -Wconversion"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
int
main (void)
{
;
return 0;
}
_ACEOF
if ac_fn_c_try_compile "$LINENO"
then :
ax_cv_check_cflags__Werror__Wconversion=yes
else $as_nop
ax_cv_check_cflags__Werror__Wconversion=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
CFLAGS=$ax_check_save_flags
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wconversion" >&5
printf "%s\n" "$ax_cv_check_cflags__Werror__Wconversion" >&6; }
if test "x$ax_cv_check_cflags__Werror__Wconversion" = xyes
then :
CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wconversion not supported" >&5
printf "%s\n" "$as_me: WARNING: -Wconversion not supported" >&2;}
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5
printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... " >&6; }
if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y}
then :
printf %s "(cached) " >&6
else $as_nop
ax_check_save_flags=$CFLAGS
CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
int
main (void)
{
;
return 0;
}
_ACEOF
if ac_fn_c_try_compile "$LINENO"
then :
ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes
else $as_nop
ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
CFLAGS=$ax_check_save_flags
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5
printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; }
if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes
then :
CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5
printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;}
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Werror=format-security" >&5
printf %s "checking whether C compiler accepts -Werror=format-security... " >&6; }
if test ${ax_cv_check_cflags__Werror__Werror_format_security+y}
then :
printf %s "(cached) " >&6
else $as_nop
ax_check_save_flags=$CFLAGS
CFLAGS="$CFLAGS -Werror -Werror=format-security"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
int
main (void)
{
;
return 0;
}
_ACEOF
if ac_fn_c_try_compile "$LINENO"
then :
ax_cv_check_cflags__Werror__Werror_format_security=yes
else $as_nop
ax_cv_check_cflags__Werror__Werror_format_security=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
CFLAGS=$ax_check_save_flags
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Werror_format_security" >&5
printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; }
if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes
then :
CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5
printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;}
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5
printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; }
if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y}
then :
printf %s "(cached) " >&6
else $as_nop
ax_check_save_flags=$CFLAGS
CFLAGS="$CFLAGS -Werror -Wbidi-chars=any"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
int
main (void)
{
;
return 0;
}
_ACEOF
if ac_fn_c_try_compile "$LINENO"
then :
ax_cv_check_cflags__Werror__Wbidi_chars_any=yes
else $as_nop
ax_cv_check_cflags__Werror__Wbidi_chars_any=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
CFLAGS=$ax_check_save_flags
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5
printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; }
if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes
then :
CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5
printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;}
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wall" >&5
printf %s "checking whether C compiler accepts -Wall... " >&6; }
if test ${ax_cv_check_cflags__Werror__Wall+y}
then :
printf %s "(cached) " >&6
else $as_nop
ax_check_save_flags=$CFLAGS
CFLAGS="$CFLAGS -Werror -Wall"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
int
main (void)
{
;
return 0;
}
_ACEOF
if ac_fn_c_try_compile "$LINENO"
then :
ax_cv_check_cflags__Werror__Wall=yes
else $as_nop
ax_cv_check_cflags__Werror__Wall=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
CFLAGS=$ax_check_save_flags
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wall" >&5
printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; }
if test "x$ax_cv_check_cflags__Werror__Wall" = xyes
then :
CFLAGS_NODIST="$CFLAGS_NODIST -Wall"
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5
printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;}
fi
fi

View File

@ -2500,16 +2500,21 @@ AS_VAR_IF([with_strict_overflow], [yes],
# Enable flags that warn and protect for potential security vulnerabilities.
# These flags should be enabled by default for all builds.
AC_MSG_CHECKING([for --disable-safety])
AC_MSG_CHECKING([for --enable-safety])
AC_ARG_ENABLE([safety],
[AS_HELP_STRING([--disable-safety], [disable usage of the security compiler options with no performance overhead])],
[AS_VAR_IF([enable_safety], [yes], [disable_safety=no], [disable_safety=yes])], [disable_safety=no])
AC_MSG_RESULT([$disable_safety])
[AS_HELP_STRING([--enable-safety], [enable usage of the security compiler options with no performance overhead])],
[AS_VAR_IF([disable_safety], [yes], [enable_safety=no], [enable_safety=yes])], [enable_safety=no])
AC_MSG_RESULT([$enable_safety])
if test "$disable_safety" = "no"
if test "$enable_safety" = "yes"
then
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wconversion], [CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"], [AC_MSG_WARN([-Wconversion not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"], [AC_MSG_WARN([-Wall not supported])], [-Werror])
fi
AC_MSG_CHECKING([for --enable-slower-safety])