tools/check_python_h_first.py - main - niceSOFT/numpy

tools/check_python_h_first.py
254 строки8 KB
DWesl
BUG: Include python-including headers first (#29281)
18 сен 2025, 11:04
18 сен 2025, 11:0422cd369
#!/usr/bin/env python
"""Check that Python.h is included before any stdlib headers.

May be a bit overzealous, but it should get the job done.
"""
import argparse
import fnmatch
import os.path
import re
import subprocess
import sys

from get_submodule_paths import get_submodule_paths

HEADER_PATTERN = re.compile(
    r'^\s*#\s*include\s*[<"]((?:\w+/)*\w+(?:\.h[hp+]{0,2})?)[>"]\s*$'
)

PYTHON_INCLUDING_HEADERS = [
    "Python.h",
    # This isn't all of Python.h, but it is the visibility macros
    "pyconfig.h",
    "numpy/npy_common.h",
    "numpy/npy_math.h",
    "numpy/arrayobject.h",
    "numpy/ndarrayobject.h",
    "numpy/ndarraytypes.h",
    "numpy/random/distributions.h",
    "npy_sort.h",
    "npy_config.h",
    "common.h",
    "npy_cpu_features.h",
    # Boost::Python
    "boost/python.hpp",
]
LEAF_HEADERS = [
    "numpy/numpyconfig.h",
    "numpy/npy_os.h",
    "numpy/npy_cpu.h",
    "numpy/utils.h",
]

C_CPP_EXTENSIONS = (".c", ".h", ".cpp", ".hpp", ".cc", ".hh", ".cxx", ".hxx")
# check against list in diff_files

PARSER = argparse.ArgumentParser(description=__doc__)
PARSER.add_argument(
    "files",
    nargs="*",
    help="Lint these files or directories; use **/*.c to lint all files\n"
    "Expects relative paths",
)


def check_python_h_included_first(name_to_check: str) -> int:
    """Check that the passed file includes Python.h first if it does at all.

    Perhaps overzealous, but that should work around concerns with
    recursion.

    Parameters
    ----------
    name_to_check : str
        The name of the file to check.

    Returns
    -------
    int
        The number of headers before Python.h
    """
    included_python = False
    included_non_python_header = []
    warned_python_construct = False
    basename_to_check = os.path.basename(name_to_check)
    in_comment = False
    includes_headers = False
    with open(name_to_check) as in_file:
        for i, line in enumerate(in_file, 1):
            # Very basic comment parsing
            # Assumes /*...*/ comments are on their own lines
            if "/*" in line:
                if "*/" not in line:
                    in_comment = True
                # else-branch could use regex to remove comment and continue
                continue
            if in_comment:
                if "*/" in line:
                    in_comment = False
                continue
            line = line.split("//", 1)[0].strip()
            match = HEADER_PATTERN.match(line)
            if match:
                includes_headers = True
                this_header = match.group(1)
                if this_header in PYTHON_INCLUDING_HEADERS:
                    if included_non_python_header and not included_python:
                        # Headers before python-including header
                        print(
                            f"Header before Python.h in file {name_to_check:s}\n"
                            f"Python.h on line {i:d}, other header(s) on line(s)"
                            f" {included_non_python_header}",
                            file=sys.stderr,
                        )
                    # else:  # no headers before python-including header
                    included_python = True
                    PYTHON_INCLUDING_HEADERS.append(basename_to_check)
                    if os.path.dirname(name_to_check).endswith("include/numpy"):
                        PYTHON_INCLUDING_HEADERS.append(f"numpy/{basename_to_check:s}")
                    # We just found out where Python.h comes in this file
                    break
                elif this_header in LEAF_HEADERS:
                    # This header is just defines, so it won't include
                    # the system headers that cause problems
                    continue
                elif not included_python and (
                    "numpy/" in this_header
                    and this_header not in LEAF_HEADERS
                    or "python" in this_header.lower()
                ):
                    print(
                        f"Python.h not included before python-including header "
                        f"in file {name_to_check:s}\n"
                        f"{this_header:s} on line {i:d}",
                        file=sys.stderr,
                    )
                    included_python = True
                    PYTHON_INCLUDING_HEADERS.append(basename_to_check)
                elif not included_python and this_header not in LEAF_HEADERS:
                    included_non_python_header.append(i)
            elif (
                not included_python
                and not warned_python_construct
                and ".h" not in basename_to_check
            ) and ("py::" in line or "PYBIND11_" in line):
                print(
                    "Python-including header not used before python constructs "
                    f"in file {name_to_check:s}\nConstruct on line {i:d}",
                    file=sys.stderr,
                )
                warned_python_construct = True
    if not includes_headers:
        LEAF_HEADERS.append(basename_to_check)
    return included_python and len(included_non_python_header)


def sort_order(path: str) -> tuple[int, str]:
    if "include/numpy" in path:
        # Want to process numpy/*.h first, to work out which of those
        # include Python.h directly
        priority = 0x00
    elif "h" in os.path.splitext(path)[1].lower():
        # Then other headers, which tend to include numpy/*.h
        priority = 0x10
    else:
        # Source files after headers, to give the best chance of
        # properly checking whether they include Python.h
        priority = 0x20
    if "common" in path:
        priority -= 8
    path_basename = os.path.basename(path)
    if path_basename.startswith("npy_"):
        priority -= 4
    elif path_basename.startswith("npy"):
        priority -= 3
    elif path_basename.startswith("np"):
        priority -= 2
    if "config" in path_basename:
        priority -= 1
    return priority, path


def process_files(file_list: list[str]) -> int:
    n_out_of_order = 0
    submodule_paths = get_submodule_paths()
    root_directory = os.path.dirname(os.path.dirname(__file__))
    for name_to_check in sorted(file_list, key=sort_order):
        name_to_check = os.path.join(root_directory, name_to_check)
        if any(submodule_path in name_to_check for submodule_path in submodule_paths):
            continue
        if ".dispatch." in name_to_check:
            continue
        try:
            n_out_of_order += check_python_h_included_first(name_to_check)
        except UnicodeDecodeError:
            print(f"File {name_to_check:s} not utf-8", sys.stdout)
    return n_out_of_order


def find_c_cpp_files(root: str) -> list[str]:

    result = []

    for dirpath, dirnames, filenames in os.walk(root):
        # I'm assuming other people have checked boost
        for name in ("build", ".git", "boost"):
            try:
                dirnames.remove(name)
            except ValueError:
                pass
        for name in fnmatch.filter(dirnames, "*.p"):
            dirnames.remove(name)
        result.extend(
            [
                os.path.join(dirpath, name)
                for name in filenames
                if os.path.splitext(name)[1].lower() in C_CPP_EXTENSIONS
            ]
        )
    # Check the headers before the source files
    result.sort(key=lambda path: "h" in os.path.splitext(path)[1], reverse=True)
    return result


def diff_files(sha: str) -> list[str]:
    """Find the diff since the given SHA.

    Adapted from lint.py
    """
    res = subprocess.run(
        [
            "git",
            "diff",
            "--name-only",
            "--diff-filter=ACMR",
            "-z",
            sha,
            "--",
            # Check against C_CPP_EXTENSIONS
            "*.[chCH]",
            "*.[ch]pp",
            "*.[ch]xx",
            "*.cc",
            "*.hh",
        ],
        stdout=subprocess.PIPE,
        encoding="utf-8",
    )
    res.check_returncode()
    return [f for f in res.stdout.split("\0") if f]


if __name__ == "__main__":
    args = PARSER.parse_args()

    if len(args.files) == 0:
        files = find_c_cpp_files("numpy")
    else:
        files = args.files
        if len(files) == 1 and os.path.isdir(files[0]):
            files = find_c_cpp_files(files[0])

    # See which of the headers include Python.h and add them to the list
    n_out_of_order = process_files(files)
    sys.exit(n_out_of_order)