Tools/clinic/libclinic/block_parser.py - main - githubmirror/cpython

Tools/clinic/libclinic/block_parser.py
256 строк9 KB
Victor Stinner
gh-113317, AC: Add libclinic.block_parser module (#116819)
14 мар 2024, 19:11
14 мар 2024, 19:11b54d7c8
from __future__ import annotations
import collections
import dataclasses as dc
import re
import shlex
from typing import Any

import libclinic
from libclinic import fail, ClinicError
from libclinic.language import Language
from libclinic.function import (
    Module, Class, Function)


@dc.dataclass(slots=True, repr=False)
class Block:
    r"""
    Represents a single block of text embedded in
    another file.  If dsl_name is None, the block represents
    verbatim text, raw original text from the file, in
    which case "input" will be the only non-false member.
    If dsl_name is not None, the block represents a Clinic
    block.

    input is always str, with embedded \n characters.
    input represents the original text from the file;
    if it's a Clinic block, it is the original text with
    the body_prefix and redundant leading whitespace removed.

    dsl_name is either str or None.  If str, it's the text
    found on the start line of the block between the square
    brackets.

    signatures is a list.
    It may only contain clinic.Module, clinic.Class, and
    clinic.Function objects.  At the moment it should
    contain at most one of each.

    output is either str or None.  If str, it's the output
    from this block, with embedded '\n' characters.

    indent is a str.  It's the leading whitespace
    that was found on every line of input.  (If body_prefix is
    not empty, this is the indent *after* removing the
    body_prefix.)

    "indent" is different from the concept of "preindent"
    (which is not stored as state on Block objects).
    "preindent" is the whitespace that
    was found in front of every line of input *before* the
    "body_prefix" (see the Language object).  If body_prefix
    is empty, preindent must always be empty too.

    To illustrate the difference between "indent" and "preindent":

    Assume that '_' represents whitespace.
    If the block processed was in a Python file, and looked like this:
      ____#/*[python]
      ____#__for a in range(20):
      ____#____print(a)
      ____#[python]*/
    "preindent" would be "____" and "indent" would be "__".

    """
    input: str
    dsl_name: str | None = None
    signatures: list[Module | Class | Function] = dc.field(default_factory=list)
    output: Any = None  # TODO: Very dynamic; probably untypeable in its current form?
    indent: str = ''

    def __repr__(self) -> str:
        dsl_name = self.dsl_name or "text"
        def summarize(s: object) -> str:
            s = repr(s)
            if len(s) > 30:
                return s[:26] + "..." + s[0]
            return s
        parts = (
            repr(dsl_name),
            f"input={summarize(self.input)}",
            f"output={summarize(self.output)}"
        )
        return f"<clinic.Block {' '.join(parts)}>"


class BlockParser:
    """
    Block-oriented parser for Argument Clinic.
    Iterator, yields Block objects.
    """

    def __init__(
            self,
            input: str,
            language: Language,
            *,
            verify: bool = True
    ) -> None:
        """
        "input" should be a str object
        with embedded \n characters.

        "language" should be a Language object.
        """
        language.validate()

        self.input = collections.deque(reversed(input.splitlines(keepends=True)))
        self.block_start_line_number = self.line_number = 0

        self.language = language
        before, _, after = language.start_line.partition('{dsl_name}')
        assert _ == '{dsl_name}'
        self.find_start_re = libclinic.create_regex(before, after,
                                                    whole_line=False)
        self.start_re = libclinic.create_regex(before, after)
        self.verify = verify
        self.last_checksum_re: re.Pattern[str] | None = None
        self.last_dsl_name: str | None = None
        self.dsl_name: str | None = None
        self.first_block = True

    def __iter__(self) -> BlockParser:
        return self

    def __next__(self) -> Block:
        while True:
            if not self.input:
                raise StopIteration

            if self.dsl_name:
                try:
                    return_value = self.parse_clinic_block(self.dsl_name)
                except ClinicError as exc:
                    exc.filename = self.language.filename
                    exc.lineno = self.line_number
                    raise
                self.dsl_name = None
                self.first_block = False
                return return_value
            block = self.parse_verbatim_block()
            if self.first_block and not block.input:
                continue
            self.first_block = False
            return block


    def is_start_line(self, line: str) -> str | None:
        match = self.start_re.match(line.lstrip())
        return match.group(1) if match else None

    def _line(self, lookahead: bool = False) -> str:
        self.line_number += 1
        line = self.input.pop()
        if not lookahead:
            self.language.parse_line(line)
        return line

    def parse_verbatim_block(self) -> Block:
        lines = []
        self.block_start_line_number = self.line_number

        while self.input:
            line = self._line()
            dsl_name = self.is_start_line(line)
            if dsl_name:
                self.dsl_name = dsl_name
                break
            lines.append(line)

        return Block("".join(lines))

    def parse_clinic_block(self, dsl_name: str) -> Block:
        in_lines = []
        self.block_start_line_number = self.line_number + 1
        stop_line = self.language.stop_line.format(dsl_name=dsl_name)
        body_prefix = self.language.body_prefix.format(dsl_name=dsl_name)

        def is_stop_line(line: str) -> bool:
            # make sure to recognize stop line even if it
            # doesn't end with EOL (it could be the very end of the file)
            if line.startswith(stop_line):
                remainder = line.removeprefix(stop_line)
                if remainder and not remainder.isspace():
                    fail(f"Garbage after stop line: {remainder!r}")
                return True
            else:
                # gh-92256: don't allow incorrectly formatted stop lines
                if line.lstrip().startswith(stop_line):
                    fail(f"Whitespace is not allowed before the stop line: {line!r}")
                return False

        # consume body of program
        while self.input:
            line = self._line()
            if is_stop_line(line) or self.is_start_line(line):
                break
            if body_prefix:
                line = line.lstrip()
                assert line.startswith(body_prefix)
                line = line.removeprefix(body_prefix)
            in_lines.append(line)

        # consume output and checksum line, if present.
        if self.last_dsl_name == dsl_name:
            checksum_re = self.last_checksum_re
        else:
            before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}')
            assert _ == '{arguments}'
            checksum_re = libclinic.create_regex(before, after, word=False)
            self.last_dsl_name = dsl_name
            self.last_checksum_re = checksum_re
        assert checksum_re is not None

        # scan forward for checksum line
        out_lines = []
        arguments = None
        while self.input:
            line = self._line(lookahead=True)
            match = checksum_re.match(line.lstrip())
            arguments = match.group(1) if match else None
            if arguments:
                break
            out_lines.append(line)
            if self.is_start_line(line):
                break

        output: str | None
        output = "".join(out_lines)
        if arguments:
            d = {}
            for field in shlex.split(arguments):
                name, equals, value = field.partition('=')
                if not equals:
                    fail(f"Mangled Argument Clinic marker line: {line!r}")
                d[name.strip()] = value.strip()

            if self.verify:
                if 'input' in d:
                    checksum = d['output']
                else:
                    checksum = d['checksum']

                computed = libclinic.compute_checksum(output, len(checksum))
                if checksum != computed:
                    fail("Checksum mismatch! "
                         f"Expected {checksum!r}, computed {computed!r}. "
                         "Suggested fix: remove all generated code including "
                         "the end marker, or use the '-f' option.")
        else:
            # put back output
            output_lines = output.splitlines(keepends=True)
            self.line_number -= len(output_lines)
            self.input.extend(reversed(output_lines))
            output = None

        return Block("".join(in_lines), dsl_name, output=output)