src/tokenizer/tokenizer.py - main - space_verse/LambdaLanguage

src/tokenizer/tokenizer.py
113 строк4 KB
spacer
REMAKE: expression parsing
18 авг 2025, 03:00
18 авг 2025, 03:009baed1f
import re

from tokenizer.abc import AbstractTokenizer
from tokenizer.exceptions import EmptyCode, UnknownToken
from tokenizer.tokens import Token


class LambdaTokenizer(AbstractTokenizer):
    def __init__(
            self,
            keywords: set[str],
            tokens_specification: list[tuple[str, str]],
            ) -> None:
        self.keywords = keywords
        self.tokens = tokens_specification

    @property
    def tokens_pattern(self) -> str:
        return "|".join([
            f"(?P<{key}>{pattern})"
            for key, pattern in self.tokens
        ])

    def pretty_err_details(
            self,
            input: str,
            line_number: int,
            column_number: int,
            token_value: str,
            ) -> str:
        start_code_line = res if (res := line_number - 2) >= 0 else 0
        end_code_line = line_number + 2
        indent = len(str(end_code_line)) + 1
        err_marker = (f"%-{indent}s" % '~') + ' ' * (column_number) + '^' * len(token_value)
        code_lines = input.split('\n')[start_code_line:end_code_line]
        pretty_lines = [
            f"%{indent}d %s" % (line_number, line)
            for line_number, line in enumerate(
                code_lines, start=start_code_line + 1)
        ]
        pretty_lines.insert(start_code_line - line_number + 1, err_marker)
        err_details = '\n'.join(pretty_lines)
        return err_details

    def tokenize(
            self,
            input: str,
            module_name: str = "__main__",
            raise_on_empty: bool = True,
            ) -> list:
        result_tokens = []

        if not input:
            if raise_on_empty:
                raise EmptyCode

        line_number = 1 # start at 1
        start_line = 0

        for matched in re.finditer(self.tokens_pattern, input):
            column_number = matched.start() - start_line + 1    # start at 1
            token_value = matched.group()
            token_type = matched.lastgroup
            if token_type is None:
                raise UnknownToken("Uncnown token at line %d, column %d" %
                                   (line_number, column_number))

            match token_type:
                case "ID":
                    if token_value in self.keywords:
                        token_type = token_value.upper()
                case "STRING":
                    replaced = {
                        '\\n': '\n',
                        '\\t': '\t',
                        '\\a': '\a',
                        '\\r': '\r',
                    }
                    token_value = token_value[1:-1]
                    for _from, to in replaced.items():
                        token_value = token_value.replace(_from, to)
                case "ENDL":
                    start_line = matched.end()
                case "MISMATCH":
                    err_details = self.pretty_err_details(
                            input=input,
                            line_number=line_number,
                            column_number=column_number,
                            token_value=token_value)
                    err_message = (
                        f"Uncnown token at %s::%d.%d =>\n%s" % (
                        module_name,
                        line_number,
                        column_number,
                        err_details,
                    ))
                    raise UnknownToken(err_message)

            if token_type != "VOID":
                result_tokens.append(Token(
                    type=token_type,
                    value=token_value,
                    line=line_number,
                    column=column_number,
                ))

            if token_type == "ENDL":
                line_number += 1
            elif token_type == "DOC_STRING":
                if type(token_value) == str:
                    line_number += token_value.count("\n")

        return result_tokens