tests/test_tokenizer.py - main - space_verse/LambdaLanguage

tests/test_tokenizer.py
118 строк5 KB
spacer
ADD: parser;
16 авг 2025, 13:28
16 авг 2025, 13:28f4fca3d
from contextlib import nullcontext as dose_not_raise

import pytest

from tokenizer.abc import AbstractTokenizer
from tokenizer.exceptions import EmptyCode
from tokenizer.tokenizer import LambdaTokenizer
from tokenizer.tokens import KEYWORDS, TOKENS_SPECIFICATION, Token


@pytest.fixture
def tokenizer() -> AbstractTokenizer:
    tokenizer = LambdaTokenizer(
        keywords=KEYWORDS,
        tokens_specification=TOKENS_SPECIFICATION)
    return tokenizer


@pytest.mark.parametrize(
        "input, result, expectation",
        [
            ("", [], pytest.raises(EmptyCode)),
            ("if", [
                Token(type="IF", value="if", line=1, column=1),
                ], dose_not_raise()),
            ("for", [
                Token(type="FOR", value="for", line=1, column=1),
                ], dose_not_raise()),
            ("while", [
                Token(type="WHILE", value="while", line=1, column=1),
                ], dose_not_raise()),
            ("in", [
                Token(type="IN", value="in", line=1, column=1),
                ], dose_not_raise()),
            ("int a\nfloat b", [
                Token(type="INT", value="int", line=1, column=1),
                Token(type="ID", value="a", line=1, column=5),
                Token(type="ENDL", value="\n", line=1, column=6),
                Token(type="FLOAT", value="float", line=2, column=1),
                Token(type="ID", value="b", line=2, column=7),
                ], dose_not_raise()),
            ("a * b", [
                Token(type="ID", value="a", line=1, column=1),
                Token(type="OP", value="*", line=1, column=3),
                Token(type="ID", value="b", line=1, column=5),
                ], dose_not_raise()),
            ("a += 1", [
                Token(type="ID", value="a", line=1, column=1),
                Token(type="OP", value="+=", line=1, column=3),
                Token(type="INT_NUMBER", value='1', line=1, column=6),
                ], dose_not_raise()),
            ("**", [
                Token(type="OP", value="**", line=1, column=1),
                ], dose_not_raise()),
            ("//", [
                Token(type="OP", value="//", line=1, column=1),
                ], dose_not_raise()),
            ("/", [
                Token(type="OP", value="/", line=1, column=1),
                ], dose_not_raise()),
            ("\"hello\"", [
                Token(type="STRING", value="hello", line=1, column=1),
                ], dose_not_raise()),
            ("'hello'", [
                Token(type="STRING", value="hello", line=1, column=1),
                ], dose_not_raise()),
            ("int a = 0", [
                Token(type="INT", value="int", line=1, column=1),
                Token(type="ID", value="a", line=1, column=5),
                Token(type="OP", value="=", line=1, column=7),
                Token(type="INT_NUMBER", value='0', line=1, column=9),
                ], dose_not_raise()),
            (":=", [
                Token(type="OP", value=":=", line=1, column=1),
                ], dose_not_raise()),
            ("'#hello'", [
                Token(type="STRING", value="#hello", line=1, column=1),
                ], dose_not_raise()),
            ("#hello", [], dose_not_raise()),
            ("#\nhello\n#", [
                Token(type="ENDL", value="\n", line=1, column=2),
                Token(type="ID", value="hello", line=2, column=1),
                Token(type="ENDL", value="\n", line=2, column=6),
                ], dose_not_raise()),
            ("hello   # some comment", [
                Token(type="ID", value="hello", line=1, column=1),
                ], dose_not_raise()),
            ("    main()", [
                Token(type="INDENT", value="    ", line=1, column=1),
                Token(type="ID", value="main", line=1, column=5),
                Token(type="LPAR", value="(", line=1, column=9),
                Token(type="RPAR", value=")", line=1, column=10),
                ], dose_not_raise()),
            ("arr:int", [
                Token(type="ARR", value="arr", line=1, column=1),
                Token(type="OP", value=":", line=1, column=4),
                Token(type="INT", value="int", line=1, column=5),
                ], dose_not_raise()),
            ("func main(): ...", [
                Token(type="FUNC", value="func", line=1, column=1),
                Token(type="ID", value="main", line=1, column=6),
                Token(type="LPAR", value="(", line=1, column=10),
                Token(type="RPAR", value=")", line=1, column=11),
                Token(type="OP", value=":", line=1, column=12),
                Token(type="ELLIPSIS", value="...", line=1, column=14),
                ], dose_not_raise()),
            ("'''hello'''", [
                Token(type="DOC_STRING", value="'''hello'''", line=1, column=1),
                ], dose_not_raise()),
        ])
def test_tokenize(
        tokenizer: AbstractTokenizer,
        input: str,
        result,
        expectation,
        ) -> None:
    with expectation:
        assert tokenizer.tokenize(input=input) == result