outlines

1
import pytest
2

3
import outlines.grammars as grammars
4
from outlines.fsm.guide import CFGGuide
5

6

7
@pytest.mark.parametrize("grammar", [grammars.json, grammars.arithmetic])
8
def test_grammar_module(grammar):
9
    class MockTokenizer:
10
        vocabulary = {"(": 1, ")": 2, "a": 3, "eos": 4}
11
        special_tokens = {"eos"}
12
        eos_token = "eos"
13
        eos_token_id = 4
14

15
        def convert_token_to_string(self, token):
16
            return token
17

18
        @property
19
        def inverse_vocabulary(self):
20
            return {v: k for k, v in self.vocabulary.items()}
21

22
        def decode(self, token_ids):
23
            return [self.inverse_vocabulary[t] for t in token_ids]
24

25
    cfg_str = """
26
        start: s
27
        s: "(" s ")" | /a+/
28
    """
29
    tokenizer = MockTokenizer()
30
    fsm = CFGGuide(cfg_str, tokenizer)
31
    assert isinstance(fsm, CFGGuide)
32