yargy

1

2
from yargy import Parser
3
from yargy.tagger import Tagger
4
from yargy.predicates import tag
5

6

7
INSIDE = 'I'
8
OUTSIDE = 'O'
9

10

11
class MyTagger(Tagger):
12
    tags = {INSIDE, OUTSIDE}
13

14
    def __call__(self, tokens):
15
        for index, token in enumerate(tokens):
16
            yield token.tagged(
17
                OUTSIDE
18
                if index % 3 == 0
19
                else INSIDE
20
            )
21

22

23
def test_tagger():
24
    text = 'a b c d e f g'
25
    A = tag('I').repeatable()
26
    parser = Parser(A, tagger=MyTagger())
27

28
    matches = parser.findall(text)
29
    spans = [_.span for _ in matches]
30
    substrings = [
31
        text[start:stop]
32
        for start, stop in spans
33
    ]
34
    assert substrings == ['b c', 'e f']
35