naeval

1

2
from os.path import (
3
    expanduser,
4
    join as join_path
5
)
6

7
from IPython.display import HTML
8
from tqdm.notebook import tqdm as log_progress
9

10
from naeval.const import (
11
    NEWS, WIKI, FICTION, SOCIAL, POETRY,
12

13
    DATASET, JL, GZ
14
)
15
from naeval.io import (
16
    format_jl,
17
    parse_jl,
18

19
    load_gz_lines,
20
    dump_gz_lines,
21
)
22
from naeval.record import (
23
    as_jsons,
24
    from_jsons
25
)
26
from naeval.dataset import sample
27
from naeval.readme import patch_readme
28
from naeval.lemma.datasets import load_dataset
29
from naeval.lemma.markup import (
30
    Markup,
31
    show_markup
32
)
33

34
CORUS_DATA_DIR = expanduser('~/proj/corus-data/gramru')
35
CORUS_FILES = {
36
    NEWS: [
37
        'dev/GramEval2020-RuEval2017-Lenta-news-dev.conllu',
38
        'train/MorphoRuEval2017-Lenta-train.conllu',
39
    ],
40
    WIKI: [
41
        'dev/GramEval2020-GSD-wiki-dev.conllu',
42
        'train/GramEval2020-GSD-train.conllu'
43
    ],
44
    FICTION: [
45
        'dev/GramEval2020-SynTagRus-dev.conllu',
46
        'train/GramEval2020-SynTagRus-train-v2.conllu',
47
        'train/MorphoRuEval2017-JZ-gold.conllu'
48
    ],
49
    SOCIAL: [
50
        'dev/GramEval2020-RuEval2017-social-dev.conllu',
51
        'train/GramEval2020-Taiga-social-train.conllu',
52
        'train/MorphoRuEval2017-VK-gold.conllu'
53
    ],
54
    POETRY: [
55
        'dev/GramEval2020-Taiga-poetry-dev.conllu',
56
        'train/GramEval2020-Taiga-poetry-train.conllu'
57
    ],
58
}
59

60
DATASETS = [NEWS, WIKI, FICTION, SOCIAL, POETRY]
61

62
DATA_DIR = expanduser('~/proj/naeval/data/lemma')
63
LEMMA = 'lemma'
64
README = expanduser('~/proj/naeval/README.md')
65