dream

1
import pickle
2
import re
3

4

5
def load_dictionaries(pickle_file):
6
    dicts = pickle.load(open(pickle_file, mode="rb"))
7
    return dicts
8

9

10
# _, _, char2id, id2char, _, _, _, _, _, _ = load_dictionaries("./datasets/conll/english/conll_eng.train_testa.pkl")
11
# print(char2id, id2char)
12

13

14
def zeros(s):
15
    """
16
    Replace every digit in a string by a zero.
17
    :param s:
18
    :return: string after replacing all digits by zeros
19
    """
20
    return re.sub("\d", "0", s)
21

22

23
def get_chunk_type(tok, idx_to_tag):
24
    """
25
    Args:
26
            tok: id of token, ex 4
27
            idx_to_tag: dictionary {4: "B-PER", ...}
28
    Returns:
29
            tuple: "B", "PER"
30
    """
31
    tag_name = idx_to_tag[tok]
32
    tag_class = tag_name.split("-")[0]
33
    tag_type = tag_name.split("-")[-1]
34
    return tag_class, tag_type
35

36

37
def get_chunks(seq, tags):
38
    """
39
    Args:
40
            seq: [4, 4, 0, 0, ...] sequence of labels
41
            tags: dict["O"] = 4
42
    Returns:
43
            list of (chunk_type, chunk_start, chunk_end)
44

45
    Example:
46
            seq = [4, 5, 0, 3]
47
            tags = {"B-PER": 4, "I-PER": 5, "B-LOC": 3}
48
            result = [("PER", 0, 2), ("LOC", 3, 4)]
49
    """
50
    default = tags["O"]
51
    idx_to_tag = {idx: tag for tag, idx in tags.items()}
52
    chunks = []
53
    chunk_type, chunk_start = None, None
54
    for i, tok in enumerate(seq):
55
        # End of a chunk 1
56
        if tok == default and chunk_type is not None:
57
            # Add a chunk.
58
            chunk = (chunk_type, chunk_start, i)
59
            chunks.append(chunk)
60
            chunk_type, chunk_start = None, None
61

62
        # End of a chunk + start of a chunk!
63
        elif tok != default:
64
            tok_chunk_class, tok_chunk_type = get_chunk_type(tok, idx_to_tag)
65
            if chunk_type is None:
66
                chunk_type, chunk_start = tok_chunk_type, i
67
            elif tok_chunk_type != chunk_type or tok_chunk_class == "B":
68
                chunk = (chunk_type, chunk_start, i)
69
                chunks.append(chunk)
70
                chunk_type, chunk_start = tok_chunk_type, i
71
        else:
72
            pass
73
    # end condition
74
    if chunk_type is not None:
75
        chunk = (chunk_type, chunk_start, len(seq))
76
        chunks.append(chunk)
77

78
    return chunks
79