math

1
"""
2

3
Clean GPT-2 merges file, removing all tokens from the tokenizer that have
4
digits, other than the "0" - "9" tokens.
5

6
"""
7

8
merges_fname = "merges_gpt2.txt"
9
new_merges_fname = "merges_gpt2_single_digit_numbers.txt"
10

11
def hasNumbers(inputString):
12
    return any(char.isdigit() for char in inputString)
13

14
with open(new_merges_fname, 'w') as f_new:
15
    with open(merges_fname, 'r') as f:
16
        lines = f.read().split("\n")
17
        for l in lines:
18
            if len(l) < 1:
19
                break
20

21
            left, right = l.split(" ")
22
            if hasNumbers(left) or hasNumbers(right):
23
                print(left, right)
24
            else:
25
                f_new.write(l + "\n")
26
                
27

28