CSS-LM

1
import csv
2

3
counter = 0
4
#max_num_sent = 1000000
5
max_num_sent = 500000
6

7
all_data_list = list()
8
with open("domain_data/all_fewshot.txt") as f:
9
    for line in f:
10
        if line == "\n":
11
            continue
12
        if counter < max_num_sent:
13
            counter+=1
14
            line = line.strip()
15
            #print(line)
16
            all_data_list.append([line])
17
        else:
18
            break
19

20
print("Domain:",counter)
21

22
#with open("openwebtext.txt") as f:
23
with open("retrieve.txt") as f:
24
    for line in f:
25
        if line == "\n":
26
            continue
27
        if counter < max_num_sent:
28
            counter+=1
29
            line = line.strip()
30
            all_data_list.append([line])
31
        else:
32
            break
33
print("All:",counter)
34

35
with open("train.txt", "w") as f:
36
    writer = csv.writer(f)
37
    writer.writerows(all_data_list)
38

39