lmops

1
import datasets
2
import os
3
import re
4

5
dataset = datasets.load_dataset('openwebtext', split='train')
6

7
os.makedirs("data/openwebtext", exist_ok=True)
8

9
num = 0
10
with open("data/openwebtext/data.txt", "w") as f:
11
    for data in dataset:
12
        f.write(re.sub(r"\n+", "<@x(x!>", data['text']) + "\n")
13
        num += 1
14

15
print("Number of lines:", num)