lama

1
#!/usr/bin/env python3
2

3

4
import tqdm
5
import webdataset as wds
6

7

8
def main(args):
9
    input_dataset = wds.Dataset(args.infile)
10
    output_dataset = wds.ShardWriter(args.outpattern)
11
    for rec in tqdm.tqdm(input_dataset):
12
        output_dataset.write(rec)
13

14

15
if __name__ == '__main__':
16
    import argparse
17

18
    aparser = argparse.ArgumentParser()
19
    aparser.add_argument('infile', type=str)
20
    aparser.add_argument('outpattern', type=str)
21

22
    main(aparser.parse_args())
23