lmops

1
import json
2
import pathlib 
3
class BufferedJsonWriter(object):
4
    def __init__(self, file_name, buffer_size=50):
5
        self.file_path = file_name
6
        self.buffer = []
7
        self.buffer_size = buffer_size
8

9
    def __enter__(self):
10
        return self
11

12
    def __exit__(self, type, value, traceback):
13
        self.write_buffer()
14

15
    def write(self, obj=None):
16
        if obj is not None:
17
            self.buffer.append(obj)
18
        if len(self.buffer)>=self.buffer_size:
19
            self.write_buffer()
20

21
    def write_buffer(self):
22
        with open(self.file_path, "a") as data_file:
23
            data_file.write(json.dumps(self.buffer))
24
            data_file.write("\n")
25
            self.buffer = []
26

27
class BufferedJsonReader(object):
28
    def __init__(self, file_name):
29
        self.file_path = file_name
30

31
    def __enter__(self):
32
        return self
33

34
    def __exit__(self, type, value, traceback):
35
        pass
36

37
    def __itr__(self):
38
        with open(self.file_path, "r") as data_file:
39
            for line in data_file:
40
                yield from json.loads(line)
41
                
42
    def read(self):
43
        return list(self.__itr__())
44

45

46

47
def get_cache_path(dataset):
48
    cache_files = dataset.cache_files
49
    if isinstance(cache_files,dict):
50
        cache_files = next(iter(cache_files.values()))
51
    return pathlib.Path(cache_files[0]['filename']).parent