DataProcessingFramework

1
from DPF import ShardsDatasetConfig, DatasetReader
2
from DPF.filters.images.info_filter import ImageInfoFilter
3
from DPF.filters.images.hash_filters import PHashFilter
4

5

6
if __name__ == "__main__":
7
    config = ShardsDatasetConfig.from_path_and_columns(
8
        'examples/example_dataset',
9
        image_name_col='image_name',
10
        text_col="caption"
11
    )
12

13
    reader = DatasetReader()
14
    processor = reader.read_from_config(config)
15

16
    print("Dataset summary:", processor.print_summary())
17

18
    datafilter = ImageInfoFilter(workers=16)
19
    print('Applying ImageInfoFilter')
20
    processor.apply_data_filter(datafilter)
21

22
    datafilter = PHashFilter(sim_hash_size=8, workers=16)
23
    print('Applying PHashFilter')
24
    processor.apply_data_filter(datafilter)
25

26
    print('Result dataset metadata')
27
    print(processor.df)