DataProcessingFramework

1
from dataclasses import dataclass
2
from typing import Literal, Optional
3

4
ModalityName = Literal["image", "video", "text"]
5

6

7
@dataclass
8
class DataModality:
9
    """Represents the modality of data
10

11
    Parameters
12
    ----------
13
    name: ModalityName
14
        Name of modality. Should be unique
15
    path_column: str
16
        Default column path to files with this modality
17
    sharded_file_name_column: str
18
        Default column name of filenames in shard with this modality
19
    column: Optional[str] = None
20
        Default column name. If this modality can be stored in a column use None.
21
    """
22
    name: ModalityName
23
    path_column: str
24
    sharded_file_name_column: str
25
    column: Optional[str] = None
26

27
    @property
28
    def can_be_column(self) -> bool:
29
        return self.column is not None
30

31
    def __hash__(self) -> int:
32
        return hash(self.name)
33

34
    def __str__(self) -> str:
35
        return self.name
36

37
    def __repr__(self) -> str:
38
        return self.name
39

40

41
MODALITIES = {
42
    'image': DataModality(
43
        'image', 'image_path',
44
        'image_name', None
45
    ),
46
    'video': DataModality(
47
        'video', 'video_path',
48
        'video_name', None
49
    ),
50
    'text': DataModality(
51
        'text', 'text_path',
52
        'text_name', 'text'
53
    )
54
}
55