llama-index

1
from typing import Any, Dict, Type
2

3
from llama_index.legacy.readers.base import BasePydanticReader
4
from llama_index.legacy.readers.discord_reader import DiscordReader
5
from llama_index.legacy.readers.elasticsearch import ElasticsearchReader
6
from llama_index.legacy.readers.google_readers.gdocs import GoogleDocsReader
7
from llama_index.legacy.readers.google_readers.gsheets import GoogleSheetsReader
8
from llama_index.legacy.readers.notion import NotionPageReader
9
from llama_index.legacy.readers.slack import SlackReader
10
from llama_index.legacy.readers.string_iterable import StringIterableReader
11
from llama_index.legacy.readers.twitter import TwitterTweetReader
12
from llama_index.legacy.readers.web import (
13
    BeautifulSoupWebReader,
14
    RssReader,
15
    SimpleWebPageReader,
16
    TrafilaturaWebReader,
17
)
18
from llama_index.legacy.readers.wikipedia import WikipediaReader
19
from llama_index.legacy.readers.youtube_transcript import YoutubeTranscriptReader
20

21
ALL_READERS: Dict[str, Type[BasePydanticReader]] = {
22
    DiscordReader.class_name(): DiscordReader,
23
    ElasticsearchReader.class_name(): ElasticsearchReader,
24
    GoogleDocsReader.class_name(): GoogleDocsReader,
25
    GoogleSheetsReader.class_name(): GoogleSheetsReader,
26
    NotionPageReader.class_name(): NotionPageReader,
27
    SlackReader.class_name(): SlackReader,
28
    StringIterableReader.class_name(): StringIterableReader,
29
    TwitterTweetReader.class_name(): TwitterTweetReader,
30
    SimpleWebPageReader.class_name(): SimpleWebPageReader,
31
    TrafilaturaWebReader.class_name(): TrafilaturaWebReader,
32
    RssReader.class_name(): RssReader,
33
    BeautifulSoupWebReader.class_name(): BeautifulSoupWebReader,
34
    WikipediaReader.class_name(): WikipediaReader,
35
    YoutubeTranscriptReader.class_name(): YoutubeTranscriptReader,
36
}
37

38

39
def load_reader(data: Dict[str, Any]) -> BasePydanticReader:
40
    if isinstance(data, BasePydanticReader):
41
        return data
42
    class_name = data.get("class_name", None)
43
    if class_name is None:
44
        raise ValueError("Must specify `class_name` in reader data.")
45

46
    if class_name not in ALL_READERS:
47
        raise ValueError(f"Reader class name {class_name} not found.")
48

49
    # remove static attribute
50
    data.pop("is_remote", None)
51

52
    return ALL_READERS[class_name].from_dict(data)
53