local-llm-with-rag

1
from langchain_community.document_loaders import (
2
    DirectoryLoader,
3
    PyPDFLoader,
4
    TextLoader,
5
)
6
import os
7
from typing import List
8
from langchain_core.documents import Document
9

10

11
def load_documents(path: str) -> List[Document]:
12
    """
13
    Loads documents from the specified directory path.
14

15
    This function supports loading of PDF, Markdown, and HTML documents by utilizing
16
    different loaders for each file type. It checks if the provided path exists and
17
    raises a FileNotFoundError if it does not. It then iterates over the supported
18
    file types and uses the corresponding loader to load the documents into a list.
19

20
    Args:
21
        path (str): The path to the directory containing documents to load.
22

23
    Returns:
24
        List[Document]: A list of loaded documents.
25

26
    Raises:
27
        FileNotFoundError: If the specified path does not exist.
28
    """
29
    if not os.path.exists(path):
30
        raise FileNotFoundError(f"The specified path does not exist: {path}")
31

32
    loaders = {
33
        ".pdf": DirectoryLoader(
34
            path,
35
            glob="**/*.pdf",
36
            loader_cls=PyPDFLoader,
37
            show_progress=True,
38
            use_multithreading=True,
39
        ),
40
        ".md": DirectoryLoader(
41
            path,
42
            glob="**/*.md",
43
            loader_cls=TextLoader,
44
            show_progress=True,
45
        ),
46
    }
47

48
    docs = []
49
    for file_type, loader in loaders.items():
50
        print(f"Loading {file_type} files")
51
        docs.extend(loader.load())
52
    return docs
53