zephyr-7b-beta-rag-demo

1
import os
2
from langchain.text_splitter import RecursiveCharacterTextSplitter
3
from langchain.vectorstores import Chroma
4
from langchain.embeddings import HuggingFaceBgeEmbeddings
5
from langchain.document_loaders import PyPDFLoader
6

7
model_name = "BAAI/bge-large-en"
8
model_kwargs = {'device': 'cpu'}
9
encode_kwargs = {'normalize_embeddings': False}
10
embeddings = HuggingFaceBgeEmbeddings(
11
    model_name=model_name,
12
    model_kwargs=model_kwargs,
13
    encode_kwargs=encode_kwargs
14
)
15

16
loader = PyPDFLoader("pet.pdf")
17
documents = loader.load()
18
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
19
texts = text_splitter.split_documents(documents)
20

21
vector_store = Chroma.from_documents(texts, embeddings, collection_metadata={"hnsw:space": "cosine"}, persist_directory="stores/pet_cosine")
22

23
print("Vector Store Created.......")