rag2.py

import os

from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import TextLoader
from langchain_community.vectorstores import Chroma
from langchain_openai import OpenAIEmbeddings

currt_dir = os.path.dirname(os.path.abspath(__file__))
rules = os.path.join(currt_dir, "RAGFiles")
db = os.path.join(currt_dir, "db")
persist_directory = os.path.join(currt_dir, "db","chroma_rules_db")


rule_files= [f for f in os.listdir(rules) if f.endswith('.txt')]

documents = []
for file in rule_files:
    file_path = os.path.join(rules, file)
    loader = TextLoader(file_path, encoding="utf-8")
    documents = loader.load()
    for doc in documents:
        doc.metadata= {"source": file}
    documents.extend(documents)
    



text_splitter = RecursiveCharacterTextSplitter(chunk_size=2080, chunk_overlap=1040)
doc= text_splitter.split_documents(documents)

print(f"Number of documents: {len(doc)}")
    #print(f"sample document:\n{doc[0].page_content}\n")
if not os.path.exists(persist_directory):
    print("Creating vector store...")

    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

    print("done creating embeddings")

    db = Chroma.from_documents(doc, embeddings, persist_directory=persist_directory)
    print("done creating vector store")

else: 
    print("Directory already exists.")
    print(f"sample document:\n{doc[0].page_content}\n")