python langchain py-langchain llama-index chromadb

Getting response None when using Chroma DB along with llamaIndex for querying custom pdf

I making a project which uses chromadb (0.3.29), llama-index (0.6.34.post1) and langchain (0.0.245), and openai (0.27.8).But I am getting response None when I tried to query in custom pdfs.even they are getting embedded successfully , below are my codes:

import os, re
import shutil
import time
from grpc import ServicerContext
import vectordb
from langchain import OpenAI
from llama_index import GPTTreeIndex, SimpleDirectoryReader, LLMPredictor,GPTVectorStoreIndex,PromptHelper, VectorStoreIndex
from llama_index import LangchainEmbedding, ServiceContext,  Prompt
from llama_index import StorageContext, load_index_from_storage
from langchain.embeddings import OpenAIEmbeddings
from langchain.llms import AzureOpenAI
# Import Azure OpenAI
#from langchain_community.llms import AzureOpenAI
import chromadb
from llama_index.vector_stores import ChromaVectorStore
 
from dotenv import load_dotenv
load_dotenv()
#openai.api_key = os.getenv["OPENAI_API_KEY"]



def regenrate_tokens(collection_name,persist_directory): 
         
        if os.path.isdir((persist_directory)):
            print("directory existed ,replacing previous directory")
            shutil.rmtree(persist_directory)
            print("Recreating Embeddings...")
            vector=vectordb.CreatingChromaDB(collection_name,persist_directory)
            vector.storage_context.persist(persist_dir= persist_directory)

        else:
            print("directory does not exit, creating new embeddings.")
            vector=vectordb.CreatingChromaDB(collection_name,persist_directory)
            vector.storage_context.persist(persist_dir= persist_directory)
        
        time.sleep(10) # Sleep for 10 seconds

        return('Token regenrated, you can ask the questions. 😄')

def query__from_knowledge_base(question):
    persist_directory = './ChromaDb'
    collection_name = "chromaVectorStore"

  
    if(question == 'regenerate tokens'):
        return(regenrate_tokens(collection_name,persist_directory))
    
    index = vectordb.LoadFromDisk(collection_name,persist_directory)
    print(index)
    # define custom Prompt
    # TEMPLATE_STR = (
    #     "We have provided context information below. \n"
    #     "---------------------\n"
    #     "{context_str}"
    #     "\n---------------------\n"
    #     "Given this information, please answer the question: {query_str}\n"
    # )
    TEMPLATE_STR = """Create a final answer to the given questions using the provided document excerpts(in no particular order) as references. ALWAYS include a "SOURCES" section in your answer including only the minimal set of sources needed to answer the question. Always include the Source Preview of source. If answer has step in document please response in step. If you are unable to answer the question, simply state that you do not know. Do not attempt to fabricate an answer and leave the SOURCES section empty.

        "---------------------\n"
        "{context_str}"
        "\n---------------------\n"
        "Given this information, please answer the question: {query_str}\n"
    """

    QA_TEMPLATE = Prompt(TEMPLATE_STR)
    
    query_engine = index.as_query_engine(text_qa_template=QA_TEMPLATE)
    print(query_engine)
    response = query_engine.query(question)
    print(question)
    # print(response)
    response = str(response)   
    response = re.sub(r'Answer:', '', response)
    response = response.strip()
    return(response)
    

#print(regenrate_tokens())
#print(query__from_knowledge_base('Enabling online archive for the user’s mailbox.'))

file vectordb.py, containing creation and querying methods are below:

def CreatingChromaDB(collection_name,persist_directory):

    documents = SimpleDirectoryReader('./static/upload/').load_data()
    # deployment_name = "text-davinci-003"
    deployment_name = "gpt-3.5-turbo"
    openai_api_version="30/08/2023"

    # Create LLM via Azure OpenAI Service
    llm = AzureOpenAI(deployment_name=deployment_name,openai_api_version=openai_api_version)
    llm_predictor = LLMPredictor(llm=llm)
    llm_predictor = LLMPredictor(llm = llm_predictor)
    embedding_llm = LangchainEmbedding(OpenAIEmbeddings())

    # Define prompt helper
    max_input_size = 3000
    num_output = 256
    chunk_size_limit = 1000 # token window size per document
    max_chunk_overlap = 20 # overlap for each token fragment
    prompt_helper = PromptHelper(max_input_size=max_input_size, num_output=num_output,
                              max_chunk_overlap=max_chunk_overlap, chunk_size_limit=chunk_size_limit)

    service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor, embed_model=embedding_llm, prompt_helper=prompt_helper)
    chroma_client = chromadb.Client(Settings(
    chroma_db_impl="duckdb+parquet",
    persist_directory= persist_directory))

    print(collection_name)

    # create a collection
    chroma_collection = chroma_client.get_or_create_collection(collection_name,embedding_function=embedding_llm)
    # https://docs.trychroma.com/api-reference
    print(chroma_collection.count())

    vector_store = ChromaVectorStore(chroma_collection)
    storage_context = StorageContext.from_defaults(vector_store=vector_store)
    index = GPTVectorStoreIndex.from_documents(documents, storage_context=storage_context, service_context=service_context)
    print(chroma_collection.count())
    print(chroma_collection.get()['documents'])
    print(chroma_collection.get()['metadatas'])

    # index.storage_context.persist()
    return index

def LoadFromDisk(collection_name,persist_directory):
    chroma_client = chromadb.Client(Settings(
    chroma_db_impl="duckdb+parquet",
    persist_directory= persist_directory))

    print(collection_name)

    chroma_collection = chroma_client.get_or_create_collection(collection_name)
    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
    index = GPTVectorStoreIndex.from_vector_store(vector_store=vector_store)
    return index

if we tried to regenerate tokens and try to query from pdfs then its shows "None" response, even if those files are embedded properly.

Solution

There were multiple issues with this code so I've taken the liberty of editing extensively.

You don't need to use Langchain, LlamaIndex is enough
You don't need to persist your storage context to disk, the Chroma client does that automatically when you create a persistent client
Your prompt needed to be a PromptTemplate

from dotenv import load_dotenv
load_dotenv()

import re
from llama_index import (
    SimpleDirectoryReader, 
    VectorStoreIndex, 
    ServiceContext, 
    StorageContext
)
from llama_index.vector_stores import ChromaVectorStore
from llama_index.llms import OpenAI
from llama_index.embeddings import OpenAIEmbedding
from llama_index.prompts import PromptTemplate
import chromadb

import logging
import sys
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

def refresh_data(collection_name,persist_directory): 

    chroma_client = chromadb.PersistentClient(path=persist_directory)
    embed_model = OpenAIEmbedding()
    documents = SimpleDirectoryReader('./static/upload/').load_data()
    chroma_collection = chroma_client.get_or_create_collection(collection_name)
    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
    storage_context = StorageContext.from_defaults(vector_store=vector_store)
    service_context = ServiceContext.from_defaults(embed_model=embed_model)
    index = VectorStoreIndex.from_documents(
        documents, storage_context=storage_context, service_context=service_context
    )
    return('Data refreshed, you can ask the questions. 😄')

def query__from_knowledge_base(question):

    persist_directory = './ChromaDb'
    collection_name = "chromaVectorStore"
  
    if(question == 'refresh_data'):
        return(refresh_data(collection_name,persist_directory))
    
    chroma_client = chromadb.PersistentClient(persist_directory)
    chroma_collection = chroma_client.get_or_create_collection(collection_name)
    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
    llm = OpenAI(model="gpt-3.5-turbo") # default, could try gpt-4
    service_context = ServiceContext.from_defaults(llm=llm)
    index = VectorStoreIndex.from_vector_store(
        vector_store,
        service_context=service_context,
    )

    TEMPLATE_STR = """Create a final answer to the given questions using the provided document excerpts(in no particular order) as references. ALWAYS include a "SOURCES" section in your answer including only the minimal set of sources needed to answer the question. Always include the Source Preview of source. If answer has step in document please response in step. If you are unable to answer the question, simply state that you do not know. Do not attempt to fabricate an answer and leave the SOURCES section empty.

        "---------------------\n"
        "{context_str}"
        "\n---------------------\n"
        "Given this information, please answer the question: {query_str}\n"
    """

    QA_TEMPLATE = PromptTemplate(TEMPLATE_STR)
    
    query_engine = index.as_query_engine(text_qa_template=QA_TEMPLATE)
    response = query_engine.query(question)
    response = str(response)   
    response = re.sub(r'Answer:', '', response)
    response = response.strip()
    return(response)
    

print(refresh_data("chromaVectorStore","./ChromaDb"))
print(query__from_knowledge_base('What are some of the latest advances in computer vision?'))

I've tested this locally with some PDFs and it seems to work and do what you wanted, good luck with your project!