mirror of
https://github.com/QuivrHQ/quivr.git
synced 2024-08-17 02:10:31 +03:00
feat(demo): app can now have a demo
This commit is contained in:
parent
8dccc1024a
commit
c41fe32cf0
@ -2,3 +2,5 @@ supabase_url = "https://lalalala.supabase.co"
|
||||
supabase_service_key = "lalalala"
|
||||
openai_api_key = "sk-lalalala"
|
||||
anthropic_api_key = ""
|
||||
self_hosted = "true"
|
||||
usage_limit = 1000
|
34
files.py
34
files.py
@ -55,9 +55,11 @@ def filter_file(file, supabase, vector_store):
|
||||
return False
|
||||
else:
|
||||
file_extension = os.path.splitext(file.name)[-1]
|
||||
print(file.name, file_extension)
|
||||
if file_extension in file_processors:
|
||||
file_processors[file_extension](vector_store, file)
|
||||
if st.secrets.self_hosted == "false":
|
||||
file_processors[file_extension](vector_store, file, stats_db=supabase)
|
||||
else:
|
||||
file_processors[file_extension](vector_store, file, stats_db=None)
|
||||
st.write(f"✅ {file.name} ")
|
||||
return True
|
||||
else:
|
||||
@ -65,15 +67,21 @@ def filter_file(file, supabase, vector_store):
|
||||
return False
|
||||
|
||||
def url_uploader(supabase, openai_key, vector_store):
|
||||
url = st.text_area("**Add an url**",placeholder="https://www.quivr.app")
|
||||
button = st.button("Add the URL to the database")
|
||||
if button:
|
||||
html = get_html(url)
|
||||
if html:
|
||||
st.write(f"Getting content ... {url} ")
|
||||
file, temp_file_path = create_html_file(url, html)
|
||||
ret = filter_file(file, supabase, vector_store)
|
||||
delete_tempfile(temp_file_path, url, ret)
|
||||
else:
|
||||
st.write(f"❌ Failed to access to {url} .")
|
||||
|
||||
url = st.text_area("**Add an url**",placeholder="https://www.quivr.app")
|
||||
button = st.button("Add the URL to the database")
|
||||
|
||||
if button:
|
||||
if not st.session_state["overused"]:
|
||||
html = get_html(url)
|
||||
if html:
|
||||
st.write(f"Getting content ... {url} ")
|
||||
file, temp_file_path = create_html_file(url, html)
|
||||
ret = filter_file(file, supabase, vector_store)
|
||||
delete_tempfile(temp_file_path, url, ret)
|
||||
else:
|
||||
st.write(f"❌ Failed to access to {url} .")
|
||||
else:
|
||||
st.write("You have reached your daily limit. Please come back later or self host the solution.")
|
||||
|
||||
|
||||
|
@ -9,11 +9,12 @@ from langchain.embeddings.openai import OpenAIEmbeddings
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
from utils import compute_sha1_from_content
|
||||
from langchain.schema import Document
|
||||
from stats import add_usage
|
||||
|
||||
|
||||
|
||||
# Create a function to transcribe audio using Whisper
|
||||
def _transcribe_audio(api_key, audio_file):
|
||||
def _transcribe_audio(api_key, audio_file, stats_db):
|
||||
openai.api_key = api_key
|
||||
transcript = ""
|
||||
|
||||
@ -27,16 +28,23 @@ def _transcribe_audio(api_key, audio_file):
|
||||
temp_audio_file.seek(0) # Move the file pointer to the beginning of the file
|
||||
|
||||
# Transcribe the temporary audio file
|
||||
if st.secrets.self_hosted == "false":
|
||||
add_usage(stats_db, "embedding", "audio", metadata={"file_name": audio_file.name,"file_type": file_extension})
|
||||
|
||||
transcript = openai.Audio.translate("whisper-1", temp_audio_file)
|
||||
|
||||
return transcript
|
||||
|
||||
def process_audio(vector_store, file_name):
|
||||
def process_audio(vector_store, file_name, stats_db):
|
||||
if st.secrets.self_hosted == "false":
|
||||
if file_name.size > 100000000:
|
||||
st.error("File size is too large. Please upload a file smaller than 2MB.")
|
||||
return
|
||||
file_sha = ""
|
||||
dateshort = time.strftime("%Y%m%d-%H%M%S")
|
||||
file_meta_name = f"audiotranscript_{dateshort}.txt"
|
||||
openai_api_key = st.secrets["openai_api_key"]
|
||||
transcript = _transcribe_audio(openai_api_key, file_name)
|
||||
transcript = _transcribe_audio(openai_api_key, file_name, stats_db)
|
||||
file_sha = compute_sha1_from_content(transcript.text.encode("utf-8"))
|
||||
## file size computed from transcript
|
||||
file_size = len(transcript.text.encode("utf-8"))
|
||||
@ -51,6 +59,7 @@ def process_audio(vector_store, file_name):
|
||||
|
||||
docs_with_metadata = [Document(page_content=text, metadata={"file_sha1": file_sha,"file_size": file_size, "file_name": file_meta_name, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort}) for text in texts]
|
||||
|
||||
|
||||
if st.secrets.self_hosted == "false":
|
||||
add_usage(stats_db, "embedding", "audio", metadata={"file_name": file_meta_name,"file_type": ".txt", "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
|
||||
vector_store.add_documents(docs_with_metadata)
|
||||
return vector_store
|
@ -4,12 +4,17 @@ from utils import compute_sha1_from_file
|
||||
from langchain.schema import Document
|
||||
import streamlit as st
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
from stats import add_usage
|
||||
|
||||
def process_file(vector_store, file, loader_class, file_suffix):
|
||||
def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
|
||||
documents = []
|
||||
file_sha = ""
|
||||
file_name = file.name
|
||||
file_size = file.size
|
||||
if st.secrets.self_hosted == "false":
|
||||
if file_size > 1000000:
|
||||
st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
|
||||
return
|
||||
dateshort = time.strftime("%Y%m%d")
|
||||
with tempfile.NamedTemporaryFile(delete=True, suffix=file_suffix) as tmp_file:
|
||||
tmp_file.write(file.getvalue())
|
||||
@ -30,4 +35,6 @@ def process_file(vector_store, file, loader_class, file_suffix):
|
||||
docs_with_metadata = [Document(page_content=doc.page_content, metadata={"file_sha1": file_sha1,"file_size":file_size ,"file_name": file_name, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort}) for doc in documents]
|
||||
|
||||
vector_store.add_documents(docs_with_metadata)
|
||||
if stats_db:
|
||||
add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,"file_type": file_suffix, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
|
||||
return
|
@ -1,5 +1,5 @@
|
||||
from .common import process_file
|
||||
from langchain.document_loaders.csv_loader import CSVLoader
|
||||
|
||||
def process_csv(vector_store, file):
|
||||
return process_file(vector_store, file, CSVLoader, ".csv")
|
||||
def process_csv(vector_store, file,stats_db):
|
||||
return process_file(vector_store, file, CSVLoader, ".csv",stats_db=stats_db)
|
@ -1,5 +1,5 @@
|
||||
from .common import process_file
|
||||
from langchain.document_loaders import Docx2txtLoader
|
||||
|
||||
def process_docx(vector_store, file):
|
||||
return process_file(vector_store, file, Docx2txtLoader, ".docx")
|
||||
def process_docx(vector_store, file, stats_db):
|
||||
return process_file(vector_store, file, Docx2txtLoader, ".docx", stats_db=stats_db)
|
@ -8,8 +8,8 @@ import os
|
||||
import streamlit as st
|
||||
from streamlit.runtime.uploaded_file_manager import UploadedFileRec, UploadedFile
|
||||
|
||||
def process_html(vector_store, file):
|
||||
return process_file(vector_store, file, UnstructuredHTMLLoader, ".html")
|
||||
def process_html(vector_store, file, stats_db):
|
||||
return process_file(vector_store, file, UnstructuredHTMLLoader, ".html", stats_db=stats_db)
|
||||
|
||||
|
||||
def get_html(url):
|
||||
|
@ -1,5 +1,5 @@
|
||||
from .common import process_file
|
||||
from langchain.document_loaders import UnstructuredMarkdownLoader
|
||||
|
||||
def process_markdown(vector_store, file):
|
||||
return process_file(vector_store, file, UnstructuredMarkdownLoader, ".md")
|
||||
def process_markdown(vector_store, file, stats_db):
|
||||
return process_file(vector_store, file, UnstructuredMarkdownLoader, ".md", stats_db=stats_db)
|
||||
|
@ -2,5 +2,5 @@ from .common import process_file
|
||||
from langchain.document_loaders import PyPDFLoader
|
||||
|
||||
|
||||
def process_pdf(vector_store, file):
|
||||
return process_file(vector_store, file, PyPDFLoader, ".pdf")
|
||||
def process_pdf(vector_store, file, stats_db):
|
||||
return process_file(vector_store, file, PyPDFLoader, ".pdf", stats_db=stats_db)
|
||||
|
@ -1,5 +1,5 @@
|
||||
from .common import process_file
|
||||
from langchain.document_loaders import UnstructuredPowerPointLoader
|
||||
|
||||
def process_powerpoint(vector_store, file):
|
||||
return process_file(vector_store, file, UnstructuredPowerPointLoader, ".pptx")
|
||||
def process_powerpoint(vector_store, file, stats_db):
|
||||
return process_file(vector_store, file, UnstructuredPowerPointLoader, ".pptx", stats_db=stats_db)
|
@ -1,5 +1,5 @@
|
||||
from .common import process_file
|
||||
from langchain.document_loaders import TextLoader
|
||||
|
||||
def process_txt(vector_store, file):
|
||||
return process_file(vector_store, file, TextLoader, ".txt")
|
||||
def process_txt(vector_store, file,stats_db):
|
||||
return process_file(vector_store, file, TextLoader, ".txt", stats_db=stats_db)
|
36
main.py
36
main.py
@ -10,12 +10,16 @@ from langchain.embeddings.openai import OpenAIEmbeddings
|
||||
from langchain.vectorstores import SupabaseVectorStore
|
||||
from supabase import Client, create_client
|
||||
from explorer import view_document
|
||||
from stats import get_usage_today
|
||||
|
||||
supabase_url = st.secrets.supabase_url
|
||||
supabase_key = st.secrets.supabase_service_key
|
||||
openai_api_key = st.secrets.openai_api_key
|
||||
anthropic_api_key = st.secrets.anthropic_api_key
|
||||
supabase: Client = create_client(supabase_url, supabase_key)
|
||||
self_hosted = st.secrets.self_hosted
|
||||
|
||||
|
||||
|
||||
embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
|
||||
vector_store = SupabaseVectorStore(
|
||||
@ -33,10 +37,25 @@ st.set_page_config(
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
st.title("🧠 Quivr - Your second brain 🧠")
|
||||
st.markdown("Store your knowledge in a vector store and query it with OpenAI's GPT-3/4.")
|
||||
st.markdown("---\n\n")
|
||||
|
||||
st.session_state["overused"] = False
|
||||
if self_hosted == "false":
|
||||
usage = get_usage_today(supabase)
|
||||
st.write(f"Usage today: {usage} tokens")
|
||||
st.write("---")
|
||||
if usage > st.secrets.usage_limit:
|
||||
|
||||
st.error(
|
||||
f"You have used {usage} tokens today, which is more than your daily limit of {st.secrets.usage_limit} tokens. Please come back in a few or self host.")
|
||||
st.session_state["overused"] = True
|
||||
|
||||
|
||||
|
||||
# Initialize session state variables
|
||||
if 'model' not in st.session_state:
|
||||
st.session_state['model'] = "gpt-3.5-turbo"
|
||||
@ -77,13 +96,22 @@ elif user_choice == 'Chat with your Brain':
|
||||
st.sidebar.title("Configuration")
|
||||
st.sidebar.markdown(
|
||||
"Choose your model and temperature for asking questions.")
|
||||
st.session_state['model'] = st.sidebar.selectbox(
|
||||
if st.secrets.self_hosted != "false":
|
||||
st.session_state['model'] = st.sidebar.selectbox(
|
||||
"Select Model", models, index=(models).index(st.session_state['model']))
|
||||
else:
|
||||
st.sidebar.write("**Model**: gpt-3.5-turbo")
|
||||
st.sidebar.write("**Self Host to unlock more models such as claude-v1 and GPT4**")
|
||||
st.session_state['model'] = "gpt-3.5-turbo"
|
||||
st.session_state['temperature'] = st.sidebar.slider(
|
||||
"Select Temperature", 0.0, 1.0, st.session_state['temperature'], 0.1)
|
||||
st.session_state['max_tokens'] = st.sidebar.slider(
|
||||
"Select Max Tokens", 256, 2048, st.session_state['max_tokens'], 2048)
|
||||
chat_with_doc(st.session_state['model'], vector_store)
|
||||
if st.secrets.self_hosted != "false":
|
||||
st.session_state['max_tokens'] = st.sidebar.slider(
|
||||
"Select Max Tokens", 256, 2048, st.session_state['max_tokens'], 2048)
|
||||
else:
|
||||
st.session_state['max_tokens'] = 100
|
||||
|
||||
chat_with_doc(st.session_state['model'], vector_store, stats_db=supabase)
|
||||
elif user_choice == 'Forget':
|
||||
st.sidebar.title("Configuration")
|
||||
|
||||
|
57
question.py
57
question.py
@ -6,6 +6,7 @@ from langchain.memory import ConversationBufferMemory
|
||||
from langchain.llms import OpenAI
|
||||
from langchain.chat_models import ChatAnthropic
|
||||
from langchain.vectorstores import SupabaseVectorStore
|
||||
from stats import add_usage
|
||||
|
||||
memory = ConversationBufferMemory(
|
||||
memory_key="chat_history", return_messages=True)
|
||||
@ -21,8 +22,8 @@ def count_tokens(question, model):
|
||||
return count
|
||||
|
||||
|
||||
def chat_with_doc(model, vector_store: SupabaseVectorStore):
|
||||
|
||||
def chat_with_doc(model, vector_store: SupabaseVectorStore, stats_db):
|
||||
|
||||
if 'chat_history' not in st.session_state:
|
||||
st.session_state['chat_history'] = []
|
||||
|
||||
@ -45,30 +46,34 @@ def chat_with_doc(model, vector_store: SupabaseVectorStore):
|
||||
|
||||
if button:
|
||||
qa = None
|
||||
if model.startswith("gpt"):
|
||||
logger.info('Using OpenAI model %s', model)
|
||||
qa = ConversationalRetrievalChain.from_llm(
|
||||
OpenAI(
|
||||
model_name=st.session_state['model'], openai_api_key=openai_api_key, temperature=st.session_state['temperature'], max_tokens=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True)
|
||||
elif anthropic_api_key and model.startswith("claude"):
|
||||
logger.info('Using Anthropics model %s', model)
|
||||
qa = ConversationalRetrievalChain.from_llm(
|
||||
ChatAnthropic(
|
||||
model=st.session_state['model'], anthropic_api_key=anthropic_api_key, temperature=st.session_state['temperature'], max_tokens_to_sample=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True, max_tokens_limit=102400)
|
||||
if not st.session_state["overused"]:
|
||||
add_usage(stats_db, "chat", "prompt" + question, {"model": model, "temperature": st.session_state['temperature']})
|
||||
if model.startswith("gpt"):
|
||||
logger.info('Using OpenAI model %s', model)
|
||||
qa = ConversationalRetrievalChain.from_llm(
|
||||
OpenAI(
|
||||
model_name=st.session_state['model'], openai_api_key=openai_api_key, temperature=st.session_state['temperature'], max_tokens=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True)
|
||||
elif anthropic_api_key and model.startswith("claude"):
|
||||
logger.info('Using Anthropics model %s', model)
|
||||
qa = ConversationalRetrievalChain.from_llm(
|
||||
ChatAnthropic(
|
||||
model=st.session_state['model'], anthropic_api_key=anthropic_api_key, temperature=st.session_state['temperature'], max_tokens_to_sample=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True, max_tokens_limit=102400)
|
||||
|
||||
|
||||
st.session_state['chat_history'].append(("You", question))
|
||||
|
||||
# Generate model's response and add it to chat history
|
||||
model_response = qa({"question": question})
|
||||
logger.info('Result: %s', model_response)
|
||||
|
||||
st.session_state['chat_history'].append(("Quivr", model_response["answer"]))
|
||||
|
||||
# Display chat history
|
||||
st.empty()
|
||||
for speaker, text in st.session_state['chat_history']:
|
||||
st.markdown(f"**{speaker}:** {text}")
|
||||
else:
|
||||
st.error("You have used all your free credits. Please try again later or self host.")
|
||||
|
||||
|
||||
st.session_state['chat_history'].append(("You", question))
|
||||
|
||||
# Generate model's response and add it to chat history
|
||||
model_response = qa({"question": question})
|
||||
logger.info('Result: %s', model_response)
|
||||
|
||||
st.session_state['chat_history'].append(("Quivr", model_response["answer"]))
|
||||
|
||||
# Display chat history
|
||||
st.empty()
|
||||
for speaker, text in st.session_state['chat_history']:
|
||||
st.markdown(f"**{speaker}:** {text}")
|
||||
|
||||
if count_button:
|
||||
st.write(count_tokens(question, model))
|
||||
|
31
stats.py
Normal file
31
stats.py
Normal file
@ -0,0 +1,31 @@
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# -- Create a table called "stats"
|
||||
# create table
|
||||
# stats (
|
||||
# -- A column called "time" with data type "timestamp"
|
||||
# time timestamp,
|
||||
# -- A column called "details" with data type "text"
|
||||
# chat boolean,
|
||||
# embedding boolean,
|
||||
# details text,
|
||||
# metadata jsonb,
|
||||
# -- An "integer" primary key column called "id" that is generated always as identity
|
||||
# id integer primary key generated always as identity
|
||||
# );
|
||||
|
||||
|
||||
def get_usage_today(supabase):
|
||||
# Returns the number of rows in the stats table for the last 24 hours
|
||||
response = supabase.table("stats").select("id", count="exact").gte("time", datetime.now() - timedelta(hours=24)).execute()
|
||||
return response.count
|
||||
|
||||
def add_usage(supabase, type, details, metadata):
|
||||
# Adds a row to the stats table
|
||||
supabase.table("stats").insert({
|
||||
"time": datetime.now().isoformat(),
|
||||
"chat": type == "chat",
|
||||
"embedding": type == "embedding",
|
||||
"details": details,
|
||||
"metadata": metadata
|
||||
}).execute()
|
Loading…
Reference in New Issue
Block a user