feat(demo): app can now have a demo

.streamlit/secrets.toml.example

@@ -2,3 +2,5 @@ supabase_url = "https://lalalala.supabase.co"
 supabase_service_key = "lalalala"
 openai_api_key = "sk-lalalala"
 anthropic_api_key = ""
+self_hosted = "true"
+usage_limit = 1000

files.py

@@ -55,9 +55,11 @@ def filter_file(file, supabase, vector_store):
         return False
     else:
         file_extension = os.path.splitext(file.name)[-1]
-        print(file.name, file_extension)
         if file_extension in file_processors:
-            file_processors[file_extension](vector_store, file)
+            if st.secrets.self_hosted == "false":
+                file_processors[file_extension](vector_store, file, stats_db=supabase)
+            else:
+                file_processors[file_extension](vector_store, file, stats_db=None)
             st.write(f"✅ {file.name} ")
             return True
         else:
@@ -65,15 +67,21 @@ def filter_file(file, supabase, vector_store):
             return False
 
 def url_uploader(supabase, openai_key, vector_store):
-    url = st.text_area("**Add an url**",placeholder="https://www.quivr.app")
-    button = st.button("Add the URL to the database")
-    if button:
-        html = get_html(url)
-        if html:
-            st.write(f"Getting content ... {url}  ")
-            file, temp_file_path = create_html_file(url, html)
-            ret = filter_file(file, supabase, vector_store)
-            delete_tempfile(temp_file_path, url, ret)
-        else:
-            st.write(f"❌ Failed to access to {url} .")
+    
+        url = st.text_area("**Add an url**",placeholder="https://www.quivr.app")
+        button = st.button("Add the URL to the database")
+
+        if button:
+            if not st.session_state["overused"]:
+                html = get_html(url)
+                if html:
+                    st.write(f"Getting content ... {url}  ")
+                    file, temp_file_path = create_html_file(url, html)
+                    ret = filter_file(file, supabase, vector_store)
+                    delete_tempfile(temp_file_path, url, ret)
+                else:
+                    st.write(f"❌ Failed to access to {url} .")
+            else:
+                st.write("You have reached your daily limit. Please come back later or self host the solution.")
+   
 

loaders/audio.py

@@ -9,11 +9,12 @@ from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from utils import compute_sha1_from_content
 from langchain.schema import Document
+from stats import add_usage
 
 
 
 # Create a function to transcribe audio using Whisper
-def _transcribe_audio(api_key, audio_file):
+def _transcribe_audio(api_key, audio_file, stats_db):
     openai.api_key = api_key
     transcript = ""
     
@@ -27,16 +28,23 @@ def _transcribe_audio(api_key, audio_file):
             temp_audio_file.seek(0)  # Move the file pointer to the beginning of the file
             
             # Transcribe the temporary audio file
+            if st.secrets.self_hosted == "false":
+                    add_usage(stats_db, "embedding", "audio", metadata={"file_name": audio_file.name,"file_type": file_extension})
+            
             transcript = openai.Audio.translate("whisper-1", temp_audio_file)
 
     return transcript
 
-def process_audio(vector_store, file_name):
+def process_audio(vector_store, file_name, stats_db):
+    if st.secrets.self_hosted == "false":
+        if file_name.size > 100000000:
+            st.error("File size is too large. Please upload a file smaller than 2MB.")
+            return
     file_sha = ""
     dateshort = time.strftime("%Y%m%d-%H%M%S")
     file_meta_name = f"audiotranscript_{dateshort}.txt"
     openai_api_key = st.secrets["openai_api_key"]
-    transcript = _transcribe_audio(openai_api_key, file_name)
+    transcript = _transcribe_audio(openai_api_key, file_name, stats_db)
     file_sha = compute_sha1_from_content(transcript.text.encode("utf-8"))
     ## file size computed from transcript
     file_size = len(transcript.text.encode("utf-8"))
@@ -51,6 +59,7 @@ def process_audio(vector_store, file_name):
 
     docs_with_metadata = [Document(page_content=text, metadata={"file_sha1": file_sha,"file_size": file_size, "file_name": file_meta_name, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort}) for text in texts]
 
-    
+    if st.secrets.self_hosted == "false":
+        add_usage(stats_db, "embedding", "audio", metadata={"file_name": file_meta_name,"file_type": ".txt", "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
     vector_store.add_documents(docs_with_metadata)
     return vector_store

loaders/common.py

@@ -4,12 +4,17 @@ from utils import compute_sha1_from_file
 from langchain.schema import Document
 import streamlit as st
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+from stats import add_usage
 
-def process_file(vector_store, file, loader_class, file_suffix):
+def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
     documents = []
     file_sha = ""
     file_name = file.name
     file_size = file.size
+    if st.secrets.self_hosted == "false":
+        if file_size > 1000000:
+            st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
+            return
     dateshort = time.strftime("%Y%m%d")
     with tempfile.NamedTemporaryFile(delete=True, suffix=file_suffix) as tmp_file:
         tmp_file.write(file.getvalue())
@@ -30,4 +35,6 @@ def process_file(vector_store, file, loader_class, file_suffix):
     docs_with_metadata = [Document(page_content=doc.page_content, metadata={"file_sha1": file_sha1,"file_size":file_size ,"file_name": file_name, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort}) for doc in documents]
     
     vector_store.add_documents(docs_with_metadata)
+    if stats_db:
+        add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,"file_type": file_suffix, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
     return 

loaders/csv.py

@@ -1,5 +1,5 @@
 from .common import process_file
 from langchain.document_loaders.csv_loader import CSVLoader
 
-def process_csv(vector_store, file):
-    return process_file(vector_store, file, CSVLoader, ".csv")
+def process_csv(vector_store, file,stats_db):
+    return process_file(vector_store, file, CSVLoader, ".csv",stats_db=stats_db)

loaders/docx.py

@@ -1,5 +1,5 @@
 from .common import process_file
 from langchain.document_loaders import Docx2txtLoader
 
-def process_docx(vector_store, file):
-    return process_file(vector_store, file, Docx2txtLoader, ".docx")
+def process_docx(vector_store, file, stats_db):
+    return process_file(vector_store, file, Docx2txtLoader, ".docx", stats_db=stats_db)

loaders/html.py

@@ -8,8 +8,8 @@ import os
 import streamlit as st
 from streamlit.runtime.uploaded_file_manager import UploadedFileRec, UploadedFile
 
-def process_html(vector_store, file):
-    return process_file(vector_store, file, UnstructuredHTMLLoader, ".html")
+def process_html(vector_store, file, stats_db):
+    return process_file(vector_store, file, UnstructuredHTMLLoader, ".html", stats_db=stats_db)
 
 
 def get_html(url):

loaders/markdown.py

@@ -1,5 +1,5 @@
 from .common import process_file
 from langchain.document_loaders import UnstructuredMarkdownLoader
 
-def process_markdown(vector_store, file):
-    return process_file(vector_store, file, UnstructuredMarkdownLoader, ".md")
+def process_markdown(vector_store, file, stats_db):
+    return process_file(vector_store, file, UnstructuredMarkdownLoader, ".md", stats_db=stats_db)

loaders/pdf.py

@@ -2,5 +2,5 @@ from .common import process_file
 from langchain.document_loaders import PyPDFLoader
 
 
-def process_pdf(vector_store, file):
-    return process_file(vector_store, file, PyPDFLoader, ".pdf")
+def process_pdf(vector_store, file, stats_db):
+    return process_file(vector_store, file, PyPDFLoader, ".pdf", stats_db=stats_db)

loaders/powerpoint.py

@@ -1,5 +1,5 @@
 from .common import process_file
 from langchain.document_loaders import UnstructuredPowerPointLoader
 
-def process_powerpoint(vector_store, file):
-    return process_file(vector_store, file, UnstructuredPowerPointLoader, ".pptx")
+def process_powerpoint(vector_store, file, stats_db):
+    return process_file(vector_store, file, UnstructuredPowerPointLoader, ".pptx", stats_db=stats_db)

loaders/txt.py

@@ -1,5 +1,5 @@
 from .common import process_file
 from langchain.document_loaders import TextLoader
 
-def process_txt(vector_store, file):
-    return process_file(vector_store, file, TextLoader, ".txt")
+def process_txt(vector_store, file,stats_db):
+    return process_file(vector_store, file, TextLoader, ".txt", stats_db=stats_db)

main.py

@@ -10,12 +10,16 @@ from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.vectorstores import SupabaseVectorStore
 from supabase import Client, create_client
 from explorer import view_document
+from stats import get_usage_today
 
 supabase_url = st.secrets.supabase_url
 supabase_key = st.secrets.supabase_service_key
 openai_api_key = st.secrets.openai_api_key
 anthropic_api_key = st.secrets.anthropic_api_key
 supabase: Client = create_client(supabase_url, supabase_key)
+self_hosted = st.secrets.self_hosted
+
+
 
 embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
 vector_store = SupabaseVectorStore(
@@ -33,10 +37,25 @@ st.set_page_config(
 )
 
 
+    
+
 st.title("🧠 Quivr - Your second brain 🧠")
 st.markdown("Store your knowledge in a vector store and query it with OpenAI's GPT-3/4.")
 st.markdown("---\n\n")
 
+st.session_state["overused"] = False
+if self_hosted == "false":
+    usage = get_usage_today(supabase)
+    st.write(f"Usage today: {usage} tokens")
+    st.write("---")
+    if usage > st.secrets.usage_limit:
+        
+        st.error(
+            f"You have used {usage} tokens today, which is more than your daily limit of {st.secrets.usage_limit} tokens. Please come back in a few or self host.")
+        st.session_state["overused"] = True
+
+
+
 # Initialize session state variables
 if 'model' not in st.session_state:
     st.session_state['model'] = "gpt-3.5-turbo"
@@ -77,13 +96,22 @@ elif user_choice == 'Chat with your Brain':
     st.sidebar.title("Configuration")
     st.sidebar.markdown(
         "Choose your model and temperature for asking questions.")
-    st.session_state['model'] = st.sidebar.selectbox(
+    if st.secrets.self_hosted != "false":
+        st.session_state['model'] = st.sidebar.selectbox(
         "Select Model", models, index=(models).index(st.session_state['model']))
+    else:
+        st.sidebar.write("**Model**: gpt-3.5-turbo")
+        st.sidebar.write("**Self Host to unlock more models such as claude-v1 and GPT4**")
+        st.session_state['model'] = "gpt-3.5-turbo"
     st.session_state['temperature'] = st.sidebar.slider(
         "Select Temperature", 0.0, 1.0, st.session_state['temperature'], 0.1)
-    st.session_state['max_tokens'] = st.sidebar.slider(
-        "Select Max Tokens", 256, 2048, st.session_state['max_tokens'], 2048)
-    chat_with_doc(st.session_state['model'], vector_store)
+    if st.secrets.self_hosted != "false":
+        st.session_state['max_tokens'] = st.sidebar.slider(
+            "Select Max Tokens", 256, 2048, st.session_state['max_tokens'], 2048)
+    else:
+        st.session_state['max_tokens'] = 100
+    
+    chat_with_doc(st.session_state['model'], vector_store, stats_db=supabase)
 elif user_choice == 'Forget':
     st.sidebar.title("Configuration")
 

question.py

@@ -6,6 +6,7 @@ from langchain.memory import ConversationBufferMemory
 from langchain.llms import OpenAI
 from langchain.chat_models import ChatAnthropic
 from langchain.vectorstores import SupabaseVectorStore
+from stats import add_usage
 
 memory = ConversationBufferMemory(
     memory_key="chat_history", return_messages=True)
@@ -21,8 +22,8 @@ def count_tokens(question, model):
     return count
 
 
-def chat_with_doc(model, vector_store: SupabaseVectorStore):
-
+def chat_with_doc(model, vector_store: SupabaseVectorStore, stats_db):
+    
     if 'chat_history' not in st.session_state:
         st.session_state['chat_history'] = []
         
@@ -45,30 +46,34 @@ def chat_with_doc(model, vector_store: SupabaseVectorStore):
 
     if button:
         qa = None
-        if model.startswith("gpt"):
-            logger.info('Using OpenAI model %s', model)
-            qa = ConversationalRetrievalChain.from_llm(
-                OpenAI(
-                    model_name=st.session_state['model'], openai_api_key=openai_api_key, temperature=st.session_state['temperature'], max_tokens=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True)
-        elif anthropic_api_key and model.startswith("claude"):
-            logger.info('Using Anthropics model %s', model)
-            qa = ConversationalRetrievalChain.from_llm(
-                ChatAnthropic(
-                    model=st.session_state['model'], anthropic_api_key=anthropic_api_key, temperature=st.session_state['temperature'], max_tokens_to_sample=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True, max_tokens_limit=102400)
+        if not st.session_state["overused"]:
+            add_usage(stats_db, "chat", "prompt" + question, {"model": model, "temperature": st.session_state['temperature']})
+            if model.startswith("gpt"):
+                logger.info('Using OpenAI model %s', model)
+                qa = ConversationalRetrievalChain.from_llm(
+                    OpenAI(
+                        model_name=st.session_state['model'], openai_api_key=openai_api_key, temperature=st.session_state['temperature'], max_tokens=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True)
+            elif anthropic_api_key and model.startswith("claude"):
+                logger.info('Using Anthropics model %s', model)
+                qa = ConversationalRetrievalChain.from_llm(
+                    ChatAnthropic(
+                        model=st.session_state['model'], anthropic_api_key=anthropic_api_key, temperature=st.session_state['temperature'], max_tokens_to_sample=st.session_state['max_tokens']), vector_store.as_retriever(), memory=memory, verbose=True, max_tokens_limit=102400)
+            
+            
+            st.session_state['chat_history'].append(("You", question))
+
+            # Generate model's response and add it to chat history
+            model_response = qa({"question": question})
+            logger.info('Result: %s', model_response)
+
+            st.session_state['chat_history'].append(("Quivr", model_response["answer"]))
+
+            # Display chat history
+            st.empty()
+            for speaker, text in st.session_state['chat_history']:
+                st.markdown(f"**{speaker}:** {text}")
+        else:
+            st.error("You have used all your free credits. Please try again later or self host.")
         
-        
-        st.session_state['chat_history'].append(("You", question))
-
-        # Generate model's response and add it to chat history
-        model_response = qa({"question": question})
-        logger.info('Result: %s', model_response)
-
-        st.session_state['chat_history'].append(("Quivr", model_response["answer"]))
-
-        # Display chat history
-        st.empty()
-        for speaker, text in st.session_state['chat_history']:
-            st.markdown(f"**{speaker}:** {text}")
-
     if count_button:
         st.write(count_tokens(question, model))

stats.py

@@ -0,0 +1,31 @@
+from datetime import datetime, timedelta
+
+# -- Create a table called "stats"
+# create table
+#   stats (
+#     -- A column called "time" with data type "timestamp"
+#     time timestamp,
+#     -- A column called "details" with data type "text"
+#     chat boolean,
+#     embedding boolean,
+#     details text,
+#     metadata jsonb,
+#     -- An "integer" primary key column called "id" that is generated always as identity
+#     id integer primary key generated always as identity
+#   );
+
+
+def get_usage_today(supabase):
+    # Returns the number of rows in the stats table for the last 24 hours
+    response = supabase.table("stats").select("id", count="exact").gte("time", datetime.now() - timedelta(hours=24)).execute()
+    return response.count
+
+def add_usage(supabase, type, details, metadata):
+    # Adds a row to the stats table
+    supabase.table("stats").insert({
+        "time": datetime.now().isoformat(),
+        "chat": type == "chat",
+        "embedding": type == "embedding",
+        "details": details,
+        "metadata": metadata
+    }).execute()