delete unused code

backend/core/parsers/html.py

@@ -1,6 +1,3 @@
-import re
-import unicodedata
-
 import requests
 from langchain.document_loaders import UnstructuredHTMLLoader
 from models.files import File
@@ -9,8 +6,17 @@ from models.settings import CommonsDep
 from .common import process_file
 
 
-def process_html(commons: CommonsDep, file: File, enable_summarization, brain_id, user_openai_api_key):
-    return process_file(commons, file, UnstructuredHTMLLoader,  enable_summarization, brain_id, user_openai_api_key)
+def process_html(
+    commons: CommonsDep, file: File, enable_summarization, brain_id, user_openai_api_key
+):
+    return process_file(
+        commons,
+        file,
+        UnstructuredHTMLLoader,
+        enable_summarization,
+        brain_id,
+        user_openai_api_key,
+    )
 
 
 def get_html(url):
@@ -19,11 +25,3 @@ def get_html(url):
         return response.text
     else:
         return None
-
-
-def slugify(text):
-    text = unicodedata.normalize('NFKD', text).encode(
-        'ascii', 'ignore').decode('utf-8')
-    text = re.sub(r'[^\w\s-]', '', text).strip().lower()
-    text = re.sub(r'[-\s]+', '-', text)
-    return text

backend/core/routes/user_routes.py

@@ -18,11 +18,6 @@ user_router = APIRouter()
 MAX_BRAIN_SIZE_WITH_OWN_KEY = int(os.getenv("MAX_BRAIN_SIZE_WITH_KEY", 209715200))
 
 
-def get_unique_documents(vectors):
-    # Convert each dictionary to a tuple of items, then to a set to remove duplicates, and then back to a dictionary
-    return [dict(t) for t in set(tuple(d.items()) for d in vectors)]
-
-
 @user_router.get("/user", dependencies=[Depends(AuthBearer())], tags=["User"])
 async def get_user_endpoint(
     request: Request, current_user: User = Depends(get_current_user)

backend/core/utils/chats.py

@@ -1,14 +0,0 @@
-from logger import get_logger
-from models.chats import ChatMessage
-from models.settings import common_dependencies
-
-logger = get_logger(__name__)
-
-
-def get_chat_name_from_first_question(chat_message: ChatMessage):
-    # Step 1: Get the summary of the first question
-    # first_question_summary = summarize_as_title(chat_message.question)
-    # Step 2: Process this summary to create a chat name by selecting the first three words
-    chat_name = " ".join(chat_message.question.split()[:3])
-
-    return chat_name

backend/core/utils/constants.py

@@ -1,8 +0,0 @@
-openai_function_compatible_models = [
-    "gpt-3.5-turbo-0613",
-    "gpt-4-0613",
-]
-
-streaming_compatible_models = ["gpt-3.5-turbo, gpt4all-j-1.3"]
-
-private_models = ["gpt4all-j-1.3"]