quivr/backend/routes/upload_routes.py

94 lines
3.4 KiB
Python
Raw Normal View History

from typing import Optional
2023-06-28 20:39:27 +03:00
from uuid import UUID
Feat/user chat history (#275) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * 🎨 separate creation and update endpoints for chat * 🩹 add feat(chat): basic source documents support * ✨ add chatName upon creation and for chats list * 💄 improve chatsList style * User chat history and multiple chats (#290) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * refactor(chat): use layout to avoid refetching all chats on every chat * refactor(chat): useChats hook instead of useQuestion * fix(chat): fix errors * refactor(chat): better folder structure * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * style(chat): better looking sidebar * resume merge * fix(backend): add os and logger imports * small fixes * chore(chat): remove empty interface * chore(chat): use const * fix(chat): merge errors * fix(chat): remove useSpeech args * chore(chat): remove unused args * fix(chat): remove duplicate components --------- Co-authored-by: gozineb <zinebe@theodo.fr> Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> * fix and refactor errors * fix(fresh): installation issues * chore(conflict): merged old code * fix(multi-chat): use update endpoint * feat(embeddings): now using users api key --------- Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> Co-authored-by: Aditya Nandan <61308761+iMADi-ARCH@users.noreply.github.com> Co-authored-by: iMADi-ARCH <nandanaditya985@gmail.com> Co-authored-by: Mamadou DICKO <mamadoudicko100@gmail.com>
2023-06-11 00:59:16 +03:00
from auth import AuthBearer, get_current_user
from celery_worker import process_file_and_notify
2023-06-28 20:39:27 +03:00
from fastapi import APIRouter, Depends, Query, Request, UploadFile
from models import Brain, UserIdentity, UserUsage
from models.databases.supabase.notifications import (
CreateNotificationProperties,
)
from models.notifications import NotificationsStatusEnum
from repository.brain import get_brain_details
from repository.files.upload_file import upload_file_storage
from repository.notification.add_notification import add_notification
from repository.user_identity import get_user_identity
from utils.file import convert_bytes, get_file_size
from routes.authorizations.brain_authorization import (
RoleEnum,
validate_brain_authorization,
)
Feat/user chat history (#275) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * 🎨 separate creation and update endpoints for chat * 🩹 add feat(chat): basic source documents support * ✨ add chatName upon creation and for chats list * 💄 improve chatsList style * User chat history and multiple chats (#290) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * refactor(chat): use layout to avoid refetching all chats on every chat * refactor(chat): useChats hook instead of useQuestion * fix(chat): fix errors * refactor(chat): better folder structure * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * style(chat): better looking sidebar * resume merge * fix(backend): add os and logger imports * small fixes * chore(chat): remove empty interface * chore(chat): use const * fix(chat): merge errors * fix(chat): remove useSpeech args * chore(chat): remove unused args * fix(chat): remove duplicate components --------- Co-authored-by: gozineb <zinebe@theodo.fr> Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> * fix and refactor errors * fix(fresh): installation issues * chore(conflict): merged old code * fix(multi-chat): use update endpoint * feat(embeddings): now using users api key --------- Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> Co-authored-by: Aditya Nandan <61308761+iMADi-ARCH@users.noreply.github.com> Co-authored-by: iMADi-ARCH <nandanaditya985@gmail.com> Co-authored-by: Mamadou DICKO <mamadoudicko100@gmail.com>
2023-06-11 00:59:16 +03:00
upload_router = APIRouter()
@upload_router.get("/upload/healthz", tags=["Health"])
async def healthz():
return {"status": "ok"}
@upload_router.post("/upload", dependencies=[Depends(AuthBearer())], tags=["Upload"])
async def upload_file(
request: Request,
uploadFile: UploadFile,
brain_id: UUID = Query(..., description="The ID of the brain"),
chat_id: Optional[UUID] = Query(None, description="The ID of the chat"),
enable_summarization: bool = False,
current_user: UserIdentity = Depends(get_current_user),
):
validate_brain_authorization(
brain_id, current_user.id, [RoleEnum.Editor, RoleEnum.Owner]
)
brain = Brain(id=brain_id)
userDailyUsage = UserUsage(
id=current_user.id,
email=current_user.email,
openai_api_key=current_user.openai_api_key,
)
userSettings = userDailyUsage.get_user_settings()
Feat/user chat history (#275) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * 🎨 separate creation and update endpoints for chat * 🩹 add feat(chat): basic source documents support * ✨ add chatName upon creation and for chats list * 💄 improve chatsList style * User chat history and multiple chats (#290) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * refactor(chat): use layout to avoid refetching all chats on every chat * refactor(chat): useChats hook instead of useQuestion * fix(chat): fix errors * refactor(chat): better folder structure * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * style(chat): better looking sidebar * resume merge * fix(backend): add os and logger imports * small fixes * chore(chat): remove empty interface * chore(chat): use const * fix(chat): merge errors * fix(chat): remove useSpeech args * chore(chat): remove unused args * fix(chat): remove duplicate components --------- Co-authored-by: gozineb <zinebe@theodo.fr> Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> * fix and refactor errors * fix(fresh): installation issues * chore(conflict): merged old code * fix(multi-chat): use update endpoint * feat(embeddings): now using users api key --------- Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> Co-authored-by: Aditya Nandan <61308761+iMADi-ARCH@users.noreply.github.com> Co-authored-by: iMADi-ARCH <nandanaditya985@gmail.com> Co-authored-by: Mamadou DICKO <mamadoudicko100@gmail.com>
2023-06-11 00:59:16 +03:00
if request.headers.get("Openai-Api-Key"):
brain.max_brain_size = userSettings.get("max_brain_size", 1000000000)
remaining_free_space = userSettings.get("max_brain_size", 1000000000)
2023-06-28 20:39:27 +03:00
file_size = get_file_size(uploadFile)
Feat/user chat history (#275) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * 🩹 add setQuestion to speach utils * 🎨 separate creation and update endpoints for chat * 🩹 add feat(chat): basic source documents support * ✨ add chatName upon creation and for chats list * 💄 improve chatsList style * User chat history and multiple chats (#290) * ♻️ refactor backend main routes * 🗃️ new user_id uuid column in users table * 🗃️ new chats table * ✨ new chat endpoints * ✨ change chat routes post to handle undef chat_id * ♻️ extract components from chat page * ✨ add chatId to useQuestion * ✨ new ChatsList * ✨ new optional dynamic route chat/{chat_id} * refactor(chat): use layout to avoid refetching all chats on every chat * refactor(chat): useChats hook instead of useQuestion * fix(chat): fix errors * refactor(chat): better folder structure * feat: self supplied key (#286) * feat(brain): increased size if api key and more * fix(key): not displayed * feat(apikey): now password input * fix(twitter): handle wrong * feat(chat): basic source documents support (#289) * style(chat): better looking sidebar * resume merge * fix(backend): add os and logger imports * small fixes * chore(chat): remove empty interface * chore(chat): use const * fix(chat): merge errors * fix(chat): remove useSpeech args * chore(chat): remove unused args * fix(chat): remove duplicate components --------- Co-authored-by: gozineb <zinebe@theodo.fr> Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> * fix and refactor errors * fix(fresh): installation issues * chore(conflict): merged old code * fix(multi-chat): use update endpoint * feat(embeddings): now using users api key --------- Co-authored-by: Matt <77928207+mattzcarey@users.noreply.github.com> Co-authored-by: Stan Girard <girard.stanislas@gmail.com> Co-authored-by: xleven <xleven@outlook.com> Co-authored-by: Aditya Nandan <61308761+iMADi-ARCH@users.noreply.github.com> Co-authored-by: iMADi-ARCH <nandanaditya985@gmail.com> Co-authored-by: Mamadou DICKO <mamadoudicko100@gmail.com>
2023-06-11 00:59:16 +03:00
if remaining_free_space - file_size < 0:
message = {
"message": f"❌ UserIdentity's brain will exceed maximum capacity with this upload. Maximum file allowed is : {convert_bytes(remaining_free_space)}",
"type": "error",
}
return message
upload_notification = None
if chat_id:
upload_notification = add_notification(
CreateNotificationProperties(
action="UPLOAD",
chat_id=chat_id,
status=NotificationsStatusEnum.Pending,
)
)
openai_api_key = request.headers.get("Openai-Api-Key", None)
if openai_api_key is None:
brain_details = get_brain_details(brain_id)
if brain_details:
openai_api_key = brain_details.openai_api_key
if openai_api_key is None:
openai_api_key = get_user_identity(current_user.id).openai_api_key
file_content = await uploadFile.read()
# filename_with_brain_id = str(brain_id) + "/" + str(uploadFile.filename)
filename_with_brain_id = str(brain_id) + "/" + str(uploadFile.filename)
upload_file_storage(file_content, filename_with_brain_id)
process_file_and_notify.delay(
file_name=filename_with_brain_id,
file_original_name=uploadFile.filename,
enable_summarization=enable_summarization,
brain_id=brain_id,
openai_api_key=openai_api_key,
notification_id=upload_notification.id if upload_notification else None,
)
return {"message": "File processing has started."}