2023-07-05 19:15:18 +03:00
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
2023-07-05 15:37:01 +03:00
|
|
|
from typing import List
|
2023-08-21 13:45:32 +03:00
|
|
|
from uuid import UUID
|
2023-07-05 15:37:01 +03:00
|
|
|
|
2023-05-22 09:39:55 +03:00
|
|
|
from logger import get_logger
|
2023-08-03 21:24:42 +03:00
|
|
|
from models.settings import get_documents_vector_store, get_embeddings, get_supabase_db
|
2023-09-14 12:56:59 +03:00
|
|
|
from pydantic import BaseModel
|
2023-05-22 09:39:55 +03:00
|
|
|
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
2023-06-19 22:15:35 +03:00
|
|
|
|
2023-11-14 16:31:02 +03:00
|
|
|
# TODO: Create interface for embeddings and implement it for Supabase and OpenAI (current Quivr)
|
2023-06-22 18:50:06 +03:00
|
|
|
class Neurons(BaseModel):
|
2023-12-04 13:59:08 +03:00
|
|
|
def create_vector(self, docs):
|
2023-08-03 21:24:42 +03:00
|
|
|
documents_vector_store = get_documents_vector_store()
|
2023-11-19 20:46:12 +03:00
|
|
|
|
2023-06-19 22:15:35 +03:00
|
|
|
try:
|
2023-12-04 13:59:08 +03:00
|
|
|
sids = documents_vector_store.add_documents(docs)
|
2023-06-19 22:15:35 +03:00
|
|
|
if sids and len(sids) > 0:
|
2023-06-28 20:39:27 +03:00
|
|
|
return sids
|
|
|
|
|
2023-06-19 22:15:35 +03:00
|
|
|
except Exception as e:
|
|
|
|
logger.error(f"Error creating vector for document {e}")
|
|
|
|
|
|
|
|
def create_embedding(self, content):
|
2023-08-03 21:24:42 +03:00
|
|
|
embeddings = get_embeddings()
|
|
|
|
return embeddings.embed_query(content)
|
2023-06-19 22:15:35 +03:00
|
|
|
|
|
|
|
|
2023-07-05 15:37:01 +03:00
|
|
|
def error_callback(exception):
|
2023-07-10 15:27:49 +03:00
|
|
|
print("An exception occurred:", exception)
|
2023-07-05 15:37:01 +03:00
|
|
|
|
|
|
|
|
2023-07-12 13:44:34 +03:00
|
|
|
def process_batch(batch_ids: List[str]):
|
2023-08-03 21:24:42 +03:00
|
|
|
supabase_db = get_supabase_db()
|
|
|
|
|
2023-07-12 13:44:34 +03:00
|
|
|
try:
|
|
|
|
if len(batch_ids) == 1:
|
2023-08-21 13:45:32 +03:00
|
|
|
return (supabase_db.get_vectors_by_batch(UUID(batch_ids[0]))).data
|
2023-07-12 13:44:34 +03:00
|
|
|
else:
|
2023-08-03 21:24:42 +03:00
|
|
|
return (supabase_db.get_vectors_in_batch(batch_ids)).data
|
2023-07-12 13:44:34 +03:00
|
|
|
except Exception as e:
|
|
|
|
logger.error("Error retrieving batched vectors", e)
|
|
|
|
|
|
|
|
|
2023-11-14 16:31:02 +03:00
|
|
|
# TODO: move to Knowledge class
|
2023-12-01 00:29:28 +03:00
|
|
|
def get_unique_files_from_vector_ids(vectors_ids):
|
2023-12-05 02:10:49 +03:00
|
|
|
# Move into Vectors class
|
2023-07-05 15:37:01 +03:00
|
|
|
"""
|
|
|
|
Retrieve unique user data vectors.
|
|
|
|
"""
|
|
|
|
|
|
|
|
# constants
|
|
|
|
BATCH_SIZE = 5
|
|
|
|
|
2023-07-05 19:15:18 +03:00
|
|
|
with ThreadPoolExecutor() as executor:
|
|
|
|
futures = []
|
|
|
|
for i in range(0, len(vectors_ids), BATCH_SIZE):
|
2023-07-10 15:27:49 +03:00
|
|
|
batch_ids = vectors_ids[i : i + BATCH_SIZE]
|
2023-07-05 19:15:18 +03:00
|
|
|
future = executor.submit(process_batch, batch_ids)
|
|
|
|
futures.append(future)
|
2023-07-05 15:37:01 +03:00
|
|
|
|
2023-07-05 19:15:18 +03:00
|
|
|
# Retrieve the results
|
|
|
|
vectors_responses = [future.result() for future in futures]
|
2023-07-10 15:27:49 +03:00
|
|
|
|
2023-07-05 15:37:01 +03:00
|
|
|
documents = [item for sublist in vectors_responses for item in sublist]
|
|
|
|
unique_files = [dict(t) for t in set(tuple(d.items()) for d in documents)]
|
|
|
|
return unique_files
|