refactor: reorg the files #2 (#2457)

# Description

Please include a summary of the changes and the related issue. Please
also include relevant motivation and context.

## Checklist before requesting a review

Please delete options that are not relevant.

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [ ] I have commented hard-to-understand areas
- [ ] I have ideally added tests that prove my fix is effective or that
my feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged

## Screenshots (if appropriate):

backend/modules/brain/api_brain_qa.py

@@ -6,8 +6,8 @@ import jq
 import requests
 from fastapi import HTTPException
 from litellm import completion
-from utils.call_brain_api import call_brain_api
-from utils.get_api_brain_definition_as_json_schema import (
+from modules.brain.service.call_brain_api import call_brain_api
+from modules.brain.service.get_api_brain_definition_as_json_schema import (
     get_api_brain_definition_as_json_schema,
 )
 from logger import get_logger

backend/modules/brain/knowledge_brain_qa.py

@@ -3,9 +3,9 @@ from typing import AsyncIterable, List, Optional
 from uuid import UUID
 
 from langchain.callbacks.streaming_aiter import AsyncIteratorCallbackHandler
-from utils.format_chat_history import format_chat_history
-from utils.get_prompt_to_use import get_prompt_to_use
-from utils.get_prompt_to_use_id import get_prompt_to_use_id
+from modules.brain.service.utils.format_chat_history import format_chat_history
+from modules.prompt.service.get_prompt_to_use import get_prompt_to_use
+from modules.brain.service.utils.get_prompt_to_use_id import get_prompt_to_use_id
 from logger import get_logger
 from models import BrainSettings
 from modules.user.service.user_usage import UserUsage

backend/modules/brain/qa_headless.py

@@ -8,12 +8,12 @@ from langchain.chains import LLMChain
 from langchain.chat_models.base import BaseChatModel
 from langchain.prompts.chat import ChatPromptTemplate, HumanMessagePromptTemplate
 from langchain_community.chat_models import ChatLiteLLM
-from utils.format_chat_history import (
+from modules.brain.service.utils.format_chat_history import (
     format_chat_history,
     format_history_to_openai_mesages,
 )
-from utils.get_prompt_to_use import get_prompt_to_use
-from utils.get_prompt_to_use_id import get_prompt_to_use_id
+from modules.prompt.service.get_prompt_to_use import get_prompt_to_use
+from modules.brain.service.utils.get_prompt_to_use_id import get_prompt_to_use_id
 from logger import get_logger
 from models import BrainSettings  # Importing settings related to the 'brain'
 from modules.brain.qa_interface import QAInterface

backend/modules/brain/rags/new_quivr_rag.py

@@ -12,7 +12,7 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate, PromptTemplate
 from langchain_core.runnables import RunnableParallel, RunnablePassthrough
 from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-from utils.get_prompt_to_use import get_prompt_to_use
+from modules.prompt.service.get_prompt_to_use import get_prompt_to_use
 from logger import get_logger
 from models import BrainSettings  # Importing settings related to the 'brain'
 from modules.brain.service.brain_service import BrainService

backend/modules/brain/rags/quivr_rag.py

@@ -14,7 +14,7 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate, PromptTemplate
 from langchain_core.runnables import RunnableLambda, RunnablePassthrough
 from langchain_openai import OpenAIEmbeddings
-from utils.get_prompt_to_use import get_prompt_to_use
+from modules.prompt.service.get_prompt_to_use import get_prompt_to_use
 from logger import get_logger
 from models import BrainSettings  # Importing settings related to the 'brain'
 from modules.brain.service.brain_service import BrainService

backend/modules/brain/service/call_brain_api.py

@@ -0,0 +1,111 @@
+from uuid import UUID
+import requests
+from logger import get_logger
+logger = get_logger(__name__)
+
+from fastapi import HTTPException
+
+from modules.brain.entity.api_brain_definition_entity import ApiBrainDefinitionSchema
+from modules.brain.service.api_brain_definition_service import ApiBrainDefinitionService
+from modules.brain.service.brain_service import BrainService
+
+brain_service = BrainService()
+api_brain_definition_service = ApiBrainDefinitionService()
+
+
+def get_api_call_response_as_text(
+    method, api_url, params, search_params, secrets
+) -> str:
+    headers = {}
+
+    api_url_with_search_params = api_url
+    if search_params:
+        api_url_with_search_params += "?"
+        for search_param in search_params:
+            api_url_with_search_params += (
+                f"{search_param}={search_params[search_param]}&"
+            )
+
+    for secret in secrets:
+        headers[secret] = secrets[secret]
+
+    try:
+        if method in ["GET", "DELETE"]:
+            response = requests.request(
+                method,
+                url=api_url_with_search_params,
+                params=params or None,
+                headers=headers or None,
+            )
+        elif method in ["POST", "PUT", "PATCH"]:
+            response = requests.request(
+                method,
+                url=api_url_with_search_params,
+                json=params or None,
+                headers=headers or None,
+            )
+        else:
+            raise ValueError(f"Invalid method: {method}")
+
+        return response.text
+
+    except Exception as e:
+        logger.error(f"Error calling API: {e}")
+        return None
+
+
+def extract_api_brain_definition_values_from_llm_output(
+    brain_schema: ApiBrainDefinitionSchema, arguments: dict
+) -> dict:
+    params_values = {}
+    properties = brain_schema.properties
+    required_values = brain_schema.required
+    for property in properties:
+        if property.name in arguments:
+            if property.type == "number":
+                params_values[property.name] = float(arguments[property.name])
+            else:
+                params_values[property.name] = arguments[property.name]
+            continue
+
+        if property.name in required_values:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Required parameter {property.name} not found in arguments",
+            )
+
+    return params_values
+
+
+def call_brain_api(brain_id: UUID, user_id: UUID, arguments: dict) -> str:
+    brain_definition = api_brain_definition_service.get_api_brain_definition(brain_id)
+
+    if brain_definition is None:
+        raise HTTPException(
+            status_code=404, detail=f"Brain definition {brain_id} not found"
+        )
+
+    brain_params_values = extract_api_brain_definition_values_from_llm_output(
+        brain_definition.params, arguments
+    )
+
+    brain_search_params_values = extract_api_brain_definition_values_from_llm_output(
+        brain_definition.search_params, arguments
+    )
+
+    secrets = brain_definition.secrets
+    secrets_values = {}
+
+    for secret in secrets:
+        secret_value = brain_service.external_api_secrets_repository.read_secret(
+            user_id=user_id, brain_id=brain_id, secret_name=secret.name
+        )
+        secrets_values[secret.name] = secret_value
+
+    return get_api_call_response_as_text(
+        api_url=brain_definition.url,
+        params=brain_params_values,
+        search_params=brain_search_params_values,
+        secrets=secrets_values,
+        method=brain_definition.method,
+    )

backend/modules/brain/service/get_api_brain_definition_as_json_schema.py

@@ -1,12 +1,30 @@
+import re
+
 from fastapi import HTTPException
-from utils.extract_api_definition import format_api_brain_property
-from utils.sanitize_function_name import sanitize_function_name
+
+from modules.brain.entity.api_brain_definition_entity import ApiBrainDefinitionSchemaProperty
 from modules.brain.entity.brain_entity import BrainEntity
 from modules.brain.service.api_brain_definition_service import ApiBrainDefinitionService
 
 api_brain_definition_service = ApiBrainDefinitionService()
 
 
+def sanitize_function_name(string):
+    sanitized_string = re.sub(r"[^a-zA-Z0-9_-]", "", string)
+
+    return sanitized_string
+
+
+def format_api_brain_property(property: ApiBrainDefinitionSchemaProperty):
+    property_data: dict = {
+        "type": property.type,
+        "description": property.description,
+    }
+    if property.enum:
+        property_data["enum"] = property.enum
+    return property_data
+
+
 def get_api_brain_definition_as_json_schema(brain: BrainEntity):
     api_brain_definition = api_brain_definition_service.get_api_brain_definition(
         brain.id

backend/modules/prompt/service/get_prompt_to_use.py

@@ -1,7 +1,7 @@
 from typing import Optional
 from uuid import UUID
 
-from utils.get_prompt_to_use_id import get_prompt_to_use_id
+from modules.brain.service.utils.get_prompt_to_use_id import get_prompt_to_use_id
 from modules.prompt.service import PromptService
 
 promptService = PromptService()

backend/utils/call_brain_api.py

@@ -1,46 +0,0 @@
-from uuid import UUID
-
-from fastapi import HTTPException
-from utils.extract_api_brain_definition_values_from_llm_output import (
-    extract_api_brain_definition_values_from_llm_output,
-)
-from utils.make_api_request import get_api_call_response_as_text
-from modules.brain.service.api_brain_definition_service import ApiBrainDefinitionService
-from modules.brain.service.brain_service import BrainService
-
-brain_service = BrainService()
-api_brain_definition_service = ApiBrainDefinitionService()
-
-
-def call_brain_api(brain_id: UUID, user_id: UUID, arguments: dict) -> str:
-    brain_definition = api_brain_definition_service.get_api_brain_definition(brain_id)
-
-    if brain_definition is None:
-        raise HTTPException(
-            status_code=404, detail=f"Brain definition {brain_id} not found"
-        )
-
-    brain_params_values = extract_api_brain_definition_values_from_llm_output(
-        brain_definition.params, arguments
-    )
-
-    brain_search_params_values = extract_api_brain_definition_values_from_llm_output(
-        brain_definition.search_params, arguments
-    )
-
-    secrets = brain_definition.secrets
-    secrets_values = {}
-
-    for secret in secrets:
-        secret_value = brain_service.external_api_secrets_repository.read_secret(
-            user_id=user_id, brain_id=brain_id, secret_name=secret.name
-        )
-        secrets_values[secret.name] = secret_value
-
-    return get_api_call_response_as_text(
-        api_url=brain_definition.url,
-        params=brain_params_values,
-        search_params=brain_search_params_values,
-        secrets=secrets_values,
-        method=brain_definition.method,
-    )

backend/utils/extract_api_brain_definition_values_from_llm_output.py

@@ -1,25 +0,0 @@
-from fastapi import HTTPException
-from modules.brain.entity.api_brain_definition_entity import ApiBrainDefinitionSchema
-
-
-def extract_api_brain_definition_values_from_llm_output(
-    brain_schema: ApiBrainDefinitionSchema, arguments: dict
-) -> dict:
-    params_values = {}
-    properties = brain_schema.properties
-    required_values = brain_schema.required
-    for property in properties:
-        if property.name in arguments:
-            if property.type == "number":
-                params_values[property.name] = float(arguments[property.name])
-            else:
-                params_values[property.name] = arguments[property.name]
-            continue
-
-        if property.name in required_values:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Required parameter {property.name} not found in arguments",
-            )
-
-    return params_values

backend/utils/extract_api_definition.py

@@ -1,13 +0,0 @@
-from modules.brain.entity.api_brain_definition_entity import (
-    ApiBrainDefinitionSchemaProperty,
-)
-
-
-def format_api_brain_property(property: ApiBrainDefinitionSchemaProperty):
-    property_data: dict = {
-        "type": property.type,
-        "description": property.description,
-    }
-    if property.enum:
-        property_data["enum"] = property.enum
-    return property_data

backend/utils/make_api_request.py

@@ -1,45 +0,0 @@
-import requests
-from logger import get_logger
-
-logger = get_logger(__name__)
-
-
-def get_api_call_response_as_text(
-    method, api_url, params, search_params, secrets
-) -> str:
-    headers = {}
-
-    api_url_with_search_params = api_url
-    if search_params:
-        api_url_with_search_params += "?"
-        for search_param in search_params:
-            api_url_with_search_params += (
-                f"{search_param}={search_params[search_param]}&"
-            )
-
-    for secret in secrets:
-        headers[secret] = secrets[secret]
-
-    try:
-        if method in ["GET", "DELETE"]:
-            response = requests.request(
-                method,
-                url=api_url_with_search_params,
-                params=params or None,
-                headers=headers or None,
-            )
-        elif method in ["POST", "PUT", "PATCH"]:
-            response = requests.request(
-                method,
-                url=api_url_with_search_params,
-                json=params or None,
-                headers=headers or None,
-            )
-        else:
-            raise ValueError(f"Invalid method: {method}")
-
-        return response.text
-
-    except Exception as e:
-        logger.error(f"Error calling API: {e}")
-        return None

backend/utils/sanitize_function_name.py

@@ -1,7 +0,0 @@
-import re
-
-
-def sanitize_function_name(string):
-    sanitized_string = re.sub(r"[^a-zA-Z0-9_-]", "", string)
-
-    return sanitized_string