feat(refacto): changed a bit of things to make better dx (#984)

.github/workflows/aws-preview.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "main" ]
     paths:
-      - 'backend/core/**'
+      - 'backend/**'
 
 env:
   AWS_REGION: eu-west-3
@@ -56,7 +56,7 @@ jobs:
         ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
         IMAGE_TAG: ${{ github.sha }}
       with:
-        context: ./backend/core/
+        context: ./backend/
         push: true
         tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}, ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest, ghcr.io/stangirard/quivr:latest
         cache-from: type=gha

.github/workflows/aws.yml

@@ -55,7 +55,7 @@ jobs:
         ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
         IMAGE_TAG: ${{ github.sha }}
       with:
-        context: ./backend/core/
+        context: ./backend/
         push: true
         tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}, ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ github.ref_name}},  ghcr.io/stangirard/quivr:${{ github.ref_name}}
         cache-from: type=gha

.github/workflows/backend-tests.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches: [main]
     paths:
-      - 'backend/core/**'
+      - 'backend/**'
   pull_request:
     branches: [main]
     paths:
-      - 'backend/core/**'
+      - 'backend/**'
   workflow_dispatch:
     inputs:
       logLevel:
@@ -34,7 +34,7 @@ jobs:
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: ./backend/core
+        working-directory: ./backend
     environment: preview
     strategy:
       matrix:

README.md

@@ -95,11 +95,11 @@ Additionally, you'll need a [Supabase](https://supabase.com/) account for:
 - **Step 2 - Bis**: Copy the `.XXXXX_env` files
 
   ```bash
-  cp .backend_env.example backend/core/.env
+  cp .backend_env.example backend/.env
   cp .frontend_env.example frontend/.env
   ```
 
-- **Step 3**: Update the `backend/core/.env` and `frontend/.env` file
+- **Step 3**: Update the `backend/.env` and `frontend/.env` file
 
   > _Your `supabase_service_key` can be found in your Supabase dashboard under Project Settings -> API. Use the `anon` `public` key found in the `Project API keys` section._
 
@@ -107,9 +107,9 @@ Additionally, you'll need a [Supabase](https://supabase.com/) account for:
 
   > _The `NEXT_PUBLIC_BACKEND_URL` is set to localhost:5050 for the docker. Update it if you are running the backend on a different machine._
 
-  > _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/core/.env`- It is an advanced feature, please be expert in GCP before trying to use it_
+  > _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/.env`- It is an advanced feature, please be expert in GCP before trying to use it_
 
-  - [ ] Change variables in `backend/core/.env`
+  - [ ] Change variables in `backend/.env`
   - [ ] Change variables in `frontend/.env`
 
 - **Step 4**: Use the `migration.sh` script to run the migration scripts

backend/private/Dockerfile

@@ -1,20 +0,0 @@
-# Use the same base image as the 'backend-core' container
-FROM python:3.11-bullseye
-
-# Install necessary packages
-RUN apt-get update && apt-get install -y liblzma-dev cmake git
-
-# Set the working directory
-WORKDIR /app
-
-# Copy the requirements file
-COPY ./requirements.txt /app/requirements.txt
-
-# Install Python dependencies
-RUN pip install --no-cache-dir -r /app/requirements.txt --timeout 100
-
-# Copy your application's code to the Docker container
-COPY . /app
-
-# Start the Uvicorn server on port 5051
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "5051"]

backend/private/logger.py

@@ -1,17 +0,0 @@
-import logging
-
-
-def get_logger(logger_name, log_level=logging.INFO):
-    logger = logging.getLogger(logger_name)
-    logger.setLevel(log_level)
-    logger.propagate = False  # Prevent log propagation to avoid double logging
-
-    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")
-
-    console_handler = logging.StreamHandler()
-    console_handler.setFormatter(formatter)
-
-    if not logger.handlers:
-        logger.addHandler(console_handler)
-
-    return logger

backend/private/main.py

@@ -1,30 +0,0 @@
-import os
-
-import sentry_sdk
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import JSONResponse
-from logger import get_logger
-from routes.completions_routes import completions_router
-
-logger = get_logger(__name__)
-
-if os.getenv("SENTRY_DSN"):
-    sentry_sdk.init(
-        dsn=os.getenv("SENTRY_DSN"),
-        # Set traces_sample_rate to 1.0 to capture 100%
-        # of transactions for performance monitoring.
-        # We recommend adjusting this value in production,
-        traces_sample_rate=1.0,
-    )
-
-app = FastAPI()
-
-app.include_router(completions_router)
-
-
-@app.exception_handler(HTTPException)
-async def http_exception_handler(_, exc):
-    return JSONResponse(
-        status_code=exc.status_code,
-        content={"detail": exc.detail},
-    )

backend/private/models/messages.py

@@ -1,18 +0,0 @@
-from typing import List
-
-from pydantic import BaseModel, Field
-
-
-class Message(BaseModel):
-    role: str = Field(
-        ...,
-        description="The role of the messages author. One of system, user, assistant, or function.",
-    )
-    content: str = Field(
-        ...,
-        description="The contents of the message. content is required for all messages, and may be null for assistant messages with function calls.",
-    )
-
-
-class Messages(BaseModel):
-    messages: List[Message]

backend/private/requirements.txt

@@ -1,11 +0,0 @@
-langchain==0.0.228
-tiktoken==0.4.0
-fastapi==0.95.2
-uvicorn==0.22.0
-transformers==4.30.1
-asyncpg==0.27.0
-flake8==6.0.0
-flake8-black==0.3.6
-sentence_transformers>=2.0.0
-sentry-sdk==1.26.0
-pyright==1.1.316

backend/private/routes/completions_routes.py

@@ -1,18 +0,0 @@
-from fastapi import APIRouter
-from logger import get_logger
-from models.messages import Messages
-
-logger = get_logger(__name__)
-
-completions_router = APIRouter()
-
-
-@completions_router.post("/chat/completions", tags=["Chat Completions"])
-async def post_chat_completions(
-    model: str,  # make union of all models
-    messages: Messages,
-) -> str:
-    logger.info(
-        f"Received chat completions request for {model} with messages {messages}"
-    )
-    return f"Hello from {model}!"

backend/private/routes/embeddings_routes.py

@@ -1,16 +0,0 @@
-from fastapi import APIRouter
-from logger import getLogger
-
-logger = getLogger(__name__)
-
-embeddings_router = APIRouter()
-
-
-@embeddings_router.post("/embeddings", tags=["Embeddings"])
-async def post_embeddings(
-    model: str,
-    input: str | list[str],
-) -> str:
-    logger.info(f"Received embeddings request for {model} with input {input}")
-
-    return f"Hello from embeddings {model}!"