From e61f437ce868ff742c6eaa2dac2101a021294d09 Mon Sep 17 00:00:00 2001 From: Matt <77928207+mattzcarey@users.noreply.github.com> Date: Mon, 17 Jul 2023 07:57:27 +0100 Subject: [PATCH] Feat/backend core (#656) --- .github/workflows/aws-preview.yml | 4 +- .github/workflows/aws.yml | 2 +- .github/workflows/backend-tests.yml | 6 +-- .github/workflows/code-review-gpt.yml | 3 +- .gitignore | 6 +-- .vscode/settings.json | 4 +- README.md | 8 ++-- backend/{ => core}/Dockerfile | 8 ++-- backend/{ => core}/auth/__init__.py | 0 backend/{ => core}/auth/api_key_handler.py | 0 backend/{ => core}/auth/auth_bearer.py | 5 +-- backend/{ => core}/auth/jwt_token_handler.py | 0 backend/{ => core}/crawl/__init__.py | 0 backend/{ => core}/crawl/crawler.py | 0 backend/{ => core}/llm/__init__.py | 0 backend/{ => core}/llm/base.py | 0 backend/{ => core}/llm/models/FunctionCall.py | 0 backend/{ => core}/llm/models/OpenAiAnswer.py | 0 backend/{ => core}/llm/openai.py | 0 backend/{ => core}/llm/openai_functions.py | 0 backend/{ => core}/llm/private_gpt4all.py | 1 + .../{ => core}/llm/prompts/CONDENSE_PROMPT.py | 0 .../{ => core}/llm/prompts/LANGUAGE_PROMPT.py | 0 backend/{ => core}/llm/qa_base.py | 0 backend/{ => core}/llm/utils/summarization.py | 0 backend/{ => core}/logger.py | 0 backend/{ => core}/main.py | 0 backend/{ => core}/middlewares/cors.py | 2 +- backend/{ => core}/models/__init__.py | 0 backend/{ => core}/models/brains.py | 0 .../models/brains_subscription_invitations.py | 37 ++++++++++----- backend/{ => core}/models/chat.py | 0 backend/{ => core}/models/chats.py | 0 backend/{ => core}/models/files.py | 0 backend/{ => core}/models/settings.py | 0 backend/{ => core}/models/users.py | 0 backend/{ => core}/parsers/__init__.py | 0 backend/{ => core}/parsers/audio.py | 0 backend/{ => core}/parsers/common.py | 0 backend/{ => core}/parsers/csv.py | 0 backend/{ => core}/parsers/docx.py | 0 backend/{ => core}/parsers/epub.py | 0 backend/{ => core}/parsers/github.py | 0 backend/{ => core}/parsers/html.py | 0 backend/{ => core}/parsers/markdown.py | 0 backend/{ => core}/parsers/notebook.py | 0 backend/{ => core}/parsers/odt.py | 0 backend/{ => core}/parsers/pdf.py | 0 backend/{ => core}/parsers/powerpoint.py | 0 backend/{ => core}/parsers/txt.py | 0 backend/{ => core}/pyrightconfig.json | 0 backend/{ => core}/repository/__init__.py | 0 .../{ => core}/repository/chat/__init__.py | 0 .../{ => core}/repository/chat/create_chat.py | 0 .../repository/chat/format_chat_history.py | 0 .../repository/chat/get_chat_by_id.py | 0 .../repository/chat/get_chat_history.py | 0 .../repository/chat/get_user_chats.py | 0 .../{ => core}/repository/chat/update_chat.py | 0 .../repository/chat/update_chat_history.py | 0 .../repository/chat/update_message_by_id.py | 0 backend/{ => core}/requirements.txt | 0 backend/{ => core}/routes/__init__.py | 0 backend/{ => core}/routes/api_key_routes.py | 0 .../routes/authorizations/__init__.py | 0 .../authorizations/brain_authorization.py | 0 backend/{ => core}/routes/brain_routes.py | 0 backend/{ => core}/routes/chat_routes.py | 0 backend/{ => core}/routes/crawl_routes.py | 0 backend/{ => core}/routes/explore_routes.py | 1 - backend/{ => core}/routes/misc_routes.py | 0 .../{ => core}/routes/subscription_routes.py | 0 backend/{ => core}/routes/upload_routes.py | 0 backend/{ => core}/routes/user_routes.py | 0 backend/{ => core}/scripts/start.sh | 0 .../supabase/functions/add-new-email/index.ts | 2 +- backend/{ => core}/tests/conftest.py | 0 backend/{ => core}/tests/test_api_key.py | 0 backend/{ => core}/tests/test_brains.py | 0 backend/{ => core}/tests/test_chats.py | 0 backend/{ => core}/tests/test_explore.py | 0 backend/{ => core}/tests/test_files/test.csv | 0 backend/{ => core}/tests/test_files/test.pdf | Bin backend/{ => core}/tests/test_files/test.txt | 0 backend/{ => core}/tests/test_upload.py | 0 backend/{ => core}/tests/test_user.py | 0 backend/{ => core}/utils/__init__.py | 0 backend/{ => core}/utils/chats.py | 0 backend/{ => core}/utils/constants.py | 0 backend/{ => core}/utils/file.py | 0 backend/{ => core}/utils/processors.py | 0 backend/{ => core}/utils/users.py | 0 backend/{ => core}/utils/vectors.py | 0 backend/{ => core}/vectorstore/__init__.py | 0 backend/{ => core}/vectorstore/supabase.py | 0 backend/private/Dockerfile | 20 +++++++++ backend/private/logger.py | 17 +++++++ backend/private/main.py | 30 +++++++++++++ backend/private/models/messages.py | 18 ++++++++ backend/private/requirements.txt | 11 +++++ backend/private/routes/completions_routes.py | 18 ++++++++ backend/private/routes/embeddings_routes.py | 16 +++++++ docker-compose.dev.yml | 10 ++--- docker-compose.private-dev.yml | 42 ++++++++++++++++++ docker-compose.yml | 10 ++--- install_helper.sh | 30 ++++++------- 106 files changed, 249 insertions(+), 62 deletions(-) rename backend/{ => core}/Dockerfile (54%) rename backend/{ => core}/auth/__init__.py (100%) rename backend/{ => core}/auth/api_key_handler.py (100%) rename backend/{ => core}/auth/auth_bearer.py (99%) rename backend/{ => core}/auth/jwt_token_handler.py (100%) rename backend/{ => core}/crawl/__init__.py (100%) rename backend/{ => core}/crawl/crawler.py (100%) rename backend/{ => core}/llm/__init__.py (100%) rename backend/{ => core}/llm/base.py (100%) rename backend/{ => core}/llm/models/FunctionCall.py (100%) rename backend/{ => core}/llm/models/OpenAiAnswer.py (100%) rename backend/{ => core}/llm/openai.py (100%) rename backend/{ => core}/llm/openai_functions.py (100%) rename backend/{ => core}/llm/private_gpt4all.py (99%) rename backend/{ => core}/llm/prompts/CONDENSE_PROMPT.py (100%) rename backend/{ => core}/llm/prompts/LANGUAGE_PROMPT.py (100%) rename backend/{ => core}/llm/qa_base.py (100%) rename backend/{ => core}/llm/utils/summarization.py (100%) rename backend/{ => core}/logger.py (100%) rename backend/{ => core}/main.py (100%) rename backend/{ => core}/middlewares/cors.py (98%) rename backend/{ => core}/models/__init__.py (100%) rename backend/{ => core}/models/brains.py (100%) rename backend/{ => core}/models/brains_subscription_invitations.py (69%) rename backend/{ => core}/models/chat.py (100%) rename backend/{ => core}/models/chats.py (100%) rename backend/{ => core}/models/files.py (100%) rename backend/{ => core}/models/settings.py (100%) rename backend/{ => core}/models/users.py (100%) rename backend/{ => core}/parsers/__init__.py (100%) rename backend/{ => core}/parsers/audio.py (100%) rename backend/{ => core}/parsers/common.py (100%) rename backend/{ => core}/parsers/csv.py (100%) rename backend/{ => core}/parsers/docx.py (100%) rename backend/{ => core}/parsers/epub.py (100%) rename backend/{ => core}/parsers/github.py (100%) rename backend/{ => core}/parsers/html.py (100%) rename backend/{ => core}/parsers/markdown.py (100%) rename backend/{ => core}/parsers/notebook.py (100%) rename backend/{ => core}/parsers/odt.py (100%) rename backend/{ => core}/parsers/pdf.py (100%) rename backend/{ => core}/parsers/powerpoint.py (100%) rename backend/{ => core}/parsers/txt.py (100%) rename backend/{ => core}/pyrightconfig.json (100%) rename backend/{ => core}/repository/__init__.py (100%) rename backend/{ => core}/repository/chat/__init__.py (100%) rename backend/{ => core}/repository/chat/create_chat.py (100%) rename backend/{ => core}/repository/chat/format_chat_history.py (100%) rename backend/{ => core}/repository/chat/get_chat_by_id.py (100%) rename backend/{ => core}/repository/chat/get_chat_history.py (100%) rename backend/{ => core}/repository/chat/get_user_chats.py (100%) rename backend/{ => core}/repository/chat/update_chat.py (100%) rename backend/{ => core}/repository/chat/update_chat_history.py (100%) rename backend/{ => core}/repository/chat/update_message_by_id.py (100%) rename backend/{ => core}/requirements.txt (100%) rename backend/{ => core}/routes/__init__.py (100%) rename backend/{ => core}/routes/api_key_routes.py (100%) rename backend/{ => core}/routes/authorizations/__init__.py (100%) rename backend/{ => core}/routes/authorizations/brain_authorization.py (100%) rename backend/{ => core}/routes/brain_routes.py (100%) rename backend/{ => core}/routes/chat_routes.py (100%) rename backend/{ => core}/routes/crawl_routes.py (100%) rename backend/{ => core}/routes/explore_routes.py (99%) rename backend/{ => core}/routes/misc_routes.py (100%) rename backend/{ => core}/routes/subscription_routes.py (100%) rename backend/{ => core}/routes/upload_routes.py (100%) rename backend/{ => core}/routes/user_routes.py (100%) rename backend/{ => core}/scripts/start.sh (100%) rename backend/{ => core}/supabase/functions/add-new-email/index.ts (95%) rename backend/{ => core}/tests/conftest.py (100%) rename backend/{ => core}/tests/test_api_key.py (100%) rename backend/{ => core}/tests/test_brains.py (100%) rename backend/{ => core}/tests/test_chats.py (100%) rename backend/{ => core}/tests/test_explore.py (100%) rename backend/{ => core}/tests/test_files/test.csv (100%) rename backend/{ => core}/tests/test_files/test.pdf (100%) rename backend/{ => core}/tests/test_files/test.txt (100%) rename backend/{ => core}/tests/test_upload.py (100%) rename backend/{ => core}/tests/test_user.py (100%) rename backend/{ => core}/utils/__init__.py (100%) rename backend/{ => core}/utils/chats.py (100%) rename backend/{ => core}/utils/constants.py (100%) rename backend/{ => core}/utils/file.py (100%) rename backend/{ => core}/utils/processors.py (100%) rename backend/{ => core}/utils/users.py (100%) rename backend/{ => core}/utils/vectors.py (100%) rename backend/{ => core}/vectorstore/__init__.py (100%) rename backend/{ => core}/vectorstore/supabase.py (100%) create mode 100644 backend/private/Dockerfile create mode 100644 backend/private/logger.py create mode 100644 backend/private/main.py create mode 100644 backend/private/models/messages.py create mode 100644 backend/private/requirements.txt create mode 100644 backend/private/routes/completions_routes.py create mode 100644 backend/private/routes/embeddings_routes.py create mode 100644 docker-compose.private-dev.yml diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index a2ad85044..3060b1834 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -4,7 +4,7 @@ on: push: branches: [ "main" ] paths: - - 'backend/**' + - 'backend/core/**' env: AWS_REGION: eu-west-3 # set this to your preferred AWS region, e.g. us-west-1 @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: ./backend + working-directory: ./backend/core environment: production steps: diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 27a1cbaa0..b09d52ccb 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: ./backend + working-directory: ./backend/core environment: production steps: diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index be7771f1a..81feeca42 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -4,17 +4,17 @@ on: push: branches: [ main ] paths: - - 'backend/**' + - 'backend/core/**' pull_request: branches: [ main ] paths: - - 'backend/**' + - 'backend/core/**' jobs: build: runs-on: ubuntu-latest defaults: run: - working-directory: ./backend + working-directory: ./backend/core environment: preview strategy: matrix: diff --git a/.github/workflows/code-review-gpt.yml b/.github/workflows/code-review-gpt.yml index 7982e6790..961fe1f40 100644 --- a/.github/workflows/code-review-gpt.yml +++ b/.github/workflows/code-review-gpt.yml @@ -6,6 +6,7 @@ on: permissions: pull-requests: write + contents: read jobs: build: @@ -25,7 +26,7 @@ jobs: run: npm install code-review-gpt - name: Run code review script - run: npx code-review-gpt --ci + run: npx code-review-gpt review --ci env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BASE_SHA: ${{ github.event.pull_request.base.sha }} diff --git a/.gitignore b/.gitignore index b6e70715c..5c99ed50c 100644 --- a/.gitignore +++ b/.gitignore @@ -48,12 +48,12 @@ quivr/* streamlit-demo/.streamlit/secrets.toml .backend_env .frontend_env -backend/pandoc-* +backend/core/pandoc-* **/.pandoc-* -backend/application_default_credentials.json +backend/core/application_default_credentials.json #local models -backend/local_models/* +backend/core/local_models/* ## scripts diff --git a/.vscode/settings.json b/.vscode/settings.json index ac993a6a2..93e0a4a0a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -40,9 +40,9 @@ "**/.next/": true, "**/build/": true, "**/.docusaurus/": true, - "**/node_modules/": true, + "**/node_modules/": true }, "python.linting.pycodestyleCategorySeverity.W": "Error", "python.defaultInterpreterPath": "python3", "python.linting.flake8CategorySeverity.W": "Error", -} \ No newline at end of file +} diff --git a/README.md b/README.md index 8d3edffc5..3baea7483 100644 --- a/README.md +++ b/README.md @@ -87,19 +87,19 @@ Additionally, you'll need a [Supabase](https://supabase.com/) account for: - **Step 2 - Bis**: Copy the `.XXXXX_env` files ```bash - cp .backend_env.example backend/.env + cp .backend_env.example backend/core/.env cp .frontend_env.example frontend/.env ``` -- **Step 3**: Update the `backend/.env` and `frontend/.env` file +- **Step 3**: Update the `backend/core/.env` and `frontend/.env` file > _Your `supabase_service_key` can be found in your Supabase dashboard under Project Settings -> API. Use the `anon` `public` key found in the `Project API keys` section._ > _Your `JWT_SECRET_KEY`can be found in your supabase settings under Project Settings -> API -> JWT Settings -> JWT Secret_ - > _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/.env`- It is an advanced feature, please be expert in GCP before trying to use it_ + > _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/core/.env`- It is an advanced feature, please be expert in GCP before trying to use it_ - - [ ] Change variables in `backend/.env` + - [ ] Change variables in `backend/core/.env` - [ ] Change variables in `frontend/.env` - **Step 4**: Run the following migration scripts on the Supabase database via the web interface (SQL Editor -> `New query`) diff --git a/backend/Dockerfile b/backend/core/Dockerfile similarity index 54% rename from backend/Dockerfile rename to backend/core/Dockerfile index 45027748b..83be66b33 100644 --- a/backend/Dockerfile +++ b/backend/core/Dockerfile @@ -5,14 +5,14 @@ RUN apt-get update && apt-get install -y libgeos-dev pandoc WORKDIR /code -COPY ./requirements.txt /code/requirements.txt +COPY ./core/requirements.txt /code/requirements.txt RUN pip install --no-cache-dir -r /code/requirements.txt --timeout 100 -#You may need to run `chmod +x ./backend/scripts/start.sh` on your host machine if you get a permission error -COPY ./scripts/start.sh /code/scripts/start.sh +#You may need to run `chmod +x ./backend/core/scripts/start.sh` on your host machine if you get a permission error +COPY ./core/scripts/start.sh /code/scripts/start.sh RUN chmod +x /code/scripts/start.sh -COPY . /code/ +COPY ./core /code/ ENTRYPOINT ["bash", "/code/scripts/start.sh"] diff --git a/backend/auth/__init__.py b/backend/core/auth/__init__.py similarity index 100% rename from backend/auth/__init__.py rename to backend/core/auth/__init__.py diff --git a/backend/auth/api_key_handler.py b/backend/core/auth/api_key_handler.py similarity index 100% rename from backend/auth/api_key_handler.py rename to backend/core/auth/api_key_handler.py diff --git a/backend/auth/auth_bearer.py b/backend/core/auth/auth_bearer.py similarity index 99% rename from backend/auth/auth_bearer.py rename to backend/core/auth/auth_bearer.py index 913cc8297..55c815890 100644 --- a/backend/auth/auth_bearer.py +++ b/backend/core/auth/auth_bearer.py @@ -1,13 +1,12 @@ import os from typing import Optional +from auth.api_key_handler import get_user_from_api_key, verify_api_key +from auth.jwt_token_handler import decode_access_token, verify_token from fastapi import Depends, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from models.users import User -from auth.api_key_handler import get_user_from_api_key, verify_api_key -from auth.jwt_token_handler import decode_access_token, verify_token - class AuthBearer(HTTPBearer): def __init__(self, auto_error: bool = True): diff --git a/backend/auth/jwt_token_handler.py b/backend/core/auth/jwt_token_handler.py similarity index 100% rename from backend/auth/jwt_token_handler.py rename to backend/core/auth/jwt_token_handler.py diff --git a/backend/crawl/__init__.py b/backend/core/crawl/__init__.py similarity index 100% rename from backend/crawl/__init__.py rename to backend/core/crawl/__init__.py diff --git a/backend/crawl/crawler.py b/backend/core/crawl/crawler.py similarity index 100% rename from backend/crawl/crawler.py rename to backend/core/crawl/crawler.py diff --git a/backend/llm/__init__.py b/backend/core/llm/__init__.py similarity index 100% rename from backend/llm/__init__.py rename to backend/core/llm/__init__.py diff --git a/backend/llm/base.py b/backend/core/llm/base.py similarity index 100% rename from backend/llm/base.py rename to backend/core/llm/base.py diff --git a/backend/llm/models/FunctionCall.py b/backend/core/llm/models/FunctionCall.py similarity index 100% rename from backend/llm/models/FunctionCall.py rename to backend/core/llm/models/FunctionCall.py diff --git a/backend/llm/models/OpenAiAnswer.py b/backend/core/llm/models/OpenAiAnswer.py similarity index 100% rename from backend/llm/models/OpenAiAnswer.py rename to backend/core/llm/models/OpenAiAnswer.py diff --git a/backend/llm/openai.py b/backend/core/llm/openai.py similarity index 100% rename from backend/llm/openai.py rename to backend/core/llm/openai.py diff --git a/backend/llm/openai_functions.py b/backend/core/llm/openai_functions.py similarity index 100% rename from backend/llm/openai_functions.py rename to backend/core/llm/openai_functions.py diff --git a/backend/llm/private_gpt4all.py b/backend/core/llm/private_gpt4all.py similarity index 99% rename from backend/llm/private_gpt4all.py rename to backend/core/llm/private_gpt4all.py index 9d1fb15d7..d8af5cbb7 100644 --- a/backend/llm/private_gpt4all.py +++ b/backend/core/llm/private_gpt4all.py @@ -1,4 +1,5 @@ from typing import Optional + from langchain.embeddings.openai import OpenAIEmbeddings from langchain.llms.base import BaseLLM from langchain.llms.gpt4all import GPT4All diff --git a/backend/llm/prompts/CONDENSE_PROMPT.py b/backend/core/llm/prompts/CONDENSE_PROMPT.py similarity index 100% rename from backend/llm/prompts/CONDENSE_PROMPT.py rename to backend/core/llm/prompts/CONDENSE_PROMPT.py diff --git a/backend/llm/prompts/LANGUAGE_PROMPT.py b/backend/core/llm/prompts/LANGUAGE_PROMPT.py similarity index 100% rename from backend/llm/prompts/LANGUAGE_PROMPT.py rename to backend/core/llm/prompts/LANGUAGE_PROMPT.py diff --git a/backend/llm/qa_base.py b/backend/core/llm/qa_base.py similarity index 100% rename from backend/llm/qa_base.py rename to backend/core/llm/qa_base.py diff --git a/backend/llm/utils/summarization.py b/backend/core/llm/utils/summarization.py similarity index 100% rename from backend/llm/utils/summarization.py rename to backend/core/llm/utils/summarization.py diff --git a/backend/logger.py b/backend/core/logger.py similarity index 100% rename from backend/logger.py rename to backend/core/logger.py diff --git a/backend/main.py b/backend/core/main.py similarity index 100% rename from backend/main.py rename to backend/core/main.py diff --git a/backend/middlewares/cors.py b/backend/core/middlewares/cors.py similarity index 98% rename from backend/middlewares/cors.py rename to backend/core/middlewares/cors.py index 494f15e16..cff01088e 100644 --- a/backend/middlewares/cors.py +++ b/backend/core/middlewares/cors.py @@ -8,7 +8,7 @@ origins = [ "https://www.quivr.app", "http://quivr.app", "http://www.quivr.app", - "*" + "*", ] diff --git a/backend/models/__init__.py b/backend/core/models/__init__.py similarity index 100% rename from backend/models/__init__.py rename to backend/core/models/__init__.py diff --git a/backend/models/brains.py b/backend/core/models/brains.py similarity index 100% rename from backend/models/brains.py rename to backend/core/models/brains.py diff --git a/backend/models/brains_subscription_invitations.py b/backend/core/models/brains_subscription_invitations.py similarity index 69% rename from backend/models/brains_subscription_invitations.py rename to backend/core/models/brains_subscription_invitations.py index fc9f9d1b6..528ee50da 100644 --- a/backend/models/brains_subscription_invitations.py +++ b/backend/core/models/brains_subscription_invitations.py @@ -28,13 +28,19 @@ class BrainSubscription(BaseModel): response = ( self.commons["supabase"] .table("brain_subscription_invitations") - .insert({"brain_id": str(self.brain_id), "email": self.email, "rights": self.rights}) + .insert( + { + "brain_id": str(self.brain_id), + "email": self.email, + "rights": self.rights, + } + ) .execute() ) return response.data def update_subscription_invitation(self): - logger.info('Updating subscription invitation') + logger.info("Updating subscription invitation") response = ( self.commons["supabase"] .table("brain_subscription_invitations") @@ -46,12 +52,19 @@ class BrainSubscription(BaseModel): return response.data def create_or_update_subscription_invitation(self): - response = self.commons["supabase"].table("brain_subscription_invitations").select("*").eq("brain_id", str(self.brain_id)).eq("email", self.email).execute() + response = ( + self.commons["supabase"] + .table("brain_subscription_invitations") + .select("*") + .eq("brain_id", str(self.brain_id)) + .eq("email", self.email) + .execute() + ) if response.data: response = self.update_subscription_invitation() else: - response = self.create_subscription_invitation() + response = self.create_subscription_invitation() return response @@ -71,13 +84,15 @@ class BrainSubscription(BaseModel): """ try: - r = resend.Emails.send({ - "from": "onboarding@resend.dev", - "to": self.email, - "subject": "Quivr - Brain Shared With You", - "html": html_body - }) - print('Resend response', r) + r = resend.Emails.send( + { + "from": "onboarding@resend.dev", + "to": self.email, + "subject": "Quivr - Brain Shared With You", + "html": html_body, + } + ) + print("Resend response", r) except Exception as e: logger.error(f"Error sending email: {e}") return diff --git a/backend/models/chat.py b/backend/core/models/chat.py similarity index 100% rename from backend/models/chat.py rename to backend/core/models/chat.py diff --git a/backend/models/chats.py b/backend/core/models/chats.py similarity index 100% rename from backend/models/chats.py rename to backend/core/models/chats.py diff --git a/backend/models/files.py b/backend/core/models/files.py similarity index 100% rename from backend/models/files.py rename to backend/core/models/files.py diff --git a/backend/models/settings.py b/backend/core/models/settings.py similarity index 100% rename from backend/models/settings.py rename to backend/core/models/settings.py diff --git a/backend/models/users.py b/backend/core/models/users.py similarity index 100% rename from backend/models/users.py rename to backend/core/models/users.py diff --git a/backend/parsers/__init__.py b/backend/core/parsers/__init__.py similarity index 100% rename from backend/parsers/__init__.py rename to backend/core/parsers/__init__.py diff --git a/backend/parsers/audio.py b/backend/core/parsers/audio.py similarity index 100% rename from backend/parsers/audio.py rename to backend/core/parsers/audio.py diff --git a/backend/parsers/common.py b/backend/core/parsers/common.py similarity index 100% rename from backend/parsers/common.py rename to backend/core/parsers/common.py diff --git a/backend/parsers/csv.py b/backend/core/parsers/csv.py similarity index 100% rename from backend/parsers/csv.py rename to backend/core/parsers/csv.py diff --git a/backend/parsers/docx.py b/backend/core/parsers/docx.py similarity index 100% rename from backend/parsers/docx.py rename to backend/core/parsers/docx.py diff --git a/backend/parsers/epub.py b/backend/core/parsers/epub.py similarity index 100% rename from backend/parsers/epub.py rename to backend/core/parsers/epub.py diff --git a/backend/parsers/github.py b/backend/core/parsers/github.py similarity index 100% rename from backend/parsers/github.py rename to backend/core/parsers/github.py diff --git a/backend/parsers/html.py b/backend/core/parsers/html.py similarity index 100% rename from backend/parsers/html.py rename to backend/core/parsers/html.py diff --git a/backend/parsers/markdown.py b/backend/core/parsers/markdown.py similarity index 100% rename from backend/parsers/markdown.py rename to backend/core/parsers/markdown.py diff --git a/backend/parsers/notebook.py b/backend/core/parsers/notebook.py similarity index 100% rename from backend/parsers/notebook.py rename to backend/core/parsers/notebook.py diff --git a/backend/parsers/odt.py b/backend/core/parsers/odt.py similarity index 100% rename from backend/parsers/odt.py rename to backend/core/parsers/odt.py diff --git a/backend/parsers/pdf.py b/backend/core/parsers/pdf.py similarity index 100% rename from backend/parsers/pdf.py rename to backend/core/parsers/pdf.py diff --git a/backend/parsers/powerpoint.py b/backend/core/parsers/powerpoint.py similarity index 100% rename from backend/parsers/powerpoint.py rename to backend/core/parsers/powerpoint.py diff --git a/backend/parsers/txt.py b/backend/core/parsers/txt.py similarity index 100% rename from backend/parsers/txt.py rename to backend/core/parsers/txt.py diff --git a/backend/pyrightconfig.json b/backend/core/pyrightconfig.json similarity index 100% rename from backend/pyrightconfig.json rename to backend/core/pyrightconfig.json diff --git a/backend/repository/__init__.py b/backend/core/repository/__init__.py similarity index 100% rename from backend/repository/__init__.py rename to backend/core/repository/__init__.py diff --git a/backend/repository/chat/__init__.py b/backend/core/repository/chat/__init__.py similarity index 100% rename from backend/repository/chat/__init__.py rename to backend/core/repository/chat/__init__.py diff --git a/backend/repository/chat/create_chat.py b/backend/core/repository/chat/create_chat.py similarity index 100% rename from backend/repository/chat/create_chat.py rename to backend/core/repository/chat/create_chat.py diff --git a/backend/repository/chat/format_chat_history.py b/backend/core/repository/chat/format_chat_history.py similarity index 100% rename from backend/repository/chat/format_chat_history.py rename to backend/core/repository/chat/format_chat_history.py diff --git a/backend/repository/chat/get_chat_by_id.py b/backend/core/repository/chat/get_chat_by_id.py similarity index 100% rename from backend/repository/chat/get_chat_by_id.py rename to backend/core/repository/chat/get_chat_by_id.py diff --git a/backend/repository/chat/get_chat_history.py b/backend/core/repository/chat/get_chat_history.py similarity index 100% rename from backend/repository/chat/get_chat_history.py rename to backend/core/repository/chat/get_chat_history.py diff --git a/backend/repository/chat/get_user_chats.py b/backend/core/repository/chat/get_user_chats.py similarity index 100% rename from backend/repository/chat/get_user_chats.py rename to backend/core/repository/chat/get_user_chats.py diff --git a/backend/repository/chat/update_chat.py b/backend/core/repository/chat/update_chat.py similarity index 100% rename from backend/repository/chat/update_chat.py rename to backend/core/repository/chat/update_chat.py diff --git a/backend/repository/chat/update_chat_history.py b/backend/core/repository/chat/update_chat_history.py similarity index 100% rename from backend/repository/chat/update_chat_history.py rename to backend/core/repository/chat/update_chat_history.py diff --git a/backend/repository/chat/update_message_by_id.py b/backend/core/repository/chat/update_message_by_id.py similarity index 100% rename from backend/repository/chat/update_message_by_id.py rename to backend/core/repository/chat/update_message_by_id.py diff --git a/backend/requirements.txt b/backend/core/requirements.txt similarity index 100% rename from backend/requirements.txt rename to backend/core/requirements.txt diff --git a/backend/routes/__init__.py b/backend/core/routes/__init__.py similarity index 100% rename from backend/routes/__init__.py rename to backend/core/routes/__init__.py diff --git a/backend/routes/api_key_routes.py b/backend/core/routes/api_key_routes.py similarity index 100% rename from backend/routes/api_key_routes.py rename to backend/core/routes/api_key_routes.py diff --git a/backend/routes/authorizations/__init__.py b/backend/core/routes/authorizations/__init__.py similarity index 100% rename from backend/routes/authorizations/__init__.py rename to backend/core/routes/authorizations/__init__.py diff --git a/backend/routes/authorizations/brain_authorization.py b/backend/core/routes/authorizations/brain_authorization.py similarity index 100% rename from backend/routes/authorizations/brain_authorization.py rename to backend/core/routes/authorizations/brain_authorization.py diff --git a/backend/routes/brain_routes.py b/backend/core/routes/brain_routes.py similarity index 100% rename from backend/routes/brain_routes.py rename to backend/core/routes/brain_routes.py diff --git a/backend/routes/chat_routes.py b/backend/core/routes/chat_routes.py similarity index 100% rename from backend/routes/chat_routes.py rename to backend/core/routes/chat_routes.py diff --git a/backend/routes/crawl_routes.py b/backend/core/routes/crawl_routes.py similarity index 100% rename from backend/routes/crawl_routes.py rename to backend/core/routes/crawl_routes.py diff --git a/backend/routes/explore_routes.py b/backend/core/routes/explore_routes.py similarity index 99% rename from backend/routes/explore_routes.py rename to backend/core/routes/explore_routes.py index 66c7e602b..192c3aae5 100644 --- a/backend/routes/explore_routes.py +++ b/backend/core/routes/explore_routes.py @@ -5,7 +5,6 @@ from fastapi import APIRouter, Depends, Query from models.brains import Brain from models.settings import common_dependencies from models.users import User - from routes.authorizations.brain_authorization import ( has_brain_authorization, validate_brain_authorization, diff --git a/backend/routes/misc_routes.py b/backend/core/routes/misc_routes.py similarity index 100% rename from backend/routes/misc_routes.py rename to backend/core/routes/misc_routes.py diff --git a/backend/routes/subscription_routes.py b/backend/core/routes/subscription_routes.py similarity index 100% rename from backend/routes/subscription_routes.py rename to backend/core/routes/subscription_routes.py diff --git a/backend/routes/upload_routes.py b/backend/core/routes/upload_routes.py similarity index 100% rename from backend/routes/upload_routes.py rename to backend/core/routes/upload_routes.py diff --git a/backend/routes/user_routes.py b/backend/core/routes/user_routes.py similarity index 100% rename from backend/routes/user_routes.py rename to backend/core/routes/user_routes.py diff --git a/backend/scripts/start.sh b/backend/core/scripts/start.sh similarity index 100% rename from backend/scripts/start.sh rename to backend/core/scripts/start.sh diff --git a/backend/supabase/functions/add-new-email/index.ts b/backend/core/supabase/functions/add-new-email/index.ts similarity index 95% rename from backend/supabase/functions/add-new-email/index.ts rename to backend/core/supabase/functions/add-new-email/index.ts index ec11b3857..d382a98fe 100644 --- a/backend/supabase/functions/add-new-email/index.ts +++ b/backend/core/supabase/functions/add-new-email/index.ts @@ -1,5 +1,5 @@ /* Instructions: -1. in .backend/supabase folder, create .env file with BEEHIIV_PUBLICATION_ID and BEEHIIV_API_KEY variables +1. in .backend/core/supabase folder, create .env file with BEEHIIV_PUBLICATION_ID and BEEHIIV_API_KEY variables 2. cd into .backend --- for the rest of these steps you will need your supabase project id which can be found in your console url: https://supabase.com/dashboard/project/ --- 3. run `supabase secrets set --env-file ./supabase/.env` to set the environment variables diff --git a/backend/tests/conftest.py b/backend/core/tests/conftest.py similarity index 100% rename from backend/tests/conftest.py rename to backend/core/tests/conftest.py diff --git a/backend/tests/test_api_key.py b/backend/core/tests/test_api_key.py similarity index 100% rename from backend/tests/test_api_key.py rename to backend/core/tests/test_api_key.py diff --git a/backend/tests/test_brains.py b/backend/core/tests/test_brains.py similarity index 100% rename from backend/tests/test_brains.py rename to backend/core/tests/test_brains.py diff --git a/backend/tests/test_chats.py b/backend/core/tests/test_chats.py similarity index 100% rename from backend/tests/test_chats.py rename to backend/core/tests/test_chats.py diff --git a/backend/tests/test_explore.py b/backend/core/tests/test_explore.py similarity index 100% rename from backend/tests/test_explore.py rename to backend/core/tests/test_explore.py diff --git a/backend/tests/test_files/test.csv b/backend/core/tests/test_files/test.csv similarity index 100% rename from backend/tests/test_files/test.csv rename to backend/core/tests/test_files/test.csv diff --git a/backend/tests/test_files/test.pdf b/backend/core/tests/test_files/test.pdf similarity index 100% rename from backend/tests/test_files/test.pdf rename to backend/core/tests/test_files/test.pdf diff --git a/backend/tests/test_files/test.txt b/backend/core/tests/test_files/test.txt similarity index 100% rename from backend/tests/test_files/test.txt rename to backend/core/tests/test_files/test.txt diff --git a/backend/tests/test_upload.py b/backend/core/tests/test_upload.py similarity index 100% rename from backend/tests/test_upload.py rename to backend/core/tests/test_upload.py diff --git a/backend/tests/test_user.py b/backend/core/tests/test_user.py similarity index 100% rename from backend/tests/test_user.py rename to backend/core/tests/test_user.py diff --git a/backend/utils/__init__.py b/backend/core/utils/__init__.py similarity index 100% rename from backend/utils/__init__.py rename to backend/core/utils/__init__.py diff --git a/backend/utils/chats.py b/backend/core/utils/chats.py similarity index 100% rename from backend/utils/chats.py rename to backend/core/utils/chats.py diff --git a/backend/utils/constants.py b/backend/core/utils/constants.py similarity index 100% rename from backend/utils/constants.py rename to backend/core/utils/constants.py diff --git a/backend/utils/file.py b/backend/core/utils/file.py similarity index 100% rename from backend/utils/file.py rename to backend/core/utils/file.py diff --git a/backend/utils/processors.py b/backend/core/utils/processors.py similarity index 100% rename from backend/utils/processors.py rename to backend/core/utils/processors.py diff --git a/backend/utils/users.py b/backend/core/utils/users.py similarity index 100% rename from backend/utils/users.py rename to backend/core/utils/users.py diff --git a/backend/utils/vectors.py b/backend/core/utils/vectors.py similarity index 100% rename from backend/utils/vectors.py rename to backend/core/utils/vectors.py diff --git a/backend/vectorstore/__init__.py b/backend/core/vectorstore/__init__.py similarity index 100% rename from backend/vectorstore/__init__.py rename to backend/core/vectorstore/__init__.py diff --git a/backend/vectorstore/supabase.py b/backend/core/vectorstore/supabase.py similarity index 100% rename from backend/vectorstore/supabase.py rename to backend/core/vectorstore/supabase.py diff --git a/backend/private/Dockerfile b/backend/private/Dockerfile new file mode 100644 index 000000000..9e94b3c8c --- /dev/null +++ b/backend/private/Dockerfile @@ -0,0 +1,20 @@ +# Use the same base image as the 'backend-core' container +FROM python:3.11-bullseye + +# Install necessary packages +RUN apt-get update && apt-get install -y liblzma-dev cmake git + +# Set the working directory +WORKDIR /app + +# Copy the requirements file +COPY ./private/requirements.txt /app/requirements.txt + +# Install Python dependencies +RUN pip install --no-cache-dir -r /app/requirements.txt --timeout 100 + +# Copy your application's code to the Docker container +COPY ./private /app + +# Start the Uvicorn server on port 5051 +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "5051"] diff --git a/backend/private/logger.py b/backend/private/logger.py new file mode 100644 index 000000000..21a896de2 --- /dev/null +++ b/backend/private/logger.py @@ -0,0 +1,17 @@ +import logging + + +def get_logger(logger_name, log_level=logging.INFO): + logger = logging.getLogger(logger_name) + logger.setLevel(log_level) + logger.propagate = False # Prevent log propagation to avoid double logging + + formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s") + + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + + if not logger.handlers: + logger.addHandler(console_handler) + + return logger diff --git a/backend/private/main.py b/backend/private/main.py new file mode 100644 index 000000000..0a21e960c --- /dev/null +++ b/backend/private/main.py @@ -0,0 +1,30 @@ +import os + +import sentry_sdk +from fastapi import FastAPI, HTTPException +from fastapi.responses import JSONResponse +from logger import get_logger +from routes.completions_routes import completions_router + +logger = get_logger(__name__) + +if os.getenv("SENTRY_DSN"): + sentry_sdk.init( + dsn=os.getenv("SENTRY_DSN"), + # Set traces_sample_rate to 1.0 to capture 100% + # of transactions for performance monitoring. + # We recommend adjusting this value in production, + traces_sample_rate=1.0, + ) + +app = FastAPI() + +app.include_router(completions_router) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(_, exc): + return JSONResponse( + status_code=exc.status_code, + content={"detail": exc.detail}, + ) diff --git a/backend/private/models/messages.py b/backend/private/models/messages.py new file mode 100644 index 000000000..31b4cb739 --- /dev/null +++ b/backend/private/models/messages.py @@ -0,0 +1,18 @@ +from typing import List + +from pydantic import BaseModel, Field + + +class Message(BaseModel): + role: str = Field( + ..., + description="The role of the messages author. One of system, user, assistant, or function.", + ) + content: str = Field( + ..., + description="The contents of the message. content is required for all messages, and may be null for assistant messages with function calls.", + ) + + +class Messages(BaseModel): + messages: List[Message] diff --git a/backend/private/requirements.txt b/backend/private/requirements.txt new file mode 100644 index 000000000..40139e80c --- /dev/null +++ b/backend/private/requirements.txt @@ -0,0 +1,11 @@ +langchain==0.0.228 +tiktoken==0.4.0 +fastapi==0.95.2 +uvicorn==0.22.0 +transformers==4.30.1 +asyncpg==0.27.0 +flake8==6.0.0 +flake8-black==0.3.6 +sentence_transformers>=2.0.0 +sentry-sdk==1.26.0 +pyright==1.1.316 \ No newline at end of file diff --git a/backend/private/routes/completions_routes.py b/backend/private/routes/completions_routes.py new file mode 100644 index 000000000..6b5624cdf --- /dev/null +++ b/backend/private/routes/completions_routes.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter +from logger import get_logger +from models.messages import Messages + +logger = get_logger(__name__) + +completions_router = APIRouter() + + +@completions_router.post("/chat/completions", tags=["Chat Completions"]) +async def post_chat_completions( + model: str, # make union of all models + messages: Messages, +) -> str: + logger.info( + f"Received chat completions request for {model} with messages {messages}" + ) + return f"Hello from {model}!" diff --git a/backend/private/routes/embeddings_routes.py b/backend/private/routes/embeddings_routes.py new file mode 100644 index 000000000..f413f46e0 --- /dev/null +++ b/backend/private/routes/embeddings_routes.py @@ -0,0 +1,16 @@ +from fastapi import APIRouter +from logger import getLogger + +logger = getLogger(__name__) + +embeddings_router = APIRouter() + + +@embeddings_router.post("/embeddings", tags=["Embeddings"]) +async def post_embeddings( + model: str, + input: str | list[str], +) -> str: + logger.info(f"Received embeddings request for {model} with input {input}") + + return f"Hello from embeddings {model}!" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 6da95e7ac..b2818abe1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -15,16 +15,16 @@ services: - /app/.next ports: - 3000:3000 - backend: + backend-core: env_file: - - ./backend/.env + - ./backend/core/.env build: context: backend - dockerfile: Dockerfile - container_name: backend + dockerfile: core/Dockerfile + container_name: backend-core restart: always volumes: - - ./backend/:/code/ + - ./backend/core/:/code/ - ~/.config/gcloud:/root/.config/gcloud ports: - 5050:5050 \ No newline at end of file diff --git a/docker-compose.private-dev.yml b/docker-compose.private-dev.yml new file mode 100644 index 000000000..4d86fef20 --- /dev/null +++ b/docker-compose.private-dev.yml @@ -0,0 +1,42 @@ +version: "3" + +services: + frontend: + env_file: + - ./frontend/.env + build: + context: frontend + dockerfile: Dockerfile.dev + container_name: web + restart: always + volumes: + - ./frontend/:/app + - /app/node_modules + - /app/.next + ports: + - 3000:3000 + backend-core: + env_file: + - ./backend/core/.env + build: + context: backend + dockerfile: core/Dockerfile + container_name: backend-core + restart: always + volumes: + - ./backend/core/:/code/ + - ~/.config/gcloud:/root/.config/gcloud + ports: + - 5050:5050 + backend-private: + env_file: + - ./backend/private/.env + build: + context: backend + dockerfile: private/Dockerfile + container_name: backend-private + restart: always + volumes: + - ./backend/private/:/app/ + ports: + - 5051:5050 diff --git a/docker-compose.yml b/docker-compose.yml index 59dca0b8a..db0dd2708 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,16 +11,16 @@ services: restart: always ports: - 3000:3000 - backend: + backend-core: env_file: - - ./backend/.env + - ./backend/core/.env build: context: backend - dockerfile: Dockerfile - container_name: backend + dockerfile: core/Dockerfile + container_name: backend-core restart: always volumes: - - ./backend/:/code/ + - ./backend/core/:/code/ - ~/.config/gcloud:/root/.config/gcloud ports: - 5050:5050 \ No newline at end of file diff --git a/install_helper.sh b/install_helper.sh index 2716aa00f..4f484e184 100755 --- a/install_helper.sh +++ b/install_helper.sh @@ -21,8 +21,8 @@ replace_in_file() { } # Step 2: Copy the .XXXXX_env files if they don't exist -if [ ! -f backend/.env ]; then - echo "Copying backend .env example file..." +if [ ! -f backend/core/.env ]; then + echo "Copying backend-core .env example file..." cp .backend_env.example backend/.env fi @@ -34,29 +34,29 @@ fi # Step 3: Ask the user for environment variables and update .env files # only if they haven't been set. -# Update backend/.env -if grep -q "SUPABASE_URL=" backend/.env; then +# Update backend/core/.env +if grep -q "SUPABASE_URL=" backend/core/.env; then echo "SUPABASE_URL can be found in your Supabase dashboard under Settings > API." - SUPABASE_URL=$(gum input --placeholder "Enter SUPABASE_URL for backend") - replace_in_file backend/.env "SUPABASE_URL=.*" "SUPABASE_URL=${SUPABASE_URL}" + SUPABASE_URL=$(gum input --placeholder "Enter SUPABASE_URL for backend/core") + replace_in_file backend/core/.env "SUPABASE_URL=.*" "SUPABASE_URL=${SUPABASE_URL}" fi -if grep -q "SUPABASE_SERVICE_KEY=" backend/.env; then +if grep -q "SUPABASE_SERVICE_KEY=" backend/core/.env; then echo "SUPABASE_SERVICE_KEY can be found in your Supabase dashboard under Settings > API. Use the anon public key found in the Project API keys section." - SUPABASE_SERVICE_KEY=$(gum input --placeholder "Enter SUPABASE_SERVICE_KEY for backend") - replace_in_file backend/.env "SUPABASE_SERVICE_KEY=.*" "SUPABASE_SERVICE_KEY=${SUPABASE_SERVICE_KEY}" + SUPABASE_SERVICE_KEY=$(gum input --placeholder "Enter SUPABASE_SERVICE_KEY for backend/core") + replace_in_file backend/core/.env "SUPABASE_SERVICE_KEY=.*" "SUPABASE_SERVICE_KEY=${SUPABASE_SERVICE_KEY}" fi -if grep -q "OPENAI_API_KEY=" backend/.env; then +if grep -q "OPENAI_API_KEY=" backend/core/.env; then echo "OPENAI_API_KEY is the API key from OpenAI, if you are using OpenAI services." - OPENAI_API_KEY=$(gum input --placeholder "Enter OPENAI_API_KEY for backend") - replace_in_file backend/.env "OPENAI_API_KEY=.*" "OPENAI_API_KEY=${OPENAI_API_KEY}" + OPENAI_API_KEY=$(gum input --placeholder "Enter OPENAI_API_KEY for backend/core") + replace_in_file backend/core/.env "OPENAI_API_KEY=.*" "OPENAI_API_KEY=${OPENAI_API_KEY}" fi -if grep -q "JWT_SECRET_KEY=" backend/.env; then +if grep -q "JWT_SECRET_KEY=" backend/core/.env; then echo "JWT_SECRET_KEY can be found in your Supabase project dashboard under Settings > API > JWT Settings > JWT Secret." - JWT_SECRET_KEY=$(gum input --placeholder "Enter JWT_SECRET_KEY for backend") - replace_in_file backend/.env "JWT_SECRET_KEY=.*" "JWT_SECRET_KEY=${JWT_SECRET_KEY}" + JWT_SECRET_KEY=$(gum input --placeholder "Enter JWT_SECRET_KEY for backend/core") + replace_in_file backend/core/.env "JWT_SECRET_KEY=.*" "JWT_SECRET_KEY=${JWT_SECRET_KEY}" fi # Update frontend/.env using the same SUPABASE_URL and SUPABASE_SERVICE_KEY