diff --git a/.gitignore b/.gitignore index fc588b440..03a102736 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ backend/bin/* backend/lib/* backend/pyvenv.cfg backend/share/* +backend/slim.report.json diff --git a/Makefile b/Makefile index da0d73fc9..d5548342c 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ test: pytest -s backend/tests dev: + docker compose -f docker-compose.dev.yml build backend-core docker compose -f docker-compose.dev.yml up --build prod: diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 000000000..0bd3f40a9 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,10 @@ +**/__pycache__ +*/.pytest_cache +**/__pycache__ +**/.benchmarks/ +**/.cache/ +**/.pytest_cache/ +**/.next/ +**/build/ +**/.docusaurus/ +**/node_modules/ \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile index 2278de788..d30b83891 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,20 +1,19 @@ # Using a slim version for a smaller base image -FROM python:3.11-slim-bullseye +FROM python:3.11.6-slim-bullseye ARG DEV_MODE ENV DEV_MODE=$DEV_MODE # Install GEOS library, Rust, and other dependencies, then clean up -RUN apt-get update && apt-get install -y \ +RUN apt-get clean && apt-get update && apt-get install -y \ libgeos-dev \ libcurl4-openssl-dev \ libssl-dev \ - pandoc \ binutils \ + pandoc \ curl \ git \ build-essential && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ rm -rf /var/lib/apt/lists/* && apt-get clean # Add Rust binaries to the PATH @@ -36,4 +35,6 @@ RUN if [ "$DEV_MODE" = "true" ]; then pip install --no-cache debugpy --timeout 2 # Copy the rest of the application COPY . . -CMD ["uvicorn", "main:app", "--reload", "--host", "0.0.0.0", "--port", "5050", "--workers", "6"] \ No newline at end of file +EXPOSE 5050 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "5050", "--workers", "6"] \ No newline at end of file diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev new file mode 100644 index 000000000..49974ab9b --- /dev/null +++ b/backend/Dockerfile.dev @@ -0,0 +1,40 @@ +# Using a slim version for a smaller base image +FROM python:3.11.6-slim-bullseye + +ARG DEV_MODE +ENV DEV_MODE=$DEV_MODE + +# Install GEOS library, Rust, and other dependencies, then clean up +RUN apt-get clean && apt-get update && apt-get install -y \ + libgeos-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + binutils \ + pandoc \ + curl \ + git \ + build-essential && \ + rm -rf /var/lib/apt/lists/* && apt-get clean + +# Add Rust binaries to the PATH +ENV PATH="/root/.cargo/bin:${PATH}" + +WORKDIR /code + +# Copy just the requirements first +COPY ./requirements.txt . + +# Upgrade pip +RUN pip install --upgrade pip + +# Increase timeout to wait for the new installation +RUN pip install --no-cache-dir -r requirements.txt --timeout 200 + +RUN if [ "$DEV_MODE" = "true" ]; then pip install --no-cache debugpy --timeout 200; fi + +# Copy the rest of the application +COPY . . + +EXPOSE 5050 + +CMD ["uvicorn", "main:app","--reload", "--host", "0.0.0.0", "--port", "5050", "--workers", "6"] \ No newline at end of file diff --git a/backend/main.py b/backend/main.py index 20e1ed9d3..892841a5e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -8,7 +8,6 @@ if __name__ == "__main__": from dotenv import load_dotenv # type: ignore load_dotenv() -import pypandoc import sentry_sdk from fastapi import FastAPI, HTTPException from fastapi.responses import JSONResponse @@ -50,10 +49,10 @@ app = FastAPI() add_cors_middleware(app) -@app.on_event("startup") -async def startup_event(): - if not os.path.exists(pypandoc.get_pandoc_path()): - pypandoc.download_pandoc() +# @app.on_event("startup") +# async def startup_event(): +# if not os.path.exists(pypandoc.get_pandoc_path()): +# pypandoc.download_pandoc() app.include_router(brain_router) diff --git a/backend/requirements.txt b/backend/requirements.txt index 2f3a84eee..3b2d597c2 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,12 +1,11 @@ -pymupdf==1.22.3 +# pymupdf==1.22.3 langchain==0.0.332 litellm==0.13.2 -Markdown==3.4.4 +# Markdown==3.4.4 openai==0.27.8 GitPython==3.1.36 pdf2image==1.16.3 pypdf==3.9.0 -StrEnum==0.4.15 supabase==1.1.0 tiktoken==0.4.0 unstructured==0.6.7 @@ -14,18 +13,15 @@ fastapi==0.95.2 python-multipart==0.0.6 uvicorn==0.22.0 pypandoc==1.11 + docx2txt==0.8 python-jose==3.3.0 -transformers==4.30.1 asyncpg==0.27.0 flake8==6.0.0 flake8-black==0.3.6 -sentence_transformers>=2.0.0 sentry-sdk==1.26.0 pyright==1.1.316 resend==0.5.1 -psycopg2-binary==2.9.6 -sqlalchemy==2.0.19 html5lib==1.1 beautifulsoup4 newspaper3k diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d09734ffc..30bad7980 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -22,7 +22,7 @@ services: - ./backend/.env build: context: backend - dockerfile: Dockerfile + dockerfile: Dockerfile.dev args: - DEV_MODE=true container_name: backend-core @@ -49,7 +49,7 @@ services: - ./backend/.env build: context: backend - dockerfile: Dockerfile + dockerfile: Dockerfile.dev container_name: worker command: celery -A celery_worker worker -l info restart: unless-stopped @@ -64,7 +64,7 @@ services: - ./backend/.env build: context: backend - dockerfile: Dockerfile + dockerfile: Dockerfile.dev container_name: flower command: celery -A celery_worker flower -l info --port=5555 restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index fba7d1b53..3c2fa9569 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,6 @@ services: dockerfile: Dockerfile container_name: backend-core restart: always - volumes: - - ./backend/:/code/ depends_on: - redis - worker @@ -49,8 +47,6 @@ services: restart: always depends_on: - redis - volumes: - - ./backend/:/code/ beat: image: backend-base @@ -64,8 +60,6 @@ services: restart: always depends_on: - redis - volumes: - - ./backend/:/code/ flower: image: backend-base @@ -77,8 +71,6 @@ services: container_name: flower command: celery -A celery_worker flower -l info --port=5555 restart: always - volumes: - - ./backend/:/code/ depends_on: - redis - worker diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 9b76e35c9..871d00a93 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,2 +1,13 @@ -.next/ -node_modules/ \ No newline at end of file +**/.next/ +**/node_modules/ +**/.vercel/ +**/__pycache__ +*/.pytest_cache +**/__pycache__ +**/.benchmarks/ +**/.cache/ +**/.pytest_cache/ +**/.next/ +**/build/ +**/.docusaurus/ +**/node_modules/ \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 475e15e7d..02c681bd1 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,49 +1,31 @@ -# First stage: Build environment -FROM node:18.13.0-alpine as builder - +FROM node:18.13.0-alpine # Install Python and essential build tools RUN apk add --update --no-cache python3 make g++ && ln -sf python3 /usr/bin/python RUN python3 -m ensurepip RUN pip3 install --no-cache --upgrade pip setuptools -# Create a Python virtual environment -RUN python3 -m venv /venv -ENV PATH="/venv/bin:$PATH" - -# Create the directory where our app will live +# Create the directory on the node image +# where our Next.js app will live RUN mkdir -p /app + +# Set /app as the working directory WORKDIR /app -# Copy package.json and yarn.lock to the working directory +# Copy package.json and yarn.lock +# to the /app working directory COPY package*.json yarn.lock ./ -# Install Node.js dependencies +# Install dependencies in /app RUN yarn install --network-timeout 1000000 -# Copy the rest of the Next.js folder into /app +# Copy the rest of our Next.js folder into /app COPY . . # Build the Next.js application RUN yarn build -# Second stage: Runtime environment -FROM node:18.13.0-alpine - -# Copy the virtual environment from the builder stage -COPY --from=builder /venv /venv -ENV PATH="/venv/bin:$PATH" - -# Set the working directory -WORKDIR /app - -# Copy built assets from the builder stage -COPY --from=builder /app/.next ./.next -COPY --from=builder /app/node_modules ./node_modules -COPY --from=builder /app/package.json ./package.json -COPY --from=builder /app/public ./public - # Ensure port 3000 is accessible to our system EXPOSE 3000 -# Run yarn start, as we would via the command line +# Run yarn start, as we would via the command line CMD ["yarn", "start"]