fix: quivr core fix tests (#2935)

# Description
- Update CI
- Fix DockerFile for quivr-core tests
- Tox parallel tests

@StanGirard  @chloedia 
- For running quivr-core tests:
```bash
cd backend/core
./scripts/run_tests.sh
```

---------

Co-authored-by: aminediro <aminedirhoussi@gmail.com>
Co-authored-by: Stan Girard <girard.stanislas@gmail.com>

.github/workflows/backend-core-tests.yml

@@ -33,12 +33,20 @@ jobs:
         cd backend/core
         python -m pip install --upgrade pip
         pip install poetry
-        poetry install -E base
+        poetry install -E base --with dev,test
+
+    - name: Run pre-commit checks
+      run: |
+        cd backend/core
+        poetry run pre-commit run --files .
 
     - name: Run tests
       env:
         TIKA_URL: http://localhost:9998/tika
         OPENAI_API_KEY: this-is-a-test-key
       run: |
+        sudo apt-get update
+        sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr  pandoc
+        tesseract --version
         cd backend/core
-        poetry run pytest tests
+        poetry run tox

backend/core/Dockerfile.test

@@ -5,6 +5,9 @@ FROM python:3.11.6-slim-bullseye
 RUN apt-get clean && apt-get update && apt-get install -y \
     curl \
     gcc \
+    autoconf \
+    automake \
+    build-essential \
     # Additional dependencies for document handling
     libmagic-dev \
     tesseract-ocr \
@@ -28,7 +31,10 @@ ENV PATH="/root/.local/bin:$PATH"
 # Copy the current directory contents into the container at /app
 COPY ./pyproject.toml ./poetry.lock* /code/
 
+RUN python3 -m pip install nltk && python3 -c "import nltk; nltk.download('punkt')" \
+    && python3 -c "import nltk; nltk.download('averaged_perceptron_tagger')"
+
 # Install project dependencies
-RUN poetry install --no-root --with test
+RUN poetry install  --with test
 
 ENV PYTHONPATH=/code

backend/core/scripts/run_tests.sh

@@ -8,7 +8,8 @@ IMAGE_NAME="quivr-core-test"
 IMAGE_TAG="latest"
 DOCKERFILE="Dockerfile.test"
 VOLUME_MAPPING="$PWD:/code"
-CMD="poetry run tox"
+TOX_DIR="/code/.tox-docker"
+CMD="poetry run tox -p auto"
 
 # Functions
 build_image() {
@@ -18,7 +19,10 @@ build_image() {
 
 run_container() {
     echo "Running tests in Docker container..."
-    docker run -it --rm -v $VOLUME_MAPPING $IMAGE_NAME:$IMAGE_TAG $CMD
+    docker run -it --rm \
+        -e TOX_WORK_DIR=$TOX_DIR \
+        -v $VOLUME_MAPPING \
+        $IMAGE_NAME:$IMAGE_TAG $CMD
 }
 
 # Main script execution

backend/core/scripts/run_tests_buildx.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+# Constants
+IMAGE_NAME="quivr-core-test"
+IMAGE_TAG="latest"
+DOCKERFILE="Dockerfile.test"
+VOLUME_MAPPING="$PWD:/code"
+CMD="poetry run tox"
+PLATFORM="linux/amd64"
+BUILDER_NAME="amd64_builder"
+
+# Functions
+build_image() {
+    echo "Building Docker image for $PLATFORM..."
+    EXISTING_BUILDER=$(docker buildx ls | grep -w $BUILDER_NAME)
+
+    # Create the builder if it doesn't exist
+    if [ -z "$EXISTING_BUILDER" ]; then
+        echo "Creating builder: $BUILDER_NAME"
+        docker buildx create --use --name $BUILDER_NAME --platform $PLATFORM
+    else
+        echo "Builder $BUILDER_NAME already exists. Skipping creation."
+    fi
+
+    docker buildx build --platform $PLATFORM -f $DOCKERFILE -t $IMAGE_NAME:$IMAGE_TAG --load .
+}
+
+run_container() {
+    echo "Running tests in Docker container..."
+    docker run -it --rm --platform $PLATFORM -v $VOLUME_MAPPING $IMAGE_NAME:$IMAGE_TAG $CMD
+}
+
+# Main script execution
+build_image
+run_container
+
+echo "Tests completed successfully."

backend/core/tox.ini

@@ -14,7 +14,7 @@ allowlist_externals =
 commands_pre =
     poetry install --no-root --with test
 commands =
-    poetry run pytest tests/ -m "not base and not tika"  \
+    poetry run pytest tests/ -m "not base"  \
         --ignore=./tests/processor/epub \
         --ignore=./tests/processor/docx \
         --ignore=./tests/processor/odt \