feat(preview): added service upload (#979)

* feat(docker): improved size image * feat(preview): added upload service * ci(aws): using matrix
2024-08-17 00:50:37 +03:00 · 2023-08-18 14:49:48 +02:00 · 2023-08-18 14:49:48 +02:00 · ce6b45e1ac
commit ce6b45e1ac
parent c746eb1830
6 changed files with 153 additions and 47 deletions
--- a/.aws/task_definition_preview.json
+++ b/.aws/task_definition_preview.json
@ -92,7 +92,7 @@
        "FARGATE"
    ],
    "cpu": "256",
-    "memory": "512",
+    "memory": "1024",
    "runtimePlatform": {
        "cpuArchitecture": "X86_64",
        "operatingSystemFamily": "LINUX"
--- a/.aws/task_definition_preview_chat.json
+++ b/.aws/task_definition_preview_chat.json
@ -91,7 +91,7 @@
        "FARGATE"
    ],
    "cpu": "256",
-    "memory": "512",
+    "memory": "1024",
    "runtimePlatform": {
        "cpuArchitecture": "X86_64",
        "operatingSystemFamily": "LINUX"
--- a/.aws/task_definition_preview_upload.json
+++ b/.aws/task_definition_preview_upload.json
@ -0,0 +1,102 @@
+{
+    "taskDefinitionArn": "arn:aws:ecs:eu-west-3:253053805092:task-definition/quivr-preview-upload:1",
+    "containerDefinitions": [
+        {
+            "name": "quivr-upload",
+            "image": "253053805092.dkr.ecr.eu-west-3.amazonaws.com/quivr:c746eb18303945a1736c89427026b509f501e715",
+            "cpu": 0,
+            "portMappings": [
+                {
+                    "name": "quivr-upload-5050-tcp",
+                    "containerPort": 5050,
+                    "hostPort": 5050,
+                    "protocol": "tcp",
+                    "appProtocol": "http"
+                }
+            ],
+            "essential": true,
+            "command": [
+                "uvicorn",
+                "upload_service:app",
+                "--host",
+                "0.0.0.0",
+                "--port",
+                "5050"
+            ],
+            "environment": [],
+            "environmentFiles": [
+                {
+                    "value": "arn:aws:s3:::quivr-env-variables/preview.env",
+                    "type": "s3"
+                }
+            ],
+            "mountPoints": [],
+            "volumesFrom": [],
+            "logConfiguration": {
+                "logDriver": "awslogs",
+                "options": {
+                    "awslogs-create-group": "true",
+                    "awslogs-group": "/ecs/quivr-preview-upload",
+                    "awslogs-region": "eu-west-3",
+                    "awslogs-stream-prefix": "ecs"
+                }
+            }
+        }
+    ],
+    "family": "quivr-preview-upload",
+    "taskRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole",
+    "executionRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole",
+    "networkMode": "awsvpc",
+    "revision": 1,
+    "volumes": [],
+    "status": "ACTIVE",
+    "requiresAttributes": [
+        {
+            "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
+        },
+        {
+            "name": "ecs.capability.execution-role-awslogs"
+        },
+        {
+            "name": "com.amazonaws.ecs.capability.ecr-auth"
+        },
+        {
+            "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
+        },
+        {
+            "name": "ecs.capability.env-files.s3"
+        },
+        {
+            "name": "com.amazonaws.ecs.capability.task-iam-role"
+        },
+        {
+            "name": "ecs.capability.execution-role-ecr-pull"
+        },
+        {
+            "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
+        },
+        {
+            "name": "ecs.capability.task-eni"
+        },
+        {
+            "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
+        }
+    ],
+    "placementConstraints": [],
+    "compatibilities": [
+        "EC2",
+        "FARGATE"
+    ],
+    "requiresCompatibilities": [
+        "FARGATE"
+    ],
+    "cpu": "256",
+    "memory": "1024",
+    "runtimePlatform": {
+        "cpuArchitecture": "X86_64",
+        "operatingSystemFamily": "LINUX"
+    },
+    "registeredAt": "2023-08-18T12:09:38.819Z",
+    "registeredBy": "arn:aws:iam::253053805092:root",
+    "tags": []
+}
--- a/.github/workflows/aws-preview.yml
+++ b/.github/workflows/aws-preview.yml
@ -7,30 +7,20 @@ on:
      - 'backend/core/**'

 env:
-  AWS_REGION: eu-west-3                 # set this to your preferred AWS region, e.g. us-west-1
-  ECR_REPOSITORY: quivr   # set this to your Amazon ECR repository name
-  ECS_SERVICE: quivr-preview                # set this to your Amazon ECS service name
-  ECS_SERVICE_CHAT: preview-service-chat                # set this to your Amazon ECS service name
-  ECS_CLUSTER: quivr                 # set this to your Amazon ECS cluster name
-  ECS_TASK_DEFINITION: .aws/task_definition_preview.json # set this to the path to your Amazon ECS task definition
-  ECS_TASK_DEFINITION_CHAT: .aws/task_definition_preview_chat.json # set this to the path to your Amazon ECS task definition
-                                               # file, e.g. .aws/task-definition.json
-  CONTAINER_NAME: quivr           # set this to the name of the container in the
-                                               # containerDefinitions section of your task definition
-  CONTAINER_NAME_CHAT: quivr-chat           # set this to the name of the container in the
-                                               # containerDefinitions section of your task definition
-
-permissions:
-  contents: read
+  AWS_REGION: eu-west-3
+  ECR_REPOSITORY: quivr
+  ECS_CLUSTER: quivr

 jobs:
-  deploy:
-    name: Deploy
+  build_and_push:
+    name: Build and Push Docker Image
    runs-on: ubuntu-latest
+    environment: production
    defaults:
      run:
        working-directory: ./backend/
-    environment: production
+    outputs:
+      image: ${{ steps.build-image.outputs.image }}

    steps:
    - name: Checkout
@ -53,40 +43,52 @@ jobs:
        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
        IMAGE_TAG: ${{ github.sha }}
      run: |
-        # Build a docker container and
-        # push it to ECR so that it can
-        # be deployed to ECS.
        docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG ./core/
        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-        echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
+        echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"

-    - name: Fill in the new image ID in the Amazon ECS task definition
+  deploy:
+    needs: build_and_push
+    runs-on: ubuntu-latest
+    environment: production
+    strategy:
+      matrix:
+        include:
+          - name: "quivr"
+            service: "quivr-preview"
+            task_definition: ".aws/task_definition_preview.json"
+            container: "quivr"
+          - name: "quivr-chat"
+            service: "preview-service-chat"
+            task_definition: ".aws/task_definition_preview_chat.json"
+            container: "quivr-chat"
+          - name: "quivr-upload"
+            service: "preview-service-upload"
+            task_definition: ".aws/task_definition_preview_upload.json"
+            container: "quivr-upload"
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ env.AWS_REGION }}
+    - name: Fill in the new image ID in the Amazon ECS task definition for ${{ matrix.name }}
      id: task-def
      uses: aws-actions/amazon-ecs-render-task-definition@v1
      with:
-        task-definition: ${{ env.ECS_TASK_DEFINITION }}
-        container-name: ${{ env.CONTAINER_NAME }}
-        image: ${{ steps.build-image.outputs.image }}
-    
-    - name: Fill in the new image ID in the Amazon ECS task definition
-      id: task-def-chat
-      uses: aws-actions/amazon-ecs-render-task-definition@v1
-      with:
-        task-definition: ${{ env.ECS_TASK_DEFINITION_CHAT }}
-        container-name: ${{ env.CONTAINER_NAME_CHAT }}
-        image: ${{ steps.build-image.outputs.image }}
+        task-definition: ${{ matrix.task_definition }}
+        container-name: ${{ matrix.container }}
+        image: ${{ needs.build_and_push.outputs.image }}

-    - name: Deploy Amazon ECS task definition
+    - name: Deploy Amazon ECS task definition for ${{ matrix.name }}
      uses: aws-actions/amazon-ecs-deploy-task-definition@v1
      with:
        task-definition: ${{ steps.task-def.outputs.task-definition }}
-        service: ${{ env.ECS_SERVICE }}
+        service: ${{ matrix.service }}
        cluster: ${{ env.ECS_CLUSTER }}
-        wait-for-service-stability: false
-    - name: Deploy Amazon ECS task definition Chat
-      uses: aws-actions/amazon-ecs-deploy-task-definition@v1
-      with:
-        task-definition: ${{ steps.task-def-chat.outputs.task-definition }}
-        service: ${{ env.ECS_SERVICE_CHAT }}
-        cluster: ${{ env.ECS_CLUSTER }}
-        wait-for-service-stability: false
+        wait-for-service-stability: false
--- a/backend/core/Dockerfile
+++ b/backend/core/Dockerfile
@ -2,7 +2,7 @@
 FROM python:3.11-slim-bullseye

 # Install GEOS library and clean up in one step
-RUN apt-get update && apt-get install -y libgeos-dev pandoc && \
+RUN apt-get update && apt-get install -y libgeos-dev pandoc binutils && \
    rm -rf /var/lib/apt/lists/* && apt-get clean

 WORKDIR /code
@ -10,6 +10,7 @@ WORKDIR /code
 # Copy just the requirements first
 COPY ./requirements.txt .

+
 # Increase timeout might not be necessary but is retained as in original
 RUN pip install --no-cache-dir -r requirements.txt --timeout 100

--- a/backend/core/main.py
+++ b/backend/core/main.py
@ -40,6 +40,7 @@ async def startup_event():
    if not os.path.exists(pypandoc.get_pandoc_path()):
        pypandoc.download_pandoc()

+
 app.include_router(brain_router)
 app.include_router(chat_router)
 app.include_router(crawl_router)