diff --git a/.aws/task_definition.json b/.aws/task_definition.json index 89bfedebe..fb57e23f8 100644 --- a/.aws/task_definition.json +++ b/.aws/task_definition.json @@ -4,7 +4,7 @@ { "name": "quivr", "image": "253053805092.dkr.ecr.eu-west-3.amazonaws.com/quivr:bada136312ad3497664c3562a36b263d43c89c53", - "cpu": 512, + "cpu": 256, "memory": 1024, "portMappings": [ { @@ -91,7 +91,7 @@ "requiresCompatibilities": [ "FARGATE" ], - "cpu": "512", + "cpu": "256", "memory": "1024", "runtimePlatform": { "cpuArchitecture": "X86_64", diff --git a/.aws/task_definition_prod_chat.json b/.aws/task_definition_prod_chat.json new file mode 100644 index 000000000..178f4ec79 --- /dev/null +++ b/.aws/task_definition_prod_chat.json @@ -0,0 +1,102 @@ +{ + "taskDefinitionArn": "arn:aws:ecs:eu-west-3:253053805092:task-definition/quivr-prod-chat:2", + "containerDefinitions": [ + { + "name": "quivr-chat", + "image": "253053805092.dkr.ecr.eu-west-3.amazonaws.com/quivr:85ae06c82935028d828f60f35c7e0a47fdee1ff2", + "cpu": 0, + "portMappings": [ + { + "name": "quivr-chat-5050-tcp", + "containerPort": 5050, + "hostPort": 5050, + "protocol": "tcp", + "appProtocol": "http" + } + ], + "essential": true, + "command": [ + "uvicorn", + "main:app", + "--host", + "0.0.0.0", + "--port", + "5050" + ], + "environment": [], + "environmentFiles": [ + { + "value": "arn:aws:s3:::quivr-env-variables/production.env", + "type": "s3" + } + ], + "mountPoints": [], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/quivr-production-chat", + "awslogs-region": "eu-west-3", + "awslogs-stream-prefix": "ecs" + } + } + } + ], + "family": "quivr-prod-chat", + "taskRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole", + "executionRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole", + "networkMode": "awsvpc", + "revision": 2, + "volumes": [], + "status": "ACTIVE", + "requiresAttributes": [ + { + "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" + }, + { + "name": "ecs.capability.execution-role-awslogs" + }, + { + "name": "com.amazonaws.ecs.capability.ecr-auth" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" + }, + { + "name": "ecs.capability.env-files.s3" + }, + { + "name": "com.amazonaws.ecs.capability.task-iam-role" + }, + { + "name": "ecs.capability.execution-role-ecr-pull" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" + }, + { + "name": "ecs.capability.task-eni" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29" + } + ], + "placementConstraints": [], + "compatibilities": [ + "EC2", + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "1024", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "registeredAt": "2023-08-18T15:31:07.701Z", + "registeredBy": "arn:aws:iam::253053805092:root", + "tags": [] +} \ No newline at end of file diff --git a/.aws/task_definition_prod_crawl.json b/.aws/task_definition_prod_crawl.json new file mode 100644 index 000000000..ab868fd5f --- /dev/null +++ b/.aws/task_definition_prod_crawl.json @@ -0,0 +1,102 @@ +{ + "taskDefinitionArn": "arn:aws:ecs:eu-west-3:253053805092:task-definition/quivr-prod-crawl:1", + "containerDefinitions": [ + { + "name": "quivr-crawl", + "image": "253053805092.dkr.ecr.eu-west-3.amazonaws.com/quivr:85ae06c82935028d828f60f35c7e0a47fdee1ff2", + "cpu": 0, + "portMappings": [ + { + "name": "quivr-crawl-5050-tcp", + "containerPort": 5050, + "hostPort": 5050, + "protocol": "tcp", + "appProtocol": "http" + } + ], + "essential": true, + "command": [ + "uvicorn", + "main:app", + "--host", + "0.0.0.0", + "--port", + "5050" + ], + "environment": [], + "environmentFiles": [ + { + "value": "arn:aws:s3:::quivr-env-variables/production.env", + "type": "s3" + } + ], + "mountPoints": [], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/quivr-production-crawl", + "awslogs-region": "eu-west-3", + "awslogs-stream-prefix": "ecs" + } + } + } + ], + "family": "quivr-prod-crawl", + "taskRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole", + "executionRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole", + "networkMode": "awsvpc", + "revision": 1, + "volumes": [], + "status": "ACTIVE", + "requiresAttributes": [ + { + "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" + }, + { + "name": "ecs.capability.execution-role-awslogs" + }, + { + "name": "com.amazonaws.ecs.capability.ecr-auth" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" + }, + { + "name": "ecs.capability.env-files.s3" + }, + { + "name": "com.amazonaws.ecs.capability.task-iam-role" + }, + { + "name": "ecs.capability.execution-role-ecr-pull" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" + }, + { + "name": "ecs.capability.task-eni" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29" + } + ], + "placementConstraints": [], + "compatibilities": [ + "EC2", + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "1024", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "registeredAt": "2023-08-18T15:54:02.492Z", + "registeredBy": "arn:aws:iam::253053805092:root", + "tags": [] +} \ No newline at end of file diff --git a/.aws/task_definition_prod_upload.json b/.aws/task_definition_prod_upload.json new file mode 100644 index 000000000..d30ff18c3 --- /dev/null +++ b/.aws/task_definition_prod_upload.json @@ -0,0 +1,102 @@ +{ + "taskDefinitionArn": "arn:aws:ecs:eu-west-3:253053805092:task-definition/quivr-prod-upload:1", + "containerDefinitions": [ + { + "name": "quivr-upload", + "image": "253053805092.dkr.ecr.eu-west-3.amazonaws.com/quivr:85ae06c82935028d828f60f35c7e0a47fdee1ff2", + "cpu": 0, + "portMappings": [ + { + "name": "quivr-upload-5050-tcp", + "containerPort": 5050, + "hostPort": 5050, + "protocol": "tcp", + "appProtocol": "http" + } + ], + "essential": true, + "command": [ + "uvicorn", + "main:app", + "--host", + "0.0.0.0", + "--port", + "5050" + ], + "environment": [], + "environmentFiles": [ + { + "value": "arn:aws:s3:::quivr-env-variables/production.env", + "type": "s3" + } + ], + "mountPoints": [], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/quivr-production-upload", + "awslogs-region": "eu-west-3", + "awslogs-stream-prefix": "ecs" + } + } + } + ], + "family": "quivr-prod-upload", + "taskRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole", + "executionRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole", + "networkMode": "awsvpc", + "revision": 1, + "volumes": [], + "status": "ACTIVE", + "requiresAttributes": [ + { + "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" + }, + { + "name": "ecs.capability.execution-role-awslogs" + }, + { + "name": "com.amazonaws.ecs.capability.ecr-auth" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" + }, + { + "name": "ecs.capability.env-files.s3" + }, + { + "name": "com.amazonaws.ecs.capability.task-iam-role" + }, + { + "name": "ecs.capability.execution-role-ecr-pull" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" + }, + { + "name": "ecs.capability.task-eni" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29" + } + ], + "placementConstraints": [], + "compatibilities": [ + "EC2", + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "1024", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "registeredAt": "2023-08-18T15:37:58.533Z", + "registeredBy": "arn:aws:iam::253053805092:root", + "tags": [] +} \ No newline at end of file diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index bda801b6e..9ebaec27b 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -3,8 +3,8 @@ name: Deploy Backend to Preview ECS on: push: branches: [ "main" ] - # paths: - # - 'backend/core/**' + paths: + - 'backend/core/**' env: AWS_REGION: eu-west-3 diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 5af5ac5ba..b81b39f6f 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -2,29 +2,20 @@ name: Deploy to Amazon ECS on: push: - tags: - - 'v*' + branches: [ "main" ] + # tags: + # - 'v*' env: - AWS_REGION: eu-west-3 # set this to your preferred AWS region, e.g. us-west-1 - ECR_REPOSITORY: quivr # set this to your Amazon ECR repository name - ECS_SERVICE: quivr-backend # set this to your Amazon ECS service name - ECS_CLUSTER: quivr # set this to your Amazon ECS cluster name - ECS_TASK_DEFINITION: .aws/task_definition.json # set this to the path to your Amazon ECS task definition - # file, e.g. .aws/task-definition.json - CONTAINER_NAME: quivr # set this to the name of the container in the - # containerDefinitions section of your task definition - -permissions: - contents: read + AWS_REGION: eu-west-3 + ECR_REPOSITORY: quivr + ECR_REGISTRY: 253053805092.dkr.ecr.eu-west-3.amazonaws.com + ECS_CLUSTER: quivr jobs: - deploy: - name: Deploy + build_and_push: + name: Build and Push Docker Image runs-on: ubuntu-latest - defaults: - run: - working-directory: ./backend/ environment: production steps: @@ -41,32 +32,77 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Create Docker Cacha Storage Backend + run: | + docker buildx create --use --driver=docker-container + - name: See the file in the runner + run: | + ls -la - name: Build, tag, and push image to Amazon ECR id: build-image + uses: docker/build-push-action@v4 env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ github.sha }} - run: | - # Build a docker container and - # push it to ECR so that it can - # be deployed to ECS. - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG ./core/ - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + with: + context: ./backend/core/ + push: true + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}, ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + needs: build_and_push + runs-on: ubuntu-latest + environment: production + strategy: + fail-fast: false + matrix: + include: + - name: "quivr" + service: "quivr-backend" + task_definition: ".aws/task_definition.json" + container: "quivr" + - name: "quivr-chat" + service: "prod-service-chat" + task_definition: ".aws/task_definition_prod_chat.json" + container: "quivr-chat" + - name: "quivr-upload" + service: "prod-service-upload" + task_definition: ".aws/task_definition_prod_upload.json" + container: "quivr-upload" + - name: "quivr-crawl" + service: "prod-service-crawl" + task_definition: ".aws/task_definition_prod_crawl.json" + container: "quivr-crawl" - - name: Fill in the new image ID in the Amazon ECS task definition + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Fill in the new image ID in the Amazon ECS task definition for ${{ matrix.name }} id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} - container-name: ${{ env.CONTAINER_NAME }} - image: ${{ steps.build-image.outputs.image }} + task-definition: ${{ matrix.task_definition }} + container-name: ${{ matrix.container }} + image: ${{env.ECR_REGISTRY}}/${{ env.ECR_REPOSITORY }}:${{ github.sha }} - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: ${{ env.ECS_SERVICE }} - cluster: ${{ env.ECS_CLUSTER }} - wait-for-service-stability: true \ No newline at end of file + # - name: Deploy Amazon ECS task definition for ${{ matrix.name }} + # uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + # with: + # task-definition: ${{ steps.task-def.outputs.task-definition }} + # service: ${{ matrix.service }} + # cluster: ${{ env.ECS_CLUSTER }} + # wait-for-service-stability: true \ No newline at end of file