mirror of
https://github.com/StanGirard/quivr.git
synced 2024-08-17 00:50:37 +03:00
feat(preview): added service upload (#979)
* feat(docker): improved size image * feat(preview): added upload service * ci(aws): using matrix
This commit is contained in:
parent
c746eb1830
commit
ce6b45e1ac
@ -92,7 +92,7 @@
|
||||
"FARGATE"
|
||||
],
|
||||
"cpu": "256",
|
||||
"memory": "512",
|
||||
"memory": "1024",
|
||||
"runtimePlatform": {
|
||||
"cpuArchitecture": "X86_64",
|
||||
"operatingSystemFamily": "LINUX"
|
||||
|
@ -91,7 +91,7 @@
|
||||
"FARGATE"
|
||||
],
|
||||
"cpu": "256",
|
||||
"memory": "512",
|
||||
"memory": "1024",
|
||||
"runtimePlatform": {
|
||||
"cpuArchitecture": "X86_64",
|
||||
"operatingSystemFamily": "LINUX"
|
||||
|
102
.aws/task_definition_preview_upload.json
Normal file
102
.aws/task_definition_preview_upload.json
Normal file
@ -0,0 +1,102 @@
|
||||
{
|
||||
"taskDefinitionArn": "arn:aws:ecs:eu-west-3:253053805092:task-definition/quivr-preview-upload:1",
|
||||
"containerDefinitions": [
|
||||
{
|
||||
"name": "quivr-upload",
|
||||
"image": "253053805092.dkr.ecr.eu-west-3.amazonaws.com/quivr:c746eb18303945a1736c89427026b509f501e715",
|
||||
"cpu": 0,
|
||||
"portMappings": [
|
||||
{
|
||||
"name": "quivr-upload-5050-tcp",
|
||||
"containerPort": 5050,
|
||||
"hostPort": 5050,
|
||||
"protocol": "tcp",
|
||||
"appProtocol": "http"
|
||||
}
|
||||
],
|
||||
"essential": true,
|
||||
"command": [
|
||||
"uvicorn",
|
||||
"upload_service:app",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
"5050"
|
||||
],
|
||||
"environment": [],
|
||||
"environmentFiles": [
|
||||
{
|
||||
"value": "arn:aws:s3:::quivr-env-variables/preview.env",
|
||||
"type": "s3"
|
||||
}
|
||||
],
|
||||
"mountPoints": [],
|
||||
"volumesFrom": [],
|
||||
"logConfiguration": {
|
||||
"logDriver": "awslogs",
|
||||
"options": {
|
||||
"awslogs-create-group": "true",
|
||||
"awslogs-group": "/ecs/quivr-preview-upload",
|
||||
"awslogs-region": "eu-west-3",
|
||||
"awslogs-stream-prefix": "ecs"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"family": "quivr-preview-upload",
|
||||
"taskRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole",
|
||||
"executionRoleArn": "arn:aws:iam::253053805092:role/ecsTaskExecutionRole",
|
||||
"networkMode": "awsvpc",
|
||||
"revision": 1,
|
||||
"volumes": [],
|
||||
"status": "ACTIVE",
|
||||
"requiresAttributes": [
|
||||
{
|
||||
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
|
||||
},
|
||||
{
|
||||
"name": "ecs.capability.execution-role-awslogs"
|
||||
},
|
||||
{
|
||||
"name": "com.amazonaws.ecs.capability.ecr-auth"
|
||||
},
|
||||
{
|
||||
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
|
||||
},
|
||||
{
|
||||
"name": "ecs.capability.env-files.s3"
|
||||
},
|
||||
{
|
||||
"name": "com.amazonaws.ecs.capability.task-iam-role"
|
||||
},
|
||||
{
|
||||
"name": "ecs.capability.execution-role-ecr-pull"
|
||||
},
|
||||
{
|
||||
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
|
||||
},
|
||||
{
|
||||
"name": "ecs.capability.task-eni"
|
||||
},
|
||||
{
|
||||
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
|
||||
}
|
||||
],
|
||||
"placementConstraints": [],
|
||||
"compatibilities": [
|
||||
"EC2",
|
||||
"FARGATE"
|
||||
],
|
||||
"requiresCompatibilities": [
|
||||
"FARGATE"
|
||||
],
|
||||
"cpu": "256",
|
||||
"memory": "1024",
|
||||
"runtimePlatform": {
|
||||
"cpuArchitecture": "X86_64",
|
||||
"operatingSystemFamily": "LINUX"
|
||||
},
|
||||
"registeredAt": "2023-08-18T12:09:38.819Z",
|
||||
"registeredBy": "arn:aws:iam::253053805092:root",
|
||||
"tags": []
|
||||
}
|
90
.github/workflows/aws-preview.yml
vendored
90
.github/workflows/aws-preview.yml
vendored
@ -7,30 +7,20 @@ on:
|
||||
- 'backend/core/**'
|
||||
|
||||
env:
|
||||
AWS_REGION: eu-west-3 # set this to your preferred AWS region, e.g. us-west-1
|
||||
ECR_REPOSITORY: quivr # set this to your Amazon ECR repository name
|
||||
ECS_SERVICE: quivr-preview # set this to your Amazon ECS service name
|
||||
ECS_SERVICE_CHAT: preview-service-chat # set this to your Amazon ECS service name
|
||||
ECS_CLUSTER: quivr # set this to your Amazon ECS cluster name
|
||||
ECS_TASK_DEFINITION: .aws/task_definition_preview.json # set this to the path to your Amazon ECS task definition
|
||||
ECS_TASK_DEFINITION_CHAT: .aws/task_definition_preview_chat.json # set this to the path to your Amazon ECS task definition
|
||||
# file, e.g. .aws/task-definition.json
|
||||
CONTAINER_NAME: quivr # set this to the name of the container in the
|
||||
# containerDefinitions section of your task definition
|
||||
CONTAINER_NAME_CHAT: quivr-chat # set this to the name of the container in the
|
||||
# containerDefinitions section of your task definition
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
AWS_REGION: eu-west-3
|
||||
ECR_REPOSITORY: quivr
|
||||
ECS_CLUSTER: quivr
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy
|
||||
build_and_push:
|
||||
name: Build and Push Docker Image
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ./backend/
|
||||
environment: production
|
||||
outputs:
|
||||
image: ${{ steps.build-image.outputs.image }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@ -53,40 +43,52 @@ jobs:
|
||||
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
|
||||
IMAGE_TAG: ${{ github.sha }}
|
||||
run: |
|
||||
# Build a docker container and
|
||||
# push it to ECR so that it can
|
||||
# be deployed to ECS.
|
||||
docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG ./core/
|
||||
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
|
||||
echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
|
||||
echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
|
||||
|
||||
- name: Fill in the new image ID in the Amazon ECS task definition
|
||||
deploy:
|
||||
needs: build_and_push
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- name: "quivr"
|
||||
service: "quivr-preview"
|
||||
task_definition: ".aws/task_definition_preview.json"
|
||||
container: "quivr"
|
||||
- name: "quivr-chat"
|
||||
service: "preview-service-chat"
|
||||
task_definition: ".aws/task_definition_preview_chat.json"
|
||||
container: "quivr-chat"
|
||||
- name: "quivr-upload"
|
||||
service: "preview-service-upload"
|
||||
task_definition: ".aws/task_definition_preview_upload.json"
|
||||
container: "quivr-upload"
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v2
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ env.AWS_REGION }}
|
||||
- name: Fill in the new image ID in the Amazon ECS task definition for ${{ matrix.name }}
|
||||
id: task-def
|
||||
uses: aws-actions/amazon-ecs-render-task-definition@v1
|
||||
with:
|
||||
task-definition: ${{ env.ECS_TASK_DEFINITION }}
|
||||
container-name: ${{ env.CONTAINER_NAME }}
|
||||
image: ${{ steps.build-image.outputs.image }}
|
||||
|
||||
- name: Fill in the new image ID in the Amazon ECS task definition
|
||||
id: task-def-chat
|
||||
uses: aws-actions/amazon-ecs-render-task-definition@v1
|
||||
with:
|
||||
task-definition: ${{ env.ECS_TASK_DEFINITION_CHAT }}
|
||||
container-name: ${{ env.CONTAINER_NAME_CHAT }}
|
||||
image: ${{ steps.build-image.outputs.image }}
|
||||
task-definition: ${{ matrix.task_definition }}
|
||||
container-name: ${{ matrix.container }}
|
||||
image: ${{ needs.build_and_push.outputs.image }}
|
||||
|
||||
- name: Deploy Amazon ECS task definition
|
||||
- name: Deploy Amazon ECS task definition for ${{ matrix.name }}
|
||||
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
|
||||
with:
|
||||
task-definition: ${{ steps.task-def.outputs.task-definition }}
|
||||
service: ${{ env.ECS_SERVICE }}
|
||||
service: ${{ matrix.service }}
|
||||
cluster: ${{ env.ECS_CLUSTER }}
|
||||
wait-for-service-stability: false
|
||||
- name: Deploy Amazon ECS task definition Chat
|
||||
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
|
||||
with:
|
||||
task-definition: ${{ steps.task-def-chat.outputs.task-definition }}
|
||||
service: ${{ env.ECS_SERVICE_CHAT }}
|
||||
cluster: ${{ env.ECS_CLUSTER }}
|
||||
wait-for-service-stability: false
|
||||
wait-for-service-stability: false
|
@ -2,7 +2,7 @@
|
||||
FROM python:3.11-slim-bullseye
|
||||
|
||||
# Install GEOS library and clean up in one step
|
||||
RUN apt-get update && apt-get install -y libgeos-dev pandoc && \
|
||||
RUN apt-get update && apt-get install -y libgeos-dev pandoc binutils && \
|
||||
rm -rf /var/lib/apt/lists/* && apt-get clean
|
||||
|
||||
WORKDIR /code
|
||||
@ -10,6 +10,7 @@ WORKDIR /code
|
||||
# Copy just the requirements first
|
||||
COPY ./requirements.txt .
|
||||
|
||||
|
||||
# Increase timeout might not be necessary but is retained as in original
|
||||
RUN pip install --no-cache-dir -r requirements.txt --timeout 100
|
||||
|
||||
|
@ -40,6 +40,7 @@ async def startup_event():
|
||||
if not os.path.exists(pypandoc.get_pandoc_path()):
|
||||
pypandoc.download_pandoc()
|
||||
|
||||
|
||||
app.include_router(brain_router)
|
||||
app.include_router(chat_router)
|
||||
app.include_router(crawl_router)
|
||||
|
Loading…
Reference in New Issue
Block a user