feat: add a cron to remove onboarding more than 7 days (#1397)

https://github.com/StanGirard/quivr/issues/1385

backend/celery_worker.py

@@ -3,6 +3,7 @@ import io
 import os
 
 from celery import Celery
+from celery.schedules import crontab
 from crawl.crawler import CrawlWebsite
 from fastapi import UploadFile
 from models.databases.supabase.notifications import NotificationUpdatableProperties
@@ -14,11 +15,15 @@ from repository.brain.update_brain_last_update_time import (
     update_brain_last_update_time,
 )
 from repository.notification.update_notification import update_notification_by_id
+from repository.onboarding.remove_onboarding_more_than_x_days import (
+    remove_onboarding_more_than_x_days,
+)
 from utils.processors import filter_file
 
 CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "")
 CELEBRY_BROKER_QUEUE_NAME = os.getenv("CELEBRY_BROKER_QUEUE_NAME", "quivr")
 
+
 if CELERY_BROKER_URL.startswith("sqs"):
     broker_transport_options = {
         CELEBRY_BROKER_QUEUE_NAME: {
@@ -165,3 +170,16 @@ def process_crawl_and_notify(
         )
     update_brain_last_update_time(brain_id)
     return True
+
+
+@celery.task
+def remove_onboarding_more_than_x_days_task():
+    remove_onboarding_more_than_x_days(7)
+
+
+celery.conf.beat_schedule = {
+    "remove_onboarding_more_than_x_days_task": {
+        "task": f"{__name__}.remove_onboarding_more_than_x_days_task",
+        "schedule": crontab(minute="0", hour="0"),
+    },
+}

backend/models/databases/repository.py

@@ -271,3 +271,7 @@ class Repository(ABC):
     @abstractmethod
     def remove_user_onboarding(self, user_id: UUID):
         pass
+
+    @abstractmethod
+    def remove_onboarding_more_than_x_days(self, days: int):
+        pass

backend/models/databases/supabase/onboarding.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from typing import Optional
 from uuid import UUID
 
@@ -92,3 +93,21 @@ class Onboarding(Repository):
             return None
 
         return OnboardingStates(**onboarding_data[0])
+
+    def remove_onboarding_more_than_x_days(self, days: int):
+        """
+        Remove onboarding if it is older than x days
+        """
+        onboarding_data = (
+            self.db.from_("onboardings")
+            .delete()
+            .lt(
+                "creation_time",
+                (datetime.now() - timedelta(days=days)).strftime(
+                    "%Y-%m-%d %H:%M:%S.%f"
+                ),
+            )
+            .execute()
+        ).data
+
+        return onboarding_data

backend/repository/onboarding/remove_onboarding_more_than_x_days.py

@@ -0,0 +1,10 @@
+from models.settings import get_supabase_db
+
+
+def remove_onboarding_more_than_x_days(days: int):
+    """
+    Remove onboarding if it is older than x days
+    """
+    supabase_db = get_supabase_db()
+
+    supabase_db.remove_onboarding_more_than_x_days(days)

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 
 services:
   frontend:
@@ -25,6 +25,7 @@ services:
     depends_on:
       - redis
       - worker
+      - beat
     ports:
       - 5050:5050
 
@@ -49,6 +50,19 @@ services:
     volumes:
       - ./backend/:/code/
 
+  beat:
+    env_file:
+      - ./backend/.env
+    build:
+      context: backend
+      dockerfile: Dockerfile
+    container_name: beat
+    command: celery -A celery_worker beat -l info
+    restart: always
+    depends_on:
+      - redis
+    volumes:
+      - ./backend/:/code/
 
   flower:
     env_file:
@@ -64,5 +78,6 @@ services:
     depends_on:
       - redis
       - worker
+      - beat
     ports:
       - 5555:5555

scripts/20231012150000_add_creation_time_to_onboardings_table.sql

@@ -0,0 +1,23 @@
+-- Add creation_time column to 'onboardings' table if it doesn't exist
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1
+    FROM information_schema.columns
+    WHERE table_name = 'onboardings'
+    AND column_name = 'creation_time'
+  ) THEN
+    ALTER TABLE onboardings ADD COLUMN creation_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP;
+  END IF;
+END
+$$;
+
+-- Insert migration record if it doesn't exist
+INSERT INTO migrations (name) 
+SELECT '20231012150000_add_creation_time_to_onboardings_table'
+WHERE NOT EXISTS (
+  SELECT 1 FROM migrations WHERE name = '20231012150000_add_creation_time_to_onboardings_table'
+);
+
+-- Commit the changes
+COMMIT;

scripts/tables.sql

@@ -283,6 +283,7 @@ CREATE TABLE IF NOT EXISTS onboardings (
   onboarding_b1 BOOLEAN NOT NULL DEFAULT true,
   onboarding_b2 BOOLEAN NOT NULL DEFAULT true,
   onboarding_b3 BOOLEAN NOT NULL DEFAULT true,
+  creation_time TIMESTAMP DEFAULT current_timestamp,
   PRIMARY KEY (user_id)
 );
 
@@ -300,9 +301,9 @@ CREATE POLICY "Access Quivr Storage 1jccrwz_2" ON storage.objects FOR UPDATE TO
 CREATE POLICY "Access Quivr Storage 1jccrwz_3" ON storage.objects FOR DELETE TO anon USING (bucket_id = 'quivr');
 
 INSERT INTO migrations (name) 
-SELECT '20231010120000_add_create_user_onboarding_function'
+SELECT '20231012150000_add_creation_time_to_onboardings_table'
 WHERE NOT EXISTS (
-    SELECT 1 FROM migrations WHERE name = '20231010120000_add_create_user_onboarding_function'
+    SELECT 1 FROM migrations WHERE name = '20231012150000_add_creation_time_to_onboardings_table'
 );