Merge branch 'main' into feat/optimization-calls

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.0.228 (2024-04-10)
+
+## What's Changed
+* fix(frontend): phone display issues by @Zewed in https://github.com/QuivrHQ/quivr/pull/2386
+* Patch 1 by @llwp in https://github.com/QuivrHQ/quivr/pull/2388
+* fix: typo in README.md by @bolens in https://github.com/QuivrHQ/quivr/pull/2391
+* feat(ingestion): Add ingestion module and routes by @StanGirard in https://github.com/QuivrHQ/quivr/pull/2393
+* feat: Add seed ingestions to supabase migrations by @StanGirard in https://github.com/QuivrHQ/quivr/pull/2399
+* feat: Add url_required field to IngestionEntity by @StanGirard in https://github.com/QuivrHQ/quivr/pull/2400
+* Feat: Bibtex file uploads by @colesnic in https://github.com/QuivrHQ/quivr/pull/2398
+* fix: logger level and telemetry function calls by @StanGirard in https://github.com/QuivrHQ/quivr/pull/2409
+* fix: Add integration brain to subscription route by @StanGirard in https://github.com/QuivrHQ/quivr/pull/2410
+* feat(frontend): onboarding V2 by @Zewed in https://github.com/QuivrHQ/quivr/pull/2394
+* fix(frontend): onboardind bug by @Zewed in https://github.com/QuivrHQ/quivr/pull/2414
+* fix(frontend): cleaner fix for onboarding by @Zewed in https://github.com/QuivrHQ/quivr/pull/2415
+* feat(analytics): added analytics page  by @Zewed in https://github.com/QuivrHQ/quivr/pull/2416
+
+## New Contributors
+* @llwp made their first contribution in https://github.com/QuivrHQ/quivr/pull/2388
+* @bolens made their first contribution in https://github.com/QuivrHQ/quivr/pull/2391
+* @colesnic made their first contribution in https://github.com/QuivrHQ/quivr/pull/2398
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.227...v0.0.228
+
 ## 0.0.227 (2024-03-28)
 
 ## What's Changed

backend/main.py

@@ -23,6 +23,7 @@ from modules.notification.controller import notification_router
 from modules.onboarding.controller import onboarding_router
 from modules.prompt.controller import prompt_router
 from modules.upload.controller import upload_router
+from modules.analytics.controller.analytics_routes import analytics_router
 from modules.user.controller import user_router
 from packages.utils import handle_request_validation_error
 from packages.utils.telemetry import maybe_send_telemetry
@@ -78,6 +79,7 @@ app.include_router(crawl_router)
 app.include_router(ingestion_router)
 app.include_router(onboarding_router)
 app.include_router(misc_router)
+app.include_router(analytics_router)
 
 app.include_router(upload_router)
 app.include_router(user_router)

backend/modules/analytics/controller/analytics_routes.py

@@ -0,0 +1,22 @@
+from uuid import UUID
+from fastapi import APIRouter, Depends, Query
+from middlewares.auth.auth_bearer import AuthBearer, get_current_user
+from modules.analytics.entity.analytics import Range
+from modules.analytics.service.analytics_service import AnalyticsService
+
+analytics_service = AnalyticsService()
+analytics_router = APIRouter()
+
+@analytics_router.get(
+    "/analytics/brains-usages", dependencies=[Depends(AuthBearer())], tags=["Analytics"]
+)
+async def get_brains_usages(
+    user: UUID = Depends(get_current_user),
+    brain_id: UUID = Query(None),
+    graph_range: Range = Query(Range.WEEK, alias="graph_range")
+):
+    """
+    Get all user brains usages
+    """
+
+    return analytics_service.get_brains_usages(user.id, graph_range, brain_id)

backend/modules/analytics/entity/analytics.py

@@ -0,0 +1,16 @@
+from enum import IntEnum
+from typing import List
+from pydantic import BaseModel
+from datetime import date
+
+class Range(IntEnum):
+    WEEK = 7
+    MONTH = 30
+    QUARTER = 90
+
+class Usage(BaseModel):
+    date: date
+    usage_count: int
+
+class BrainsUsages(BaseModel):
+    usages: List[Usage]

backend/modules/analytics/repository/analytics.py

@@ -0,0 +1,47 @@
+from collections import defaultdict
+from datetime import datetime, timedelta
+from typing import Optional
+from uuid import UUID
+from models.settings import get_supabase_client
+from modules.analytics.entity.analytics import BrainsUsages, Range, Usage
+from modules.brain.service.brain_user_service import BrainUserService
+
+brain_user_service = BrainUserService()
+
+class Analytics:
+    def __init__(self):
+        supabase_client = get_supabase_client()
+        self.db = supabase_client
+
+    def get_brains_usages(self, user_id: UUID, graph_range: Range, brain_id: Optional[UUID] = None) -> BrainsUsages:
+        user_brains = brain_user_service.get_user_brains(user_id)
+        if brain_id is not None:
+            user_brains = [brain for brain in user_brains if brain.id == brain_id]
+
+        usage_per_day = defaultdict(int) 
+
+        for brain in user_brains:
+            chat_history = (
+                self.db.from_("chat_history")
+                .select("*")
+                .filter("brain_id", "eq", str(brain.id))
+                .execute()
+            ).data
+
+            for chat in chat_history:
+                message_time = datetime.strptime(chat['message_time'], "%Y-%m-%dT%H:%M:%S.%f")
+                usage_per_day[message_time.date()] += 1
+
+        start_date = datetime.now().date() - timedelta(days=graph_range)
+        all_dates = [start_date + timedelta(days=i) for i in range(graph_range)]
+        
+        for date in all_dates:
+            usage_per_day[date] += 0 
+
+        usages = sorted(
+            [Usage(date=date, usage_count=count) for date, count in usage_per_day.items() if start_date <= date <= datetime.now().date()],
+            key=lambda usage: usage.date
+        )
+
+        return BrainsUsages(usages=usages)
+

backend/modules/analytics/repository/analytics_interface.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+from uuid import UUID
+from modules.analytics.entity.analytics import BrainsUsages, Range
+
+class AnalyticsInterface(ABC):
+    @abstractmethod
+    def get_brains_usages(self, user_id: UUID, graph_range: Range = Range.WEEK, brain_id: Optional[UUID] = None) -> BrainsUsages:
+        """
+        Get user brains usage
+        Args:
+            user_id (UUID): The id of the user
+            brain_id (Optional[UUID]): The id of the brain, optional
+        """
+        pass

backend/modules/analytics/service/analytics_service.py

@@ -0,0 +1,13 @@
+
+from modules.analytics.repository.analytics import Analytics
+from modules.analytics.repository.analytics_interface import AnalyticsInterface
+
+class AnalyticsService:
+    repository: AnalyticsInterface
+
+    def __init__(self):
+        self.repository = Analytics()
+
+    def get_brains_usages(self, user_id, graph_range, brain_id = None):
+
+        return self.repository.get_brains_usages(user_id, graph_range, brain_id)

backend/modules/knowledge/repository/storage.py

@@ -4,7 +4,6 @@ from modules.knowledge.repository.storage_interface import StorageInterface
 
 logger = get_logger(__name__)
 
-
 class Storage(StorageInterface):
     def __init__(self):
         supabase_client = get_supabase_client()

frontend/app/studio/BrainsTabs/components/Analytics/Analytics.module.scss

@@ -0,0 +1,16 @@
+@use "@/styles/Spacings.module.scss";
+
+.analytics_wrapper {
+  padding: Spacings.$spacing06;
+  display: flex;
+  flex-direction: column;
+
+  .selectors_wrapper {
+    display: flex;
+    justify-content: space-between;
+
+    .selector {
+      width: 300px;
+    }
+  }
+}

frontend/app/studio/BrainsTabs/components/Analytics/Analytics.tsx

@@ -0,0 +1,178 @@
+import {
+  CategoryScale,
+  ChartDataset,
+  Chart as ChartJS,
+  Filler,
+  Legend,
+  LinearScale,
+  LineElement,
+  PointElement,
+  ScriptableContext,
+  Title,
+  Tooltip,
+} from "chart.js";
+import { useLayoutEffect, useState } from "react";
+import { Line } from "react-chartjs-2";
+
+import { formatMinimalBrainsToSelectComponentInput } from "@/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/utils/formatMinimalBrainsToSelectComponentInput";
+import { Range } from "@/lib/api/analytics/types";
+import { useAnalytics } from "@/lib/api/analytics/useAnalyticsApi";
+import { LoaderIcon } from "@/lib/components/ui/LoaderIcon/LoaderIcon";
+import { MessageInfoBox } from "@/lib/components/ui/MessageInfoBox/MessageInfoBox";
+import { SingleSelector } from "@/lib/components/ui/SingleSelector/SingleSelector";
+import { useBrainContext } from "@/lib/context/BrainProvider/hooks/useBrainContext";
+import { useDevice } from "@/lib/hooks/useDevice";
+
+import styles from "./Analytics.module.scss";
+
+ChartJS.register(
+  CategoryScale,
+  Filler,
+  LinearScale,
+  PointElement,
+  LineElement,
+  Title,
+  Tooltip,
+  Legend
+);
+
+export const Analytics = (): JSX.Element => {
+  const { isMobile } = useDevice();
+  const { getBrainsUsages } = useAnalytics();
+  const { allBrains } = useBrainContext();
+  const [chartData, setChartData] = useState({
+    labels: [] as Date[],
+    datasets: [{}] as ChartDataset<"line", number[]>[],
+  });
+  const [currentChartRange, setCurrentChartRange] = useState(
+    Range.WEEK as number
+  );
+  const [selectedBrainId, setSelectedBrainId] = useState<string | null>(null);
+
+  const graphRangeOptions = [
+    { label: "Last 7 days", value: Range.WEEK },
+    { label: "Last 30 days", value: Range.MONTH },
+    { label: "Last 90 days", value: Range.QUARTER },
+  ];
+
+  const brainsWithUploadRights =
+    formatMinimalBrainsToSelectComponentInput(allBrains);
+
+  const selectedGraphRangeOption = graphRangeOptions.find(
+    (option) => option.value === currentChartRange
+  );
+
+  const handleGraphRangeChange = (newValue: number) => {
+    setCurrentChartRange(newValue);
+  };
+
+  useLayoutEffect(() => {
+    void (async () => {
+      try {
+        const res = await getBrainsUsages(selectedBrainId, currentChartRange);
+        const chartLabels = res?.usages.map((usage) => usage.date) as Date[];
+        const chartDataset = res?.usages.map(
+          (usage) => usage.usage_count
+        ) as number[];
+
+        setChartData({
+          labels: chartLabels,
+          datasets: [
+            {
+              label: `Daily questions to ${
+                selectedBrainId
+                  ? allBrains.find((brain) => brain.id === selectedBrainId)
+                      ?.name
+                  : "your brains"
+              }`,
+              data: chartDataset,
+              borderColor: "rgb(75, 192, 192)",
+              backgroundColor: (context: ScriptableContext<"line">) => {
+                const ctx = context.chart.ctx;
+                const gradient = ctx.createLinearGradient(100, 100, 100, 250);
+                gradient.addColorStop(0, "rgba(75, 192, 192, 0.4)");
+                gradient.addColorStop(1, "rgba(75, 192, 192, 0.05)");
+
+                return gradient;
+              },
+              fill: true,
+              tension: 0.2,
+            },
+          ],
+        });
+      } catch (error) {
+        console.error(error);
+      }
+    })();
+  }, [chartData.labels.length, currentChartRange, selectedBrainId]);
+
+  const options = {
+    type: "line",
+    scales: {
+      x: {
+        grid: {
+          display: false,
+        },
+      },
+      y: {
+        beginAtZero: true,
+        grid: {
+          display: false,
+        },
+        ticks: {
+          stepSize: 1,
+        },
+      },
+    },
+  };
+
+  return (
+    <div className={styles.analytics_wrapper}>
+      {!isMobile ? (
+        <div>
+          {chartData.labels.length ? (
+            <>
+              <div className={styles.selectors_wrapper}>
+                <div className={styles.selector}>
+                  <SingleSelector
+                    iconName="calendar"
+                    options={graphRangeOptions}
+                    onChange={(option) => handleGraphRangeChange(option)}
+                    selectedOption={selectedGraphRangeOption}
+                    placeholder="Select range"
+                  />
+                </div>
+                <div className={styles.selector}>
+                  <SingleSelector
+                    iconName="brain"
+                    options={brainsWithUploadRights}
+                    onChange={(brainId) => setSelectedBrainId(brainId)}
+                    onBackClick={() => setSelectedBrainId(null)}
+                    selectedOption={
+                      selectedBrainId
+                        ? {
+                            value: selectedBrainId,
+                            label: allBrains.find(
+                              (brain) => brain.id === selectedBrainId
+                            )?.name as string,
+                          }
+                        : undefined
+                    }
+                    placeholder="Select specific brain"
+                  />
+                </div>
+              </div>
+              <Line data={chartData} options={options} />
+            </>
+          ) : (
+            <LoaderIcon size="big" color="accent" />
+          )}
+        </div>
+      ) : (
+        <MessageInfoBox type="warning">
+          This feature is not available on small screens
+        </MessageInfoBox>
+      )}
+    </div>
+  );
+};

frontend/app/studio/BrainsTabs/components/ManageBrains/BrainsList/BrainsList.tsx

@@ -1,9 +1,8 @@
 import { MinimalBrainForUser } from "@/lib/context/BrainProvider/types";
 
+import { BrainItem } from "./BrainItem/BrainItem";
 import styles from "./BrainsList.module.scss";
 
-import { BrainItem } from "../BrainItem/BrainItem";
-
 type BrainsListProps = {
   brains: MinimalBrainForUser[];
 };

frontend/app/studio/BrainsTabs/components/ManageBrains/ManageBrains.tsx

@@ -1,10 +1,10 @@
 import Spinner from "@/lib/components/ui/Spinner";
 
+import { BrainSearchBar } from "./BrainSearchBar";
+import { BrainsList } from "./BrainsList/BrainsList";
 import styles from "./ManageBrains.module.scss";
 
 import { useBrainsTabs } from "../../hooks/useBrainsTabs";
-import { BrainSearchBar } from "../BrainSearchBar";
-import { BrainsList } from "../BrainsList/BrainsList";
 
 export const ManageBrains = (): JSX.Element => {
   const { searchQuery, isFetchingBrains, setSearchQuery, brains } =

frontend/app/studio/page.tsx

@@ -11,6 +11,7 @@ import { useKnowledgeToFeedContext } from "@/lib/context/KnowledgeToFeedProvider
 import { ButtonType } from "@/lib/types/QuivrButton";
 import { Tab } from "@/lib/types/Tab";
 
+import { Analytics } from "./BrainsTabs/components/Analytics/Analytics";
 import { ManageBrains } from "./BrainsTabs/components/ManageBrains/ManageBrains";
 import styles from "./page.module.scss";
 
@@ -27,10 +28,9 @@ const Studio = (): JSX.Element => {
       iconName: "edit",
     },
     {
-      label: "Analytics - Coming soon",
+      label: "Analytics",
       isSelected: selectedTab === "Analytics",
       onClick: () => setSelectedTab("Analytics"),
-      disabled: true,
       iconName: "graph",
     },
   ];
@@ -66,6 +66,7 @@ const Studio = (): JSX.Element => {
       <div className={styles.content_wrapper}>
         <Tabs tabList={studioTabs} />
         {selectedTab === "Manage my brains" && <ManageBrains />}
+        {selectedTab === "Analytics" && <Analytics />}
       </div>
       <UploadDocumentModal />
       <AddBrainModal />

frontend/lib/api/analytics/analytics.ts

@@ -0,0 +1,23 @@
+import { AxiosInstance } from "axios";
+
+import { BrainsUsages, Range } from "./types";
+
+export const getBrainsUsages = async (
+  axiosInstance: AxiosInstance,
+  brain_id: string | null,
+  graph_range: Range
+): Promise<BrainsUsages | undefined> => {
+  const params = {
+    graph_range: graph_range,
+    brain_id: brain_id,
+  };
+
+  const brainsUsages = (
+    await axiosInstance.get<BrainsUsages | undefined>(
+      "/analytics/brains-usages",
+      { params: params }
+    )
+  ).data;
+
+  return brainsUsages;
+};

frontend/lib/api/analytics/types.ts

@@ -0,0 +1,14 @@
+interface Usage {
+  date: Date;
+  usage_count: number;
+}
+
+export interface BrainsUsages {
+  usages: Usage[];
+}
+
+export enum Range {
+  WEEK = 7,
+  MONTH = 30,
+  QUARTER = 90,
+}

frontend/lib/api/analytics/useAnalyticsApi.ts

@@ -0,0 +1,14 @@
+import { useAxios } from "@/lib/hooks";
+
+import { getBrainsUsages } from "./analytics";
+import { Range } from "./types";
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+export const useAnalytics = () => {
+  const { axiosInstance } = useAxios();
+
+  return {
+    getBrainsUsages: async (brain_id: string | null, graph_range: Range) =>
+      getBrainsUsages(axiosInstance, brain_id, graph_range),
+  };
+};

frontend/lib/components/ui/SingleSelector/SingleSelector.tsx

@@ -19,6 +19,7 @@ type SelectProps<T> = {
   selectedOption: SelectOptionProps<T> | undefined;
   placeholder: string;
   iconName: keyof typeof iconList;
+  onBackClick?: () => void;
 };
 
 export const SingleSelector = <T extends string | number | UUID>({
@@ -27,6 +28,7 @@ export const SingleSelector = <T extends string | number | UUID>({
   selectedOption,
   placeholder,
   iconName,
+  onBackClick,
 }: SelectProps<T>): JSX.Element => {
   const [search, setSearch] = useState<string>("");
   const [folded, setFolded] = useState<boolean>(true);
@@ -60,11 +62,16 @@ export const SingleSelector = <T extends string | number | UUID>({
           className={styles.left}
           onClick={() => {
             setFolded(!folded);
+            if (!folded) {
+              onBackClick?.();
+            }
           }}
         >
           <div className={styles.icon}>
             <Icon
-              name={folded ? "chevronDown" : "chevronRight"}
+              name={
+                folded ? "chevronDown" : onBackClick ? "back" : "chevronRight"
+              }
               size="normal"
               color="black"
             />

frontend/lib/helpers/iconList.ts

@@ -7,6 +7,7 @@ import {
 import { CgSoftwareDownload } from "react-icons/cg";
 import { CiFlag1 } from "react-icons/ci";
 import {
+  FaCalendar,
   FaCheck,
   FaCheckCircle,
   FaDiscord,
@@ -68,7 +69,7 @@ import {
   MdUploadFile,
 } from "react-icons/md";
 import { PiOfficeChairFill } from "react-icons/pi";
-import { RiHashtag } from "react-icons/ri";
+import { RiDeleteBackLine, RiHashtag } from "react-icons/ri";
 import { SlOptions } from "react-icons/sl";
 import { TbNetwork } from "react-icons/tb";
 import { VscGraph } from "react-icons/vsc";
@@ -76,8 +77,10 @@ import { VscGraph } from "react-icons/vsc";
 export const iconList: { [name: string]: IconType } = {
   add: LuPlusCircle,
   addWithoutCircle: IoIosAdd,
+  back: RiDeleteBackLine,
   brain: LuBrain,
   brainCircuit: LuBrainCircuit,
+  calendar: FaCalendar,
   chair: PiOfficeChairFill,
   chat: BsChatLeftText,
   check: FaCheck,

frontend/package.json

@@ -63,6 +63,7 @@
     "autoprefixer": "10.4.16",
     "axios": "^1.6.7",
     "change-case": "^5.4.2",
+    "chart.js": "^4.4.2",
     "class-variance-authority": "0.7.0",
     "clsx": "1.2.1",
     "date-fns": "2.30.0",
@@ -86,6 +87,7 @@
     "prettier": "2.8.8",
     "pretty-bytes": "6.1.1",
     "react": "^18.2.0",
+    "react-chartjs-2": "^5.2.0",
     "react-dom": "^18.2.0",
     "react-dropzone": "14.2.3",
     "react-ga4": "2.1.0",

frontend/yarn.lock

@@ -642,6 +642,11 @@
     typescript "^4.9.5"
     unfetch "^4.1.0"
 
+"@kurkle/color@^0.3.0":
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/@kurkle/color/-/color-0.3.2.tgz#5acd38242e8bde4f9986e7913c8fdf49d3aa199f"
+  integrity sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==
+
 "@lukeed/csprng@^1.1.0":
   version "1.1.0"
   resolved "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz"
@@ -3523,6 +3528,13 @@ character-entities@^2.0.0:
   resolved "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz"
   integrity sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==
 
+chart.js@^4.4.2:
+  version "4.4.2"
+  resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-4.4.2.tgz#95962fa6430828ed325a480cc2d5f2b4e385ac31"
+  integrity sha512-6GD7iKwFpP5kbSD4MeRRRlTnQvxfQREy36uEtm1hzHzcOqwWx0YEHuspuoNlslu+nciLIB7fjjsHkUv/FzFcOg==
+  dependencies:
+    "@kurkle/color" "^0.3.0"
+
 check-error@^1.0.2:
   version "1.0.2"
   resolved "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz"
@@ -7311,6 +7323,11 @@ rc@^1.2.7:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
+react-chartjs-2@^5.2.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/react-chartjs-2/-/react-chartjs-2-5.2.0.tgz#43c1e3549071c00a1a083ecbd26c1ad34d385f5d"
+  integrity sha512-98iN5aguJyVSxp5U3CblRLH67J8gkfyGNbiK3c+l1QI/G4irHMPQw44aEPmjVag+YKTyQ260NcF82GTQ3bdscA==
+
 react-dom@^18.2.0:
   version "18.2.0"
   resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz"

supabase/seed.sql

@@ -205,8 +205,8 @@ INSERT INTO "public"."brains_users" ("brain_id", "rights", "default_brain", "use
 -- Data for Name: integrations; Type: TABLE DATA; Schema: public; Owner: postgres
 --
 
-INSERT INTO "public"."integrations" ("created_at", "integration_name", "integration_logo_url", "connection_settings", "id", "description", "integration_type", "max_files", "information", "tags", "allow_model_change", "integration_display_name") VALUES
-	('2024-03-06 21:21:07.251232+00', 'doc', 'https://quivr-cms.s3.eu-west-3.amazonaws.com/225911_200_46634039e4.png', NULL, 'b37a2275-61b3-460b-b4ab-94dfdf3642fb', 'Default description', 'custom', 5000, NULL, '{}', true, 'Brain');
+INSERT INTO "public"."integrations" ("created_at", "integration_name", "integration_logo_url", "connection_settings", "id", "description", "integration_type", "max_files", "information", "tags", "allow_model_change", "integration_display_name", "onboarding_brain") VALUES
+	('2024-03-06 21:21:07.251232+00', 'doc', 'https://quivr-cms.s3.eu-west-3.amazonaws.com/225911_200_46634039e4.png', NULL, 'b37a2275-61b3-460b-b4ab-94dfdf3642fb', 'Default description', 'custom', 5000, NULL, '{}', true, 'Brain', true);
 
 
 --