QuivrHQ/quivr
1
1
Fork 0
mirror of https://github.com/QuivrHQ/quivr.git synced 2024-12-15 17:43:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: add multiple upload and crawl in parallel (#1118)

Browse Source 
* feat: explicit accepted files

* feat: un-synchronize upload and chat FileUploader

* feat: add uploading file new ui

* feat: rename +UrlDisplayer to FeedTitleDisplayer

* feat: add icon per file type

* feat: remove file extension on display

* feat: send feed items to backend

* feat: track file upload

* chore: improve dx
This commit is contained in:
Mamadou DICKO 2023-09-07 10:00:45 +02:00 committed by GitHub
parent 0c1a8a9cdd
commit 711eff0863
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 729 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
frontend/app/chat/[chatId]/components/ActionsBar/ActionsBar.tsx
View File

@ -1,18 +1,35 @@
import { ChatInput, KnowledgeToFeed } from "./components";
import { useActionBar } from "./hooks/useActionBar";
import { useKnowledgeUploader } from "./hooks/useKnowledgeUploader";
export const ActionsBar = (): JSX.Element => {
const { isUploading, setIsUploading } = useActionBar();
const { shouldDisplayUploadCard, setShouldDisplayUploadCard } =
useActionBar();
const { addContent, contents, feedBrain, removeContent } =
useKnowledgeUploader();
return (
<div className={isUploading ? "h-full flex flex-col flex-auto" : ""}>
{isUploading && (
<div
className={
shouldDisplayUploadCard ? "h-full flex flex-col flex-auto" : ""
}
>
{shouldDisplayUploadCard && (
<div className="flex flex-1 overflow-y-scroll shadow-md dark:shadow-primary/25 hover:shadow-xl transition-shadow rounded-xl bg-white dark:bg-black border border-black/10 dark:border-white/25 p-6">
<KnowledgeToFeed onClose={() => setIsUploading(false)} />
<KnowledgeToFeed
onClose={() => setShouldDisplayUploadCard(false)}
contents={contents}
addContent={addContent}
removeContent={removeContent}
/>
</div>
)}
<div className="flex mt-1 flex-col w-full shadow-md dark:shadow-primary/25 hover:shadow-xl transition-shadow rounded-xl bg-white dark:bg-black border border-black/10 dark:border-white/25 p-6">
<ChatInput isUploading={isUploading} setIsUploading={setIsUploading} />
<ChatInput
shouldDisplayUploadCard={shouldDisplayUploadCard}
setShouldDisplayUploadCard={setShouldDisplayUploadCard}
feedBrain={() => void feedBrain()}
/>
</div>
</div>
);

21
frontend/app/chat/[chatId]/components/ActionsBar/components/ChatInput/index.tsx
View File

@ -12,13 +12,15 @@ import { FeedBrainInput } from "./components/FeedBrainInput";
import { useChatInput } from "./hooks/useChatInput";
type ChatInputProps = {
isUploading: boolean;
setIsUploading: (isUploading: boolean) => void;
shouldDisplayUploadCard: boolean;
feedBrain: () => void;
setShouldDisplayUploadCard: (shouldDisplayUploadCard: boolean) => void;
};
export const ChatInput = ({
isUploading,
setIsUploading,
shouldDisplayUploadCard,
feedBrain,
setShouldDisplayUploadCard,
}: ChatInputProps): JSX.Element => {
const { setMessage, submitQuestion, chatId, generatingAnswer, message } =
useChatInput();
@ -35,14 +37,14 @@ export const ChatInput = ({
}}
className="sticky flex items-star bottom-0 bg-white dark:bg-black w-full flex justify-center gap-2 z-20"
>
{!isUploading && shouldDisplayUploadButton && (
{!shouldDisplayUploadCard && shouldDisplayUploadButton && (
<div className="flex items-start">
<Button
className="p-0"
variant={"tertiary"}
data-testid="upload-button"
type="button"
onClick={() => setIsUploading(true)}
onClick={() => setShouldDisplayUploadCard(true)}
>
<MdAddCircle className="text-3xl" />
</Button>
@ -50,7 +52,7 @@ export const ChatInput = ({
)}
<div className="flex flex-1 flex-col items-center">
{isUploading ? (
{shouldDisplayUploadCard ? (
<FeedBrainInput />
) : (
<ChatBar
@ -62,12 +64,13 @@ export const ChatInput = ({
</div>
<div className="flex flex-row items-end">
{isUploading ? (
{shouldDisplayUploadCard ? (
<div className="flex items-center">
<Button
disabled={currentBrainId === null}
variant="tertiary"
onClick={() => setIsUploading(false)}
onClick={feedBrain}
type="button"
>
<MdSend className="text-3xl transform -rotate-90" />
</Button>

19
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/KnowledgeToFeed.tsx
View File

@ -7,14 +7,25 @@ import { Divider } from "@/lib/components/ui/Divider";
import { FeedItems } from "./components";
import { Crawler } from "./components/Crawler";
import { FileUploader } from "./components/FileUploader";
import { useKnowledgeToFeed } from "./hooks/useKnowledgeToFeed";
import { FeedItemType, FeedItemUploadType } from "../../types";
type FeedProps = {
onClose: () => void;
contents: FeedItemType[];
addContent: (content: FeedItemType) => void;
removeContent: (index: number) => void;
};
export const KnowledgeToFeed = ({ onClose }: FeedProps): JSX.Element => {
export const KnowledgeToFeed = ({
onClose,
contents,
addContent,
removeContent,
}: FeedProps): JSX.Element => {
const { t } = useTranslation(["translation"]);
const { addContent, contents, removeContent } = useKnowledgeToFeed();
const files: File[] = (
contents.filter((c) => c.source === "upload") as FeedItemUploadType[]
).map((c) => c.file);
return (
<div className="flex flex-col w-full table relative pb-5">
@ -25,7 +36,7 @@ export const KnowledgeToFeed = ({ onClose }: FeedProps): JSX.Element => {
</span>
</Button>
</div>
<FileUploader />
<FileUploader addContent={addContent} files={files} />
<Divider text={t("or")} className="m-5" />
<Crawler addContent={addContent} />
<FeedItems contents={contents} removeContent={removeContent} />

2
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/Crawler/hooks/useCrawler.ts
View File

@ -7,7 +7,7 @@ import { useToast } from "@/lib/hooks";
import { redirectToLogin } from "@/lib/router/redirectToLogin";
import { useEventTracking } from "@/services/analytics/useEventTracking";
import { FeedItemType } from "../../../types";
import { FeedItemType } from "../../../../../types";
import { isValidUrl } from "../helpers/isValidUrl";
type UseCrawlerProps = {

2
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/Crawler/index.tsx
View File

@ -6,7 +6,7 @@ import Button from "@/lib/components/ui/Button";
import Field from "@/lib/components/ui/Field";
import { useCrawler } from "./hooks/useCrawler";
import { FeedItemType } from "../../types";
import { FeedItemType } from "../../../../types";
type CrawlerProps = {
addContent: (content: FeedItemType) => void;

33
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/FeedItems.tsx
View File

@ -1,9 +1,8 @@
import { Fragment } from "react";
import { IoMdCloseCircle } from "react-icons/io";
import { MdLink } from "react-icons/md";
import { UrlDisplay } from "./components";
import { FeedItemType } from "../../types";
import { CrawlFeedItem } from "./components/CrawlFeedItem";
import { FileFeedItem } from "./components/FileFeedItem/FileFeedItem";
import { FeedItemType } from "../../../../types";
type FeedItemsProps = {
contents: FeedItemType[];
@ -20,21 +19,21 @@ export const FeedItems = ({
return (
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-2 gap-4 mt-5 shadow-md shadow-md dark:shadow-primary/25 hover:shadow-xl transition-shadow rounded-xl bg-white dark:bg-black border border-black/10 dark:border-white/25 p-6">
{contents.map((item, index) => (
<div
{contents.map((item, index) =>
item.source === "crawl" ? (
<CrawlFeedItem
key={item.url}
className="relative bg-gray-100 p-4 rounded-lg shadow-sm"
>
<IoMdCloseCircle
className="absolute top-2 right-2 cursor-pointer text-gray-400 text-2xl"
onClick={() => removeContent(index)}
url={item.url}
onRemove={() => removeContent(index)}
/>
<div className="flex items-center">
<MdLink className="mr-2 text-2xl" />
<UrlDisplay url={item.url} />
</div>
</div>
))}
) : (
<FileFeedItem
key={item.file.name}
file={item.file}
onRemove={() => removeContent(index)}
/>
)
)}
</div>
);
};

26
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/CrawlFeedItem.tsx Normal file
View File

@ -0,0 +1,26 @@
import { IoMdCloseCircle } from "react-icons/io";
import { MdLink } from "react-icons/md";
import { FeedTitleDisplayer } from "./FeedTitleDisplayer";
type CrawlFeedItemProps = {
url: string;
onRemove: () => void;
};
export const CrawlFeedItem = ({
url,
onRemove,
}: CrawlFeedItemProps): JSX.Element => {
return (
<div className="relative bg-gray-100 p-4 rounded-lg shadow-sm">
<IoMdCloseCircle
className="absolute top-2 right-2 cursor-pointer text-gray-400 text-2xl"
onClick={onRemove}
/>
<div className="flex items-center">
<MdLink className="mr-2 text-2xl" />
<FeedTitleDisplayer title={url} />
</div>
</div>
);
};

40
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FeedTitleDisplayer/FeedTitleDisplayer.tsx Normal file
View File

@ -0,0 +1,40 @@
import { useState } from "react";
import { enhanceUrlDisplay } from "./utils/enhanceUrlDisplay";
import { removeFileExtension } from "./utils/removeFileExtension";
type FeedTitleDisplayerProps = {
title: string;
truncate?: boolean;
};
export const FeedTitleDisplayer = ({
title,
truncate = false,
}: FeedTitleDisplayerProps): JSX.Element => {
const [showFullUrl, setShowFullUrl] = useState(false);
const toggleShowFullUrl = () => {
setShowFullUrl(!showFullUrl);
};
if (truncate) {
return (
<div className="overflow-hidden">
<span className="cursor-pointer" onClick={toggleShowFullUrl}>
<p className={showFullUrl ? "" : "truncate"}>
{removeFileExtension(title)}
</p>
</span>
</div>
);
}
return (
<div>
<span className="cursor-pointer" onClick={toggleShowFullUrl}>
{showFullUrl ? title : enhanceUrlDisplay(title)}
</span>
</div>
);
};

1
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FeedTitleDisplayer/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./FeedTitleDisplayer";

0
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/UrlDisplay/utils/enhanceUrlDisplay.ts → frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FeedTitleDisplayer/utils/enhanceUrlDisplay.ts
View File

8
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FeedTitleDisplayer/utils/removeFileExtension.ts Normal file
View File

@ -0,0 +1,8 @@
export const removeFileExtension = (fileName: string): string => {
const lastDotIndex = fileName.lastIndexOf(".");
if (lastDotIndex !== -1) {
return fileName.substring(0, lastDotIndex);
}
return fileName;
};

29
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FileFeedItem/FileFeedItem.tsx Normal file
View File

@ -0,0 +1,29 @@
import { IoMdCloseCircle } from "react-icons/io";
import { getFileIcon } from "./helpers/getFileIcon";
import { FeedTitleDisplayer } from "../FeedTitleDisplayer";
type FileFeedItemProps = {
file: File;
onRemove: () => void;
};
export const FileFeedItem = ({
file,
onRemove,
}: FileFeedItemProps): JSX.Element => {
const icon = getFileIcon(file.name);
return (
<div className="relative bg-gray-100 p-4 rounded-lg shadow-sm">
<IoMdCloseCircle
className="absolute top-2 right-2 cursor-pointer text-gray-400 text-2xl"
onClick={onRemove}
/>
<div className="flex items-center">
{icon}
<FeedTitleDisplayer title={file.name} truncate />
</div>
</div>
);
};

52
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FileFeedItem/helpers/getFileIcon.tsx Normal file
View File

@ -0,0 +1,52 @@
import {
BsFiletypeCsv,
BsFiletypeDocx,
BsFiletypeHtml,
BsFiletypeMd,
BsFiletypeMp3,
BsFiletypeMp4,
BsFiletypePdf,
BsFiletypePptx,
BsFiletypePy,
BsFiletypeTxt,
BsFiletypeXls,
BsFiletypeXlsx,
} from "react-icons/bs";
import { FaFile, FaRegFileAudio } from "react-icons/fa";
import { LiaFileVideo } from "react-icons/lia";
import { IconType } from "react-icons/lib";
import { getFileType } from "./getFileType";
import { SupportedFileExtensions } from "../../../../FileUploader/types";
const fileTypeIcons: Record<SupportedFileExtensions, IconType> = {
pdf: BsFiletypePdf,
mp3: BsFiletypeMp3,
mp4: BsFiletypeMp4,
html: BsFiletypeHtml,
txt: BsFiletypeTxt,
csv: BsFiletypeCsv,
md: BsFiletypeMd,
markdown: BsFiletypeMd,
m4a: LiaFileVideo,
mpga: FaRegFileAudio,
mpeg: LiaFileVideo,
webm: LiaFileVideo,
wav: FaRegFileAudio,
pptx: BsFiletypePptx,
docx: BsFiletypeDocx,
odt: BsFiletypeDocx,
xlsx: BsFiletypeXlsx,
xls: BsFiletypeXls,
epub: FaFile,
ipynb: BsFiletypePy,
py: BsFiletypePy,
};
export const getFileIcon = (fileName: string): JSX.Element => {
const fileType = getFileType(fileName);
const Icon = fileType !== undefined ? fileTypeIcons[fileType] : FaFile;
return <Icon className="text-2xl mr-2" />;
};

15
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FileFeedItem/helpers/getFileType.ts Normal file
View File

@ -0,0 +1,15 @@
import {
SupportedFileExtensions,
supportedFileExtensions,
} from "../../../../FileUploader/types";
export const getFileType = (
fileName: string
): SupportedFileExtensions | undefined => {
const extension = fileName.split(".").pop()?.toLowerCase() ?? "";
if (supportedFileExtensions.includes(extension as SupportedFileExtensions)) {
return extension as SupportedFileExtensions;
}
return undefined;
};

1
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/FileFeedItem/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./FileFeedItem";

23
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/UrlDisplay/UrlDisplay.tsx
View File

@ -1,23 +0,0 @@
import { useState } from "react";
import { enhanceUrlDisplay } from "./utils/enhanceUrlDisplay";
type UrlDisplayProps = {
url: string;
};
export const UrlDisplay = ({ url }: UrlDisplayProps): JSX.Element => {
const [showFullUrl, setShowFullUrl] = useState(false);
const toggleShowFullUrl = () => {
setShowFullUrl(!showFullUrl);
};
return (
<div>
<span className="cursor-pointer" onClick={toggleShowFullUrl}>
{showFullUrl ? url : enhanceUrlDisplay(url)}
</span>
</div>
);
};

1
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/UrlDisplay/index.ts
View File

@ -1 +0,0 @@
export * from "./UrlDisplay";

2
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FeedItems/components/index.ts
View File

@ -1 +1 @@
export * from "./UrlDisplay";
export * from "./FeedTitleDisplayer";

127
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FileUploader/hooks/useFileUploader.ts
View File

@ -1,27 +1,29 @@
/* eslint-disable max-lines */
import axios from "axios";
import { UUID } from "crypto";
import { useCallback, useState } from "react";
import { FileRejection, useDropzone } from "react-dropzone";
import { useTranslation } from "react-i18next";
import { useUploadApi } from "@/lib/api/upload/useUploadApi";
import { useBrainContext } from "@/lib/context/BrainProvider/hooks/useBrainContext";
import { useSupabase } from "@/lib/context/SupabaseProvider";
import { useToast } from "@/lib/hooks";
import { redirectToLogin } from "@/lib/router/redirectToLogin";
import { useEventTracking } from "@/services/analytics/useEventTracking";
import { FeedItemType } from "../../../../../types";
import { SupportedFileExtensionsWithDot } from "../types";
type UseFileUploaderProps = {
addContent: (content: FeedItemType) => void;
files: File[];
};
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
export const useFileUploader = () => {
const { track } = useEventTracking();
const [isPending, setIsPending] = useState(false);
export const useFileUploader = ({
addContent,
files,
}: UseFileUploaderProps) => {
const { publish } = useToast();
const [files, setFiles] = useState<File[]>([]);
const { session } = useSupabase();
const { uploadFile } = useUploadApi();
const { currentBrain } = useBrainContext();
const { track } = useEventTracking();
if (session === null) {
redirectToLogin();
@ -29,52 +31,23 @@ export const useFileUploader = () => {
const { t } = useTranslation(["upload"]);
const upload = useCallback(
async (file: File, brainId: UUID) => {
const formData = new FormData();
formData.append("uploadFile", file);
try {
void track("FILE_UPLOADED");
const response = await uploadFile({ brainId, formData });
publish({
variant: response.data.type,
text:
response.data.type === "success"
? t("success", { ns: "upload" })
: t("error", { message: response.data.message, ns: "upload" }),
});
} catch (e: unknown) {
if (axios.isAxiosError(e) && e.response?.status === 403) {
publish({
variant: "danger",
text: `${JSON.stringify(
(
e.response as {
data: { detail: string };
}
).data.detail
)}`,
});
const onDrop = (acceptedFiles: File[], fileRejections: FileRejection[]) => {
if (fileRejections.length > 0) {
const firstRejection = fileRejections[0];
if (firstRejection.errors[0].code === "file-invalid-type") {
publish({ variant: "danger", text: t("invalidFileType") });
} else {
publish({
variant: "danger",
text: t("error", { message: e, ns: "upload" }),
text: t("maxSizeError", { ns: "upload" }),
});
}
}
},
[publish, t, track, uploadFile]
);
const onDrop = (acceptedFiles: File[], fileRejections: FileRejection[]) => {
if (fileRejections.length > 0) {
publish({ variant: "danger", text: t("maxSizeError", { ns: "upload" }) });
return;
}
for (let i = 0; i < acceptedFiles.length; i++) {
const file = acceptedFiles[i];
for (const file of acceptedFiles) {
const isAlreadyInFiles =
files.filter((f) => f.name === file.name && f.size === file.size)
.length > 0;
@ -83,49 +56,53 @@ export const useFileUploader = () => {
variant: "warning",
text: t("alreadyAdded", { fileName: file.name, ns: "upload" }),
});
acceptedFiles.splice(i, 1);
} else {
void track("FILE_UPLOADED");
addContent({
source: "upload",
file: file,
});
}
}
// eslint-disable-next-line @typescript-eslint/no-shadow
setFiles((files) => [...files, ...acceptedFiles]);
};
const uploadAllFiles = async () => {
if (files.length === 0) {
publish({
text: t("addFiles", { ns: "upload" }),
variant: "warning",
});
return;
}
setIsPending(true);
if (currentBrain?.id !== undefined) {
await Promise.all(files.map((file) => upload(file, currentBrain.id)));
setFiles([]);
} else {
publish({
text: t("selectBrain", { ns: "upload" }),
variant: "warning",
});
}
setIsPending(false);
const accept: Record<string, SupportedFileExtensionsWithDot[]> = {
"text/plain": [".txt"],
"text/csv": [".csv"],
"text/markdown": [".md", ".markdown"],
"audio/x-m4a": [".m4a"],
"audio/mpeg": [".mp3", ".mpga", ".mpeg"],
"audio/webm": [".webm"],
"video/mp4": [".mp4"],
"audio/wav": [".wav"],
"application/pdf": [".pdf"],
"text/html": [".html"],
"application/vnd.openxmlformats-officedocument.presentationml.presentation":
[".pptx"],
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
".docx",
],
"application/vnd.oasis.opendocument.text": [".odt"],
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [
".xlsx",
".xls",
],
"application/epub+zip": [".epub"],
"application/x-ipynb+json": [".ipynb"],
"text/x-python": [".py"],
};
const { getInputProps, getRootProps, isDragActive, open } = useDropzone({
onDrop,
noClick: true,
maxSize: 100000000, // 1 MB
accept,
});
return {
isPending,
getInputProps,
getRootProps,
isDragActive,
open,
uploadAllFiles,
files,
setFiles,
};
};

48
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FileUploader/index.tsx
View File

@ -1,24 +1,23 @@
"use client";
import { AnimatePresence } from "framer-motion";
import { useTranslation } from "react-i18next";
import Button from "@/lib/components/ui/Button";
import Card from "@/lib/components/ui/Card";
import FileComponent from "./components/FileComponent";
import { useFileUploader } from "./hooks/useFileUploader";
import { FeedItemType } from "../../../../types";
export const FileUploader = (): JSX.Element => {
const {
getInputProps,
getRootProps,
isDragActive,
isPending,
open,
uploadAllFiles,
type FileUploaderProps = {
addContent: (content: FeedItemType) => void;
files: File[];
};
export const FileUploader = ({
addContent,
files,
setFiles,
} = useFileUploader();
}: FileUploaderProps): JSX.Element => {
const { getInputProps, getRootProps, isDragActive, open } = useFileUploader({
addContent,
files,
});
const { t } = useTranslation(["translation", "upload"]);
@ -45,29 +44,6 @@ export const FileUploader = (): JSX.Element => {
</div>
</Card>
</div>
{files.length > 0 && (
<div className="flex-1 w-full">
<Card className="h-52 py-3 overflow-y-auto">
{files.length > 0 ? (
<AnimatePresence mode="popLayout">
{files.map((file) => (
<FileComponent
key={`${file.name} ${file.size}`}
file={file}
setFiles={setFiles}
/>
))}
</AnimatePresence>
) : null}
</Card>
</div>
)}
</div>
<div className="flex flex-col items-center justify-center">
<Button isLoading={isPending} onClick={() => void uploadAllFiles()}>
{isPending ? t("uploadingButton") : t("uploadButton")}
</Button>
</div>
</section>
);

27
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FileUploader/types.ts Normal file
View File

@ -0,0 +1,27 @@
export const supportedFileExtensions = [
"txt",
"csv",
"md",
"markdown",
"m4a",
"mp3",
"mpga",
"mpeg",
"webm",
"mp4",
"wav",
"pdf",
"html",
"pptx",
"docx",
"odt",
"xlsx",
"xls",
"epub",
"ipynb",
"py",
] as const;
export type SupportedFileExtensions = (typeof supportedFileExtensions)[number];
export type SupportedFileExtensionsWithDot = `.${SupportedFileExtensions}`;

22
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/hooks/useKnowledgeToFeed.ts
View File

@ -1,22 +0,0 @@
import { useState } from "react";
import { FeedItemType } from "../types";
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
export const useKnowledgeToFeed = () => {
const [contents, setContents] = useState<FeedItemType[]>([]);
const addContent = (content: FeedItemType) => {
setContents((prevContents) => [...prevContents, content]);
};
const removeContent = (index: number) => {
setContents((prevContents) => prevContents.filter((_, i) => i !== index));
};
return {
addContent,
contents,
setContents,
removeContent,
};
};

6
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/types.ts
View File

@ -1,6 +0,0 @@
export type FeedItemSource = "crawl" | "upload";
export type FeedItemType = {
source: FeedItemSource;
url: string;
};

6
frontend/app/chat/[chatId]/components/ActionsBar/hooks/useActionBar.ts
View File

@ -2,10 +2,10 @@ import { useState } from "react";
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
export const useActionBar = () => {
const [isUploading, setIsUploading] = useState(false);
const [shouldDisplayUploadCard, setShouldDisplayUploadCard] = useState(false);
return {
isUploading,
setIsUploading,
shouldDisplayUploadCard,
setShouldDisplayUploadCard,
};
};

127
frontend/app/chat/[chatId]/components/ActionsBar/hooks/useKnowledgeUploader.ts Normal file
View File

@ -0,0 +1,127 @@
/* eslint-disable max-lines */
import axios from "axios";
import { UUID } from "crypto";
import { useCallback, useState } from "react";
import { useTranslation } from "react-i18next";
import { useCrawlApi } from "@/lib/api/crawl/useCrawlApi";
import { useUploadApi } from "@/lib/api/upload/useUploadApi";
import { useBrainContext } from "@/lib/context/BrainProvider/hooks/useBrainContext";
import { useToast } from "@/lib/hooks";
import { FeedItemCrawlType, FeedItemType, FeedItemUploadType } from "../types";
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
export const useKnowledgeUploader = () => {
const [contents, setContents] = useState<FeedItemType[]>([]);
const { publish } = useToast();
const { uploadFile } = useUploadApi();
const { t } = useTranslation(["upload"]);
const { crawlWebsiteUrl } = useCrawlApi();
const { currentBrainId } = useBrainContext();
const addContent = (content: FeedItemType) => {
setContents((prevContents) => [...prevContents, content]);
};
const removeContent = (index: number) => {
setContents((prevContents) => prevContents.filter((_, i) => i !== index));
};
const crawlWebsiteHandler = useCallback(
async (url: string, brainId: UUID) => {
// Configure parameters
const config = {
url: url,
js: false,
depth: 1,
max_pages: 100,
max_time: 60,
};
try {
await crawlWebsiteUrl({
brainId,
config,
});
} catch (error: unknown) {
publish({
variant: "danger",
text: t("crawlFailed", {
message: JSON.stringify(error),
ns: "upload",
}),
});
}
},
[crawlWebsiteUrl, publish, t]
);
const uploadFileHandler = useCallback(
async (file: File, brainId: UUID) => {
const formData = new FormData();
formData.append("uploadFile", file);
try {
await uploadFile({
brainId: brainId,
formData,
});
} catch (e: unknown) {
if (axios.isAxiosError(e) && e.response?.status === 403) {
publish({
variant: "danger",
text: `${JSON.stringify(
(
e.response as {
data: { detail: string };
}
).data.detail
)}`,
});
} else {
publish({
variant: "danger",
text: t("error", { message: e, ns: "upload" }),
});
}
}
},
[publish, t, uploadFile]
);
const files: File[] = (
contents.filter((c) => c.source === "upload") as FeedItemUploadType[]
).map((c) => c.file);
const urls: string[] = (
contents.filter((c) => c.source === "crawl") as FeedItemCrawlType[]
).map((c) => c.url);
const feedBrain = async (): Promise<void> => {
if (currentBrainId === null) {
publish({
variant: "danger",
text: t("selectBrainFirst"),
});
return;
}
try {
await Promise.all([
...files.map((file) => uploadFileHandler(file, currentBrainId)),
...urls.map((url) => crawlWebsiteHandler(url, currentBrainId)),
]);
} catch (e: unknown) {
publish({
variant: "danger",
text: JSON.stringify(e),
});
}
};
return {
addContent,
contents,
removeContent,
feedBrain,
};
};

13
frontend/app/chat/[chatId]/components/ActionsBar/types.ts
View File

@ -1,3 +1,16 @@
export const mentionTriggers = ["@", "#"] as const;
export type MentionTriggerType = (typeof mentionTriggers)[number];
export type FeedItemSource = "crawl" | "upload";
export type FeedItemCrawlType = {
source: "crawl";
url: string;
};
export type FeedItemUploadType = {
source: "upload";
file: File;
};
export type FeedItemType = FeedItemCrawlType | FeedItemUploadType;

0
frontend/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/FileUploader/components/FileComponent.tsx → frontend/app/upload/FileUploader/components/FileComponent.tsx
View File

164
frontend/app/upload/FileUploader/hooks/useFileUploader.ts Normal file
View File

@ -0,0 +1,164 @@
/* eslint-disable max-lines */
import axios from "axios";
import { UUID } from "crypto";
import { useCallback, useState } from "react";
import { FileRejection, useDropzone } from "react-dropzone";
import { useTranslation } from "react-i18next";
import { useUploadApi } from "@/lib/api/upload/useUploadApi";
import { useBrainContext } from "@/lib/context/BrainProvider/hooks/useBrainContext";
import { useSupabase } from "@/lib/context/SupabaseProvider";
import { useToast } from "@/lib/hooks";
import { redirectToLogin } from "@/lib/router/redirectToLogin";
import { useEventTracking } from "@/services/analytics/useEventTracking";
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
export const useFileUploader = () => {
const { track } = useEventTracking();
const [isPending, setIsPending] = useState(false);
const { publish } = useToast();
const [files, setFiles] = useState<File[]>([]);
const { session } = useSupabase();
const { uploadFile } = useUploadApi();
const { currentBrain } = useBrainContext();
if (session === null) {
redirectToLogin();
}
const { t } = useTranslation(["upload"]);
const upload = useCallback(
async (file: File, brainId: UUID) => {
const formData = new FormData();
formData.append("uploadFile", file);
try {
void track("FILE_UPLOADED");
const response = await uploadFile({ brainId, formData });
publish({
variant: response.data.type,
text:
response.data.type === "success"
? t("success", { ns: "upload" })
: t("error", { message: response.data.message, ns: "upload" }),
});
} catch (e: unknown) {
if (axios.isAxiosError(e) && e.response?.status === 403) {
publish({
variant: "danger",
text: `${JSON.stringify(
(
e.response as {
data: { detail: string };
}
).data.detail
)}`,
});
} else {
publish({
variant: "danger",
text: t("error", { message: e, ns: "upload" }),
});
}
}
},
[publish, t, track, uploadFile]
);
const onDrop = (acceptedFiles: File[], fileRejections: FileRejection[]) => {
if (fileRejections.length > 0) {
const firstRejection = fileRejections[0];
if (firstRejection.errors[0].code === "file-invalid-type") {
publish({ variant: "danger", text: t("invalidFileType") });
} else {
publish({
variant: "danger",
text: t("maxSizeError", { ns: "upload" }),
});
}
return;
}
for (let i = 0; i < acceptedFiles.length; i++) {
const file = acceptedFiles[i];
const isAlreadyInFiles =
files.filter((f) => f.name === file.name && f.size === file.size)
.length > 0;
if (isAlreadyInFiles) {
publish({
variant: "warning",
text: t("alreadyAdded", { fileName: file.name, ns: "upload" }),
});
acceptedFiles.splice(i, 1);
}
}
// eslint-disable-next-line @typescript-eslint/no-shadow
setFiles((files) => [...files, ...acceptedFiles]);
};
const uploadAllFiles = async () => {
if (files.length === 0) {
publish({
text: t("addFiles", { ns: "upload" }),
variant: "warning",
});
return;
}
setIsPending(true);
if (currentBrain?.id !== undefined) {
await Promise.all(files.map((file) => upload(file, currentBrain.id)));
setFiles([]);
} else {
publish({
text: t("selectBrain", { ns: "upload" }),
variant: "warning",
});
}
setIsPending(false);
};
const { getInputProps, getRootProps, isDragActive, open } = useDropzone({
onDrop,
noClick: true,
maxSize: 100000000, // 1 MB
accept: {
"text/plain": [".txt"],
"text/csv": [".csv"],
"text/markdown": [".md", ".markdown"],
"audio/x-m4a": [".m4a"],
"audio/mpeg": [".mp3", ".mpga", ".mpeg"],
"audio/webm": [".webm"],
"video/mp4": [".mp4"],
"audio/wav": [".wav"],
"application/pdf": [".pdf"],
"text/html": [".html"],
"application/vnd.openxmlformats-officedocument.presentationml.presentation":
[".pptx"],
"application/vnd.openxmlformats-officedocument.wordprocessingml.document":
[".docx"],
"application/vnd.oasis.opendocument.text": [".odt"],
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [
".xlsx",
".xls",
],
"application/epub+zip": [".epub"],
"application/x-ipynb+json": [".ipynb"],
"text/x-python": [".py"],
},
});
return {
isPending,
getInputProps,
getRootProps,
isDragActive,
open,
uploadAllFiles,
files,
setFiles,
};
};

74
frontend/app/upload/FileUploader/index.tsx Normal file
View File

@ -0,0 +1,74 @@
"use client";
import { AnimatePresence } from "framer-motion";
import { useTranslation } from "react-i18next";
import Button from "@/lib/components/ui/Button";
import Card from "@/lib/components/ui/Card";
import FileComponent from "./components/FileComponent";
import { useFileUploader } from "./hooks/useFileUploader";
export const FileUploader = (): JSX.Element => {
const {
getInputProps,
getRootProps,
isDragActive,
isPending,
open,
uploadAllFiles,
files,
setFiles,
} = useFileUploader();
const { t } = useTranslation(["translation", "upload"]);
return (
<section
{...getRootProps()}
className="w-full outline-none flex flex-col gap-10 items-center justify-center px-6 py-3"
>
<div className="flex flex-col sm:flex-row max-w-3xl w-full items-center gap-5">
<div className="flex-1 w-full">
<Card className="h-52 flex justify-center items-center">
<input {...getInputProps()} />
<div className="text-center p-6 max-w-sm w-full flex flex-col gap-5 items-center">
{isDragActive ? (
<p className="text-blue-600">{t("drop", { ns: "upload" })}</p>
) : (
<button
onClick={open}
className="opacity-50 h-full cursor-pointer hover:opacity-100 hover:underline transition-opacity"
>
{t("dragAndDrop", { ns: "upload" })}
</button>
)}
</div>
</Card>
</div>
{files.length > 0 && (
<div className="flex-1 w-full">
<Card className="h-52 py-3 overflow-y-auto">
{files.length > 0 ? (
<AnimatePresence mode="popLayout">
{files.map((file) => (
<FileComponent
key={`${file.name} ${file.size}`}
file={file}
setFiles={setFiles}
/>
))}
</AnimatePresence>
) : null}
</Card>
</div>
)}
</div>
<div className="flex flex-col items-center justify-center">
<Button isLoading={isPending} onClick={() => void uploadAllFiles()}>
{isPending ? t("uploadingButton") : t("uploadButton")}
</Button>
</div>
</section>
);
};

2
frontend/app/upload/page.tsx
View File

@ -10,8 +10,8 @@ import { useSupabase } from "@/lib/context/SupabaseProvider";
import { redirectToLogin } from "@/lib/router/redirectToLogin";
import { Crawler } from "./Crawler";
import { FileUploader } from "./FileUploader";
import { requiredRolesForUpload } from "./config";
import { FileUploader } from "../chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components";
const UploadPage = (): JSX.Element => {
const { currentBrain } = useBrainContext();

3
frontend/public/locales/en/upload.json
View File

@ -14,5 +14,6 @@
"crawlFailed": "Failed to crawl website: {{message}}",
"ohNo": "Oh no!",
"selectBrainFirst": "You need to select a brain first. 🧠💡🥲",
"missingNecessaryRole": "You don't have the necessary role to upload content to the selected brain. 🧠💡🥲"
"missingNecessaryRole": "You don't have the necessary role to upload content to the selected brain. 🧠💡🥲",
"invalidFileType": "Invalid file type"
}

3
frontend/public/locales/es/upload.json
View File

@ -14,5 +14,6 @@
"subtitle": "Texto, documento, hojas de cálculo, presentación, audio, video y URLs admitidos",
"success": "Archivo subido correctamente",
"title": "Subir conocimiento",
"webSite": "Ingrese la URL del sitio web"
"webSite": "Ingrese la URL del sitio web",
"invalidFileType": "Tipo de archivo no permitido"
}

3
frontend/public/locales/fr/upload.json
View File

@ -14,5 +14,6 @@
"crawlFailed": "Échec de l'exploration du site web : {{message}}",
"ohNo": "Oh non !",
"selectBrainFirst": "Vous devez d'abord sélectionner un cerveau. 🧠💡🥲",
"missingNecessaryRole": "Vous n'avez pas le rôle nécessaire pour télécharger du contenu dans le cerveau sélectionné. 🧠💡🥲"
"missingNecessaryRole": "Vous n'avez pas le rôle nécessaire pour télécharger du contenu dans le cerveau sélectionné. 🧠💡🥲",
"invalidFileType": "Type de fichier invalide"
}

3
frontend/public/locales/pt-br/upload.json
View File

@ -14,5 +14,6 @@
"crawlFailed": "Falha ao rastrear o site: {{message}}",
"ohNo": "Oh, não!",
"selectBrainFirst": "Você precisa selecionar um cérebro primeiro. 🧠💡🥲",
"missingNecessaryRole": "Você não possui a função necessária para enviar conteúdo para o cérebro selecionado. 🧠💡🥲"
"missingNecessaryRole": "Você não possui a função necessária para enviar conteúdo para o cérebro selecionado. 🧠💡🥲",
"invalidFileType": "Tipo de arquivo inválido"
}

3
frontend/public/locales/ru/upload.json
View File

@ -14,5 +14,6 @@
"crawlFailed": "Не удалось извлечь информацию из сайта: {{message}}",
"ohNo": "О нет!",
"selectBrainFirst": "Сначала вам нужно выбрать мозг. 🧠💡🥲",
"missingNecessaryRole": "У вас нет необходимой роли для загрузки контента в выбранный мозг. 🧠💡🥲"
"missingNecessaryRole": "У вас нет необходимой роли для загрузки контента в выбранный мозг. 🧠💡🥲",
"invalidFileType": "Неверный тип файла"
}

3
frontend/public/locales/zh-cn/upload.json
View File

@ -14,5 +14,6 @@
"crawlFailed": "抓取网站失败：{{message}}",
"ohNo": "不好了!",
"selectBrainFirst": "你需要先选择一个大脑。 🧠💡🥲",
"missingNecessaryRole": "您没有所选大脑的上传权限。 🧠💡🥲"
"missingNecessaryRole": "您没有所选大脑的上传权限。 🧠💡🥲",
"invalidFileType": "文件类型不受支持"
}