mirror of
https://github.com/StanGirard/quivr.git
synced 2024-09-11 13:15:41 +03:00
feat(crawler): add multiple urls support (#1112)
* feat(Field): add icon support * feat(Crawler): replace submit button with send icon * feat(crawler): add multiple urls support * feat: add <FeedItems/> component to display adding items * feat(FeedItems): add remove icon * feat: add url displayer * feat: add invalid url message * fix: add crawler to upload page * feat: clean sueCrawler * feat: rename Feed to KnowledgeToFeed * feat: add tracking
This commit is contained in:
parent
02964c4077
commit
f230bc10de
@ -1,14 +1,14 @@
|
||||
import { ChatInput, Feed } from "./components";
|
||||
import { ChatInput, KnowledgeToFeed } from "./components";
|
||||
import { useActionBar } from "./hooks/useActionBar";
|
||||
|
||||
export const ActionsBar = (): JSX.Element => {
|
||||
const { isUploading, setIsUploading } = useActionBar();
|
||||
|
||||
return (
|
||||
<div className={isUploading ? "h-full" : ""}>
|
||||
<div className={isUploading ? "h-full flex flex-col flex-auto" : ""}>
|
||||
{isUploading && (
|
||||
<div className="shadow-md dark:shadow-primary/25 hover:shadow-xl transition-shadow rounded-xl bg-white dark:bg-black border border-black/10 dark:border-white/25 p-6">
|
||||
<Feed onClose={() => setIsUploading(false)} />
|
||||
<div className="flex flex-1 overflow-y-scroll shadow-md dark:shadow-primary/25 hover:shadow-xl transition-shadow rounded-xl bg-white dark:bg-black border border-black/10 dark:border-white/25 p-6">
|
||||
<KnowledgeToFeed onClose={() => setIsUploading(false)} />
|
||||
</div>
|
||||
)}
|
||||
<div className="flex mt-1 flex-col w-full shadow-md dark:shadow-primary/25 hover:shadow-xl transition-shadow rounded-xl bg-white dark:bg-black border border-black/10 dark:border-white/25 p-6">
|
||||
|
@ -1,9 +0,0 @@
|
||||
export const isValidUrl = (string: string): boolean => {
|
||||
try {
|
||||
new URL(string);
|
||||
|
||||
return true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
};
|
@ -1 +0,0 @@
|
||||
export * from "./Feed";
|
@ -4,17 +4,20 @@ import { MdClose } from "react-icons/md";
|
||||
import Button from "@/lib/components/ui/Button";
|
||||
import { Divider } from "@/lib/components/ui/Divider";
|
||||
|
||||
import { FeedItems } from "./components";
|
||||
import { Crawler } from "./components/Crawler";
|
||||
import { FileUploader } from "./components/FileUploader";
|
||||
import { useKnowledgeToFeed } from "./hooks/useKnowledgeToFeed";
|
||||
|
||||
type FeedProps = {
|
||||
onClose: () => void;
|
||||
};
|
||||
export const Feed = ({ onClose }: FeedProps): JSX.Element => {
|
||||
export const KnowledgeToFeed = ({ onClose }: FeedProps): JSX.Element => {
|
||||
const { t } = useTranslation(["translation"]);
|
||||
const { addContent, contents, removeContent } = useKnowledgeToFeed();
|
||||
|
||||
return (
|
||||
<div className="flex flex-col w-full relative">
|
||||
<div className="flex flex-col w-full table relative pb-5">
|
||||
<div className="absolute right-2 top-1">
|
||||
<Button variant={"tertiary"} onClick={onClose}>
|
||||
<span>
|
||||
@ -24,7 +27,8 @@ export const Feed = ({ onClose }: FeedProps): JSX.Element => {
|
||||
</div>
|
||||
<FileUploader />
|
||||
<Divider text={t("or")} className="m-5" />
|
||||
<Crawler />
|
||||
<Crawler addContent={addContent} />
|
||||
<FeedItems contents={contents} removeContent={removeContent} />
|
||||
</div>
|
||||
);
|
||||
};
|
@ -0,0 +1,13 @@
|
||||
export const isValidUrl = (urlString: string): boolean => {
|
||||
const urlPattern = new RegExp(
|
||||
"^(https?:\\/\\/)?" + // validate protocol
|
||||
"((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|" + // validate domain name
|
||||
"((\\d{1,3}\\.){3}\\d{1,3}))" + // validate OR ip (v4) address
|
||||
"(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*" + // validate port and path
|
||||
"(\\?[;&a-z\\d%_.~+=-]*)?" + // validate query string
|
||||
"(\\#[-a-z\\d_]*)?$",
|
||||
"i"
|
||||
); // validate fragment locator
|
||||
|
||||
return !!urlPattern.test(urlString);
|
||||
};
|
@ -0,0 +1,57 @@
|
||||
"use client";
|
||||
import { useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import { useSupabase } from "@/lib/context/SupabaseProvider";
|
||||
import { useToast } from "@/lib/hooks";
|
||||
import { redirectToLogin } from "@/lib/router/redirectToLogin";
|
||||
import { useEventTracking } from "@/services/analytics/useEventTracking";
|
||||
|
||||
import { FeedItemType } from "../../../types";
|
||||
import { isValidUrl } from "../helpers/isValidUrl";
|
||||
|
||||
type UseCrawlerProps = {
|
||||
addContent: (content: FeedItemType) => void;
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
||||
export const useCrawler = ({ addContent }: UseCrawlerProps) => {
|
||||
const urlInputRef = useRef<HTMLInputElement | null>(null);
|
||||
const { session } = useSupabase();
|
||||
const { publish } = useToast();
|
||||
const { t } = useTranslation(["translation", "upload"]);
|
||||
const [urlToCrawl, setUrlToCrawl] = useState<string>("");
|
||||
const { track } = useEventTracking();
|
||||
|
||||
if (session === null) {
|
||||
redirectToLogin();
|
||||
}
|
||||
|
||||
const handleSubmit = () => {
|
||||
if (urlToCrawl === "") {
|
||||
return;
|
||||
}
|
||||
if (!isValidUrl(urlToCrawl)) {
|
||||
void track("URL_INVALID");
|
||||
publish({
|
||||
variant: "danger",
|
||||
text: t("invalidUrl"),
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
void track("URL_CRAWLED");
|
||||
addContent({
|
||||
source: "crawl",
|
||||
url: urlToCrawl,
|
||||
});
|
||||
setUrlToCrawl("");
|
||||
};
|
||||
|
||||
return {
|
||||
urlInputRef,
|
||||
urlToCrawl,
|
||||
setUrlToCrawl,
|
||||
handleSubmit,
|
||||
};
|
||||
};
|
@ -0,0 +1,49 @@
|
||||
"use client";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { MdSend } from "react-icons/md";
|
||||
|
||||
import Button from "@/lib/components/ui/Button";
|
||||
import Field from "@/lib/components/ui/Field";
|
||||
|
||||
import { useCrawler } from "./hooks/useCrawler";
|
||||
import { FeedItemType } from "../../types";
|
||||
|
||||
type CrawlerProps = {
|
||||
addContent: (content: FeedItemType) => void;
|
||||
};
|
||||
|
||||
export const Crawler = ({ addContent }: CrawlerProps): JSX.Element => {
|
||||
const { urlInputRef, urlToCrawl, handleSubmit, setUrlToCrawl } = useCrawler({
|
||||
addContent,
|
||||
});
|
||||
const { t } = useTranslation(["translation", "upload"]);
|
||||
|
||||
return (
|
||||
<div className="w-full flex justify-center items-center">
|
||||
<div className="max-w-xl w-full">
|
||||
<form
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
handleSubmit();
|
||||
}}
|
||||
className="w-full"
|
||||
>
|
||||
<Field
|
||||
name="crawlurl"
|
||||
ref={urlInputRef}
|
||||
type="text"
|
||||
placeholder={t("webSite", { ns: "upload" })}
|
||||
className="w-full"
|
||||
value={urlToCrawl}
|
||||
onChange={(e) => setUrlToCrawl(e.target.value)}
|
||||
icon={
|
||||
<Button variant={"tertiary"}>
|
||||
<MdSend />
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
@ -0,0 +1,40 @@
|
||||
import { Fragment } from "react";
|
||||
import { IoMdCloseCircle } from "react-icons/io";
|
||||
import { MdLink } from "react-icons/md";
|
||||
|
||||
import { UrlDisplay } from "./components";
|
||||
import { FeedItemType } from "../../types";
|
||||
|
||||
type FeedItemsProps = {
|
||||
contents: FeedItemType[];
|
||||
removeContent: (index: number) => void;
|
||||
};
|
||||
|
||||
export const FeedItems = ({
|
||||
contents,
|
||||
removeContent,
|
||||
}: FeedItemsProps): JSX.Element => {
|
||||
if (contents.length === 0) {
|
||||
return <Fragment />;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-2 gap-4 mt-5 shadow-md shadow-md dark:shadow-primary/25 hover:shadow-xl transition-shadow rounded-xl bg-white dark:bg-black border border-black/10 dark:border-white/25 p-6">
|
||||
{contents.map((item, index) => (
|
||||
<div
|
||||
key={item.url}
|
||||
className="relative bg-gray-100 p-4 rounded-lg shadow-sm"
|
||||
>
|
||||
<IoMdCloseCircle
|
||||
className="absolute top-2 right-2 cursor-pointer text-gray-400 text-2xl"
|
||||
onClick={() => removeContent(index)}
|
||||
/>
|
||||
<div className="flex items-center">
|
||||
<MdLink className="mr-2 text-2xl" />
|
||||
<UrlDisplay url={item.url} />
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
@ -0,0 +1,23 @@
|
||||
import { useState } from "react";
|
||||
|
||||
import { enhanceUrlDisplay } from "./utils/enhanceUrlDisplay";
|
||||
|
||||
type UrlDisplayProps = {
|
||||
url: string;
|
||||
};
|
||||
|
||||
export const UrlDisplay = ({ url }: UrlDisplayProps): JSX.Element => {
|
||||
const [showFullUrl, setShowFullUrl] = useState(false);
|
||||
|
||||
const toggleShowFullUrl = () => {
|
||||
setShowFullUrl(!showFullUrl);
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<span className="cursor-pointer" onClick={toggleShowFullUrl}>
|
||||
{showFullUrl ? url : enhanceUrlDisplay(url)}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
};
|
@ -0,0 +1 @@
|
||||
export * from "./UrlDisplay";
|
@ -0,0 +1,24 @@
|
||||
export const enhanceUrlDisplay = (url: string): string => {
|
||||
const parts = url.split("/");
|
||||
|
||||
// Check if the URL has at least 3 parts (protocol, domain, and one more segment)
|
||||
if (parts.length >= 3) {
|
||||
const domain = parts[2];
|
||||
const path = parts.slice(3).join("/");
|
||||
|
||||
// Split the domain by "." to check for subdomains and remove "www"
|
||||
const domainParts = domain.split(".");
|
||||
if (domainParts[0] === "www") {
|
||||
domainParts.shift(); // Remove "www"
|
||||
}
|
||||
|
||||
// Combine the beginning (subdomain/domain) and the end (trimmed path)
|
||||
const beginning = domainParts.join(".");
|
||||
const trimmedPath = path.slice(0, 5) + "..." + path.slice(-5); // Display the beginning and end of the path
|
||||
|
||||
return `${beginning}/${trimmedPath}`;
|
||||
}
|
||||
|
||||
// If the URL doesn't have enough parts, return it as is
|
||||
return url;
|
||||
};
|
@ -0,0 +1 @@
|
||||
export * from "./UrlDisplay";
|
@ -0,0 +1 @@
|
||||
export * from "./FeedItems";
|
@ -1,2 +1,3 @@
|
||||
export * from "./Crawler";
|
||||
export * from "./FeedItems";
|
||||
export * from "./FileUploader";
|
@ -0,0 +1,22 @@
|
||||
import { useState } from "react";
|
||||
|
||||
import { FeedItemType } from "../types";
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
||||
export const useKnowledgeToFeed = () => {
|
||||
const [contents, setContents] = useState<FeedItemType[]>([]);
|
||||
|
||||
const addContent = (content: FeedItemType) => {
|
||||
setContents((prevContents) => [...prevContents, content]);
|
||||
};
|
||||
const removeContent = (index: number) => {
|
||||
setContents((prevContents) => prevContents.filter((_, i) => i !== index));
|
||||
};
|
||||
|
||||
return {
|
||||
addContent,
|
||||
contents,
|
||||
setContents,
|
||||
removeContent,
|
||||
};
|
||||
};
|
@ -0,0 +1 @@
|
||||
export * from "./KnowledgeToFeed";
|
@ -0,0 +1,6 @@
|
||||
export type FeedItemSource = "crawl" | "upload";
|
||||
|
||||
export type FeedItemType = {
|
||||
source: FeedItemSource;
|
||||
url: string;
|
||||
};
|
@ -1,2 +1,2 @@
|
||||
export * from "./ChatInput";
|
||||
export * from "./Feed";
|
||||
export * from "./KnowledgeToFeed";
|
||||
|
@ -3,14 +3,13 @@ import { UUID } from "crypto";
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import { isValidUrl } from "@/app/chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components/Crawler/helpers/isValidUrl";
|
||||
import { useCrawlApi } from "@/lib/api/crawl/useCrawlApi";
|
||||
import { useSupabase } from "@/lib/context/SupabaseProvider";
|
||||
import { useToast } from "@/lib/hooks";
|
||||
import { redirectToLogin } from "@/lib/router/redirectToLogin";
|
||||
import { useEventTracking } from "@/services/analytics/useEventTracking";
|
||||
|
||||
import { isValidUrl } from "../helpers/isValidUrl";
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
||||
export const useCrawler = () => {
|
||||
const [isCrawling, setCrawling] = useState(false);
|
@ -9,11 +9,9 @@ import { useBrainContext } from "@/lib/context/BrainProvider/hooks/useBrainConte
|
||||
import { useSupabase } from "@/lib/context/SupabaseProvider";
|
||||
import { redirectToLogin } from "@/lib/router/redirectToLogin";
|
||||
|
||||
import { Crawler } from "./Crawler";
|
||||
import { requiredRolesForUpload } from "./config";
|
||||
import {
|
||||
Crawler,
|
||||
FileUploader,
|
||||
} from "../chat/[chatId]/components/ActionsBar/components/Feed/components";
|
||||
import { FileUploader } from "../chat/[chatId]/components/ActionsBar/components/KnowledgeToFeed/components";
|
||||
|
||||
const UploadPage = (): JSX.Element => {
|
||||
const { currentBrain } = useBrainContext();
|
||||
|
@ -1,4 +1,3 @@
|
||||
/* eslint-disable */
|
||||
import {
|
||||
DetailedHTMLProps,
|
||||
forwardRef,
|
||||
@ -15,28 +14,36 @@ interface FieldProps
|
||||
> {
|
||||
label?: string;
|
||||
name: string;
|
||||
icon?: React.ReactNode;
|
||||
}
|
||||
|
||||
const Field = forwardRef(
|
||||
(
|
||||
{ label, className, name, required = false, ...props }: FieldProps,
|
||||
{ label, className, name, required = false, icon, ...props }: FieldProps,
|
||||
forwardedRef
|
||||
) => {
|
||||
return (
|
||||
<fieldset className={cn("flex flex-col w-full", className)} name={name}>
|
||||
{label && (
|
||||
{label !== undefined && (
|
||||
<label htmlFor={name} className="text-sm">
|
||||
{label}
|
||||
{required && <span>*</span>}
|
||||
</label>
|
||||
)}
|
||||
<input
|
||||
ref={forwardedRef as RefObject<HTMLInputElement>}
|
||||
className="w-full bg-gray-50 dark:bg-gray-900 px-4 py-2 border rounded-md border-black/10 dark:border-white/25"
|
||||
name={name}
|
||||
id={name}
|
||||
{...props}
|
||||
/>
|
||||
<div className="relative">
|
||||
<input
|
||||
ref={forwardedRef as RefObject<HTMLInputElement>}
|
||||
className={`w-full bg-gray-50 dark:bg-gray-900 px-4 py-2 border rounded-md border-black/10 dark:border-white/25`}
|
||||
name={name}
|
||||
id={name}
|
||||
{...props}
|
||||
/>
|
||||
{icon !== undefined && (
|
||||
<div className="absolute inset-y-0 right-0 pr-3 flex items-center">
|
||||
{icon}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</fieldset>
|
||||
);
|
||||
}
|
||||
|
@ -27,7 +27,6 @@
|
||||
"updateButton": "Update",
|
||||
"uploadButton": "Upload",
|
||||
"uploadingButton": "Uploading...",
|
||||
"crawlButton": "Crawl",
|
||||
"chatButton": "Chat",
|
||||
"deleteButton": "Delete",
|
||||
"deleteForeverButton": "Delete forever",
|
||||
@ -44,5 +43,7 @@
|
||||
"Viewer": "Viewer",
|
||||
"Editor": "Editor",
|
||||
"Owner": "Owner",
|
||||
"saveButton": "Save"
|
||||
"saveButton": "Save",
|
||||
"invalidUrl": "Invalid URL",
|
||||
"crawlButton": "Crawl"
|
||||
}
|
||||
|
@ -4,7 +4,6 @@
|
||||
"Chat": "Conversar",
|
||||
"chatButton": "Conversar",
|
||||
"comingSoon": "Próximamente",
|
||||
"crawlButton": "Rastrear",
|
||||
"createButton": "Crear",
|
||||
"deleteButton": "Borrar",
|
||||
"deleteForeverButton": "Borrar para siempre",
|
||||
@ -44,5 +43,7 @@
|
||||
"uploadButton": "Subir",
|
||||
"uploadingButton": "Subiendo...",
|
||||
"Viewer": "Espectador",
|
||||
"saveButton": "Guardar"
|
||||
"saveButton": "Guardar",
|
||||
"invalidUrl": "URL inválida",
|
||||
"crawlButton": "Rastrear"
|
||||
}
|
||||
|
@ -27,7 +27,6 @@
|
||||
"updateButton": "Mettre à jour",
|
||||
"uploadButton": "Télécharger",
|
||||
"uploadingButton": "Téléchargement...",
|
||||
"crawlButton": "Explorer",
|
||||
"chatButton": "Chat",
|
||||
"deleteButton": "Supprimer",
|
||||
"deleteForeverButton": "Supprimer définitivement",
|
||||
@ -44,5 +43,7 @@
|
||||
"Viewer": "Visualiseur",
|
||||
"Editor": "Éditeur",
|
||||
"Owner": "Propriétaire",
|
||||
"saveButton": "Sauvegarder"
|
||||
"saveButton": "Sauvegarder",
|
||||
"invalidUrl": "URL invalide",
|
||||
"crawlButton": "Crawler"
|
||||
}
|
||||
|
@ -27,7 +27,6 @@
|
||||
"updateButton": "Atualizar",
|
||||
"uploadButton": "Enviar",
|
||||
"uploadingButton": "Enviando...",
|
||||
"crawlButton": "Rastrear",
|
||||
"chatButton": "Chat",
|
||||
"deleteButton": "Excluir",
|
||||
"deleteForeverButton": "Excluir permanentemente",
|
||||
@ -44,5 +43,7 @@
|
||||
"Viewer": "Visualizador",
|
||||
"Editor": "Editor",
|
||||
"Owner": "Proprietário",
|
||||
"saveButton": "Salvar"
|
||||
"saveButton": "Salvar",
|
||||
"invalidUrl": "URL inválida",
|
||||
"crawlButton": "Rastrear"
|
||||
}
|
||||
|
@ -27,7 +27,6 @@
|
||||
"updateButton": "Обновить",
|
||||
"uploadButton": "Загрузить",
|
||||
"uploadingButton": "Загрузка...",
|
||||
"crawlButton": "Извлечь информацию",
|
||||
"chatButton": "Чат",
|
||||
"deleteButton": "Удалить",
|
||||
"deleteForeverButton": "Удалить навсегда",
|
||||
@ -44,5 +43,7 @@
|
||||
"Viewer": "Просмотр",
|
||||
"Editor": "Редактор",
|
||||
"Owner": "Владелец",
|
||||
"saveButton": "Сохранить"
|
||||
"saveButton": "Сохранить",
|
||||
"invalidUrl": "Неверный URL",
|
||||
"crawlButton": "Поиск"
|
||||
}
|
||||
|
@ -27,7 +27,6 @@
|
||||
"updateButton": "更新",
|
||||
"uploadButton": "上传",
|
||||
"uploadingButton": "上传中...",
|
||||
"crawlButton": "抓取",
|
||||
"chatButton": "聊天",
|
||||
"deleteButton": "删除",
|
||||
"deleteForeverButton": "永久删除",
|
||||
@ -44,5 +43,7 @@
|
||||
"Viewer": "查看",
|
||||
"Editor": "编辑",
|
||||
"Owner": "作者",
|
||||
"saveButton": "保存"
|
||||
"saveButton": "保存",
|
||||
"invalidUrl": "无效的URL",
|
||||
"crawlButton": "爬取"
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user