Merge pull request #46 from StanGirard/new-app

NextJS & Fast API migration first iteration

frontend/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

frontend/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

frontend/README.md

@@ -0,0 +1,34 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

frontend/app/(home)/Hero.tsx

@@ -0,0 +1,39 @@
+import Link from "next/link";
+import { FC } from "react";
+import Button from "../components/ui/Button";
+import { MdNorthEast } from "react-icons/md";
+
+interface HeroProps {}
+
+const Hero: FC<HeroProps> = ({}) => {
+  return (
+    <section className="relative w-full flex flex-col gap-24 items-center text-center min-h-[768px] py-12">
+      <div className="flex flex-col gap-2 items-center justify-center mt-12">
+        <h1 className="text-7xl font-bold max-w-xl">
+          Get a Second Brain with <span className="text-primary">Quivr</span>
+        </h1>
+        <p className="text-base max-w-sm text-gray-500 mb-10">
+          Quivr is your second brain in the cloud, designed to easily store and
+          retrieve unstructured information.
+        </p>
+        <Link href={"/upload"}>
+          <Button>Try Demo</Button>
+        </Link>
+        <Link target="_blank" href={"https://github.com/StanGirard/quivr/"}>
+          <Button variant={"tertiary"}>
+            Github <MdNorthEast />
+          </Button>
+        </Link>
+      </div>
+      <video
+        className="rounded-md max-w-screen-lg shadow-lg dark:shadow-white/25 border dark:border-white/25 w-full"
+        src="https://user-images.githubusercontent.com/19614572/238774100-80721777-2313-468f-b75e-09379f694653.mp4"
+        autoPlay
+        muted
+        loop
+      />
+    </section>
+  );
+};
+
+export default Hero;

frontend/app/(home)/page.tsx

@@ -0,0 +1,12 @@
+import Link from "next/link";
+import Hero from "./Hero";
+import { Analytics } from '@vercel/analytics/react';
+
+export default function HomePage() {
+  return (
+    <main className="">
+      <Hero />
+      <Analytics />
+    </main>
+  );
+}

frontend/app/chat/page.tsx

@@ -0,0 +1,41 @@
+'use client';
+import { useState } from 'react';
+import axios from 'axios';
+
+export default function ChatPage() {
+  const [question, setQuestion] = useState('');
+  const [history, setHistory] = useState([['User', 'Hello!']]);
+
+  const askQuestion = async () => {
+    const response = await axios.post('http://localhost:8000/chat/', {
+      model: 'gpt-3.5-turbo',
+      question,
+      history
+    });
+    setHistory(response.data.history);
+    setQuestion('');
+  };
+
+  return (
+    <div className="flex flex-col items-center justify-center min-h-screen bg-gray-200 py-2">
+      <div className="px-4 py-5 bg-white shadow rounded-lg w-full md:w-1/2">
+        {history.map(([speaker, text], idx) => (
+          <p key={idx} className="mb-4 text-black">
+            <b className="mr-2 text-black">{speaker}:</b> {text}
+          </p>
+        ))}
+      </div>
+      <div className="mt-6">
+        <input
+          type="text"
+          value={question}
+          onChange={(e) => setQuestion(e.target.value)}
+          className="px-3 py-2 border border-gray-300 shadow-sm rounded-md focus:outline-none text-black focus:ring-2 focus:ring-indigo-600"
+        />
+        <button onClick={askQuestion} className="ml-2 px-4 py-2 bg-indigo-600 text-white rounded-md shadow-sm hover:bg-indigo-500 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
+          Ask
+        </button>
+      </div>
+    </div>
+  );
+}

frontend/app/components/NavBar/DarkModeToggle.tsx

@@ -0,0 +1,41 @@
+"use client";
+import { FC, useEffect, useLayoutEffect, useState } from "react";
+import Button from "../ui/Button";
+import { MdDarkMode, MdLightMode } from "react-icons/md";
+
+interface DarkModeToggleProps {}
+
+const DarkModeToggle: FC<DarkModeToggleProps> = ({}) => {
+  const [dark, setDark] = useState(false);
+
+  useLayoutEffect(() => {
+    const isDark = localStorage.getItem("dark");
+    if (isDark && isDark === "true") {
+      document.body.parentElement?.classList.add("dark");
+      setDark(true);
+    }
+  }, []);
+
+  useEffect(() => {
+    if (dark) {
+      document.body.parentElement?.classList.add("dark");
+      localStorage.setItem("dark", "true");
+    } else {
+      document.body.parentElement?.classList.remove("dark");
+      localStorage.setItem("dark", "false");
+    }
+  }, [dark]);
+
+  return (
+    <Button
+      aria-label="toggle dark mode"
+      className="focus:outline-none"
+      onClick={() => setDark((d) => !d)}
+      variant={"tertiary"}
+    >
+      {dark ? <MdLightMode /> : <MdDarkMode />}
+    </Button>
+  );
+};
+
+export default DarkModeToggle;

frontend/app/components/NavBar/index.tsx

@@ -0,0 +1,44 @@
+import Image from "next/image";
+import { FC } from "react";
+import logo from "../../logo.png";
+import Link from "next/link";
+import Button from "../ui/Button";
+import DarkModeToggle from "./DarkModeToggle";
+
+interface NavBarProps {}
+
+const NavBar: FC<NavBarProps> = ({}) => {
+  return (
+    <header className="sticky top-0 border-b border-b-black/10 dark:border-b-white/25 bg-white/50 dark:bg-black/50 bg-opacity-0 backdrop-blur-md z-50">
+      <nav className="max-w-screen-xl mx-auto py-3 flex items-center gap-8">
+        <Link href={"/"} className="flex items-center gap-4">
+          <Image
+            className="rounded-full"
+            src={logo}
+            alt="Quivr Logo"
+            width={48}
+            height={48}
+          />
+          <h1 className="font-bold">Quivr</h1>
+        </Link>
+        <ul className="flex gap-4 text-sm flex-1">
+          <li>
+            <Link href={"https://github.com/StanGirard/quivr"}>Github</Link>
+          </li>
+          <li>
+            <Link href={"https://discord.gg/HUpRgp2HG8"}>Join the discord</Link>
+          </li>
+          
+        </ul>
+        <div className="flex">
+          <Link href={"https://try-quivr.streamlit.app"}>
+            <Button variant={"secondary"}>Try Demo</Button>
+          </Link>
+          <DarkModeToggle />
+        </div>
+      </nav>
+    </header>
+  );
+};
+
+export default NavBar;

frontend/app/components/ui/Button.tsx

@@ -0,0 +1,50 @@
+import { ButtonHTMLAttributes, FC } from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+import { cn } from "@/lib/utils";
+
+const ButtonVariants = cva(
+  "px-8 py-3 text-sm font-medium rounded-md focus:ring ring-primary/10 outline-none flex items-center gap-2 disabled:opacity-50 transition-opacity",
+  {
+    variants: {
+      variant: {
+        primary:
+          "bg-black text-white dark:bg-white dark:text-black hover:bg-gray-700 dark:hover:bg-gray-200 transition-colors",
+        tertiary: "text-black dark:text-white bg-transparent py-2 px-4",
+        secondary:
+          "border border-black dark:border-white bg-white dark:bg-black text-black dark:text-white focus:bg-black dark:focus:bg-white hover:bg-black dark:hover:bg-white hover:text-white dark:hover:text-black focus:text-white transition-colors py-2 px-4 shadow-none",
+      },
+      brightness: {
+        dim: "",
+        default: "",
+      },
+    },
+    defaultVariants: {
+      variant: "primary",
+      brightness: "default",
+    },
+  }
+);
+export interface ButtonProps
+  extends ButtonHTMLAttributes<HTMLButtonElement>,
+    VariantProps<typeof ButtonVariants> {
+  isLoading?: boolean;
+}
+
+const Button: FC<ButtonProps> = ({
+  className,
+  children,
+  variant,
+  brightness,
+  ...props
+}) => {
+  return (
+    <button
+      className={cn(ButtonVariants({ variant, brightness, className }))}
+      {...props}
+    >
+      {children}
+    </button>
+  );
+};
+
+export default Button;

frontend/app/globals.css

@@ -0,0 +1,12 @@
+@import 'tailwindcss/base';
+@import 'tailwindcss/components';
+@import 'tailwindcss/utilities';
+
+
+main {
+  @apply max-w-screen-xl mx-auto flex flex-col min-h-screen;
+}
+
+header, section {
+  @apply px-5 md:px-10;
+}

frontend/app/layout.tsx

@@ -0,0 +1,28 @@
+import NavBar from "./components/NavBar";
+import "./globals.css";
+import { Inter } from "next/font/google";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata = {
+  title: "Quivr - Get a Second Brain with Generative AI",
+  description:
+    "Quivr is your second brain in the cloud, designed to easily store and retrieve unstructured information.",
+};
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <html lang="en">
+      <body
+        className={`bg-white text-black dark:bg-black dark:text-white min-h-screen w-full ${inter.className}`}
+      >
+        <NavBar />
+        {children}
+      </body>
+    </html>
+  );
+}

frontend/app/upload/page.tsx

@@ -0,0 +1,65 @@
+"use client";
+import { useCallback, useState } from "react";
+import { useDropzone } from "react-dropzone";
+import axios from "axios";
+import { Message } from "@/lib/types";
+
+export default function UploadPage() {
+    const [message, setMessage] = useState<Message | null>(null);
+
+    const onDrop = useCallback(async (acceptedFiles: File[]) => {
+        const file = acceptedFiles[0];
+        const formData = new FormData();
+        formData.append("file", file);
+        try {
+            const response = await axios.post(
+                "http://localhost:8000/upload",
+                formData
+            );
+            setMessage({
+                type: "success",
+                text:
+                    "File uploaded successfully: " +
+                    JSON.stringify(response.data),
+            });
+        } catch (error: any) {
+            setMessage({
+                type: "error",
+                text: "Failed to upload file: " + error.toString(),
+            });
+        }
+    }, []);
+
+    const { getRootProps, getInputProps, isDragActive } = useDropzone({
+        onDrop,
+    });
+
+    return (
+        <div
+            {...getRootProps()}
+            className="flex flex-col items-center justify-center h-screen bg-gray-100"
+        >
+            <input {...getInputProps()} />
+            <div className="mt-2 p-6 max-w-sm mx-auto bg-white rounded-xl shadow-md flex items-center space-x-4">
+                {isDragActive ? (
+                    <p className="text-blue-600">Drop the files here...</p>
+                ) : (
+                    <p className="text-gray-500">
+                        Drag and drop some files here, or click to select files
+                    </p>
+                )}
+            </div>
+            {message && (
+                <div
+                    className={`mt-4 p-2 rounded ${
+                        message.type === "success"
+                            ? "bg-green-500"
+                            : "bg-red-500"
+                    }`}
+                >
+                    <p className="text-white">{message.text}</p>
+                </div>
+            )}
+        </div>
+    );
+}

frontend/lib/types.ts

@@ -0,0 +1,4 @@
+export type Message = {
+    type: "success" | "error" | "warning";
+    text: string;
+};

frontend/lib/utils.ts

@@ -0,0 +1,6 @@
+import clsx, { ClassValue } from "clsx";
+import { twMerge } from "tailwind-merge";
+
+export function cn(...inputs: ClassValue[]) {
+    return twMerge(clsx(inputs));
+}

frontend/next.config.js

@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {}
+
+module.exports = nextConfig

frontend/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "frontend",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@types/node": "20.1.7",
+    "@types/react": "18.2.6",
+    "@types/react-dom": "18.2.4",
+    "@vercel/analytics": "^1.0.1",
+    "autoprefixer": "10.4.14",
+    "axios": "^1.4.0",
+    "class-variance-authority": "^0.6.0",
+    "clsx": "^1.2.1",
+    "eslint": "8.40.0",
+    "eslint-config-next": "13.4.2",
+    "next": "13.4.2",
+    "postcss": "8.4.23",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-dropzone": "^14.2.3",
+    "tailwind-merge": "^1.12.0",
+    "tailwindcss": "3.3.2",
+    "typescript": "5.0.4"
+  },
+  "devDependencies": {
+    "react-icons": "^4.8.0"
+  }
+}

frontend/pages/about.tsx

@@ -0,0 +1,3 @@
+export default function About() {
+    return <div>About</div>;
+  }

frontend/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

frontend/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

frontend/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

frontend/tailwind.config.js

@@ -0,0 +1,23 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  darkMode: "class",
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
+        'gradient-conic':
+          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+      },
+      colors: {
+        black: "#00121F",
+        primary: "#4F46E5",
+      }
+    },
+  },
+  plugins: [],
+}

frontend/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

requirements.txt

@@ -9,3 +9,6 @@ supabase==1.0.3
 tiktoken==0.4.0
 unstructured==0.6.5
 anthropic==0.2.8
+fastapi==0.95.2
+python-multipart==0.0.6
+uvicorn==0.22.0

v2/api.py

@@ -0,0 +1,107 @@
+from fastapi import FastAPI, UploadFile, File, HTTPException
+import os
+from pydantic import BaseModel
+from typing import List, Tuple
+from supabase import create_client, Client
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.memory import ConversationBufferMemory
+from langchain.vectorstores import SupabaseVectorStore
+from langchain.chains import ConversationalRetrievalChain
+from langchain.llms import OpenAI
+
+
+from common import file_already_exists
+from txt import process_txt
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI()
+
+origins = [
+    "http://localhost.tiangolo.com",
+    "https://localhost.tiangolo.com",
+    "http://localhost",
+    "http://localhost:3000",
+    "http://localhost:8080",
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+supabase_url = os.environ.get("SUPABASE_URL")
+supabase_key = os.environ.get("SUPABASE_SERVICE_KEY")
+openai_api_key = os.environ.get("OPENAI_API_KEY")
+anthropic_api_key = ""
+supabase: Client = create_client(supabase_url, supabase_key)
+embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
+vector_store = SupabaseVectorStore(
+    supabase, embeddings, table_name="documents")
+memory = ConversationBufferMemory(
+    memory_key="chat_history", return_messages=True)
+
+
+class ChatMessage(BaseModel):
+    model: str
+    question: str
+    history: List[Tuple[str, str]]  # A list of tuples where each tuple is (speaker, text)
+
+file_processors = {
+    ".txt": process_txt,
+}
+
+async def filter_file(file: UploadFile, supabase, vector_store):
+    if await file_already_exists(supabase, file):
+        return f"😎 {file.filename} is already in the database."
+    elif file.file._file.tell() < 1:
+        return f"💨 {file.filename} is empty."
+    else:
+        file_extension = os.path.splitext(file.filename)[-1]
+        if file_extension in file_processors:
+            await file_processors[file_extension](vector_store, file, stats_db=None)
+            return f"✅ {file.filename} "
+        else:
+            return f"❌ {file.filename} is not a valid file type."
+
+@app.post("/upload")
+async def upload_file(file: UploadFile):
+    # Modify your code to work with FastAPI
+    # Here we assume that you have some way to get `supabase`, `openai_key`, and `vector_store`
+   
+    print(f"Received file: {file.filename}")
+   
+    message = await filter_file(file, supabase, vector_store)
+    
+    return {"message": message}
+
+@app.post("/chat/")
+async def chat_endpoint(chat_message: ChatMessage):
+    model = chat_message.model
+    question = chat_message.question
+    history = chat_message.history
+    temperature = 0
+    max_tokens = 100
+
+    # Logic from your Streamlit app goes here. For example:
+    qa = None
+    if model.startswith("gpt"):
+        qa = ConversationalRetrievalChain.from_llm(
+            OpenAI(
+                model_name=model, openai_api_key=openai_api_key, temperature=temperature, max_tokens=max_tokens), vector_store.as_retriever(), memory=memory, verbose=True)
+    elif anthropic_api_key and model.startswith("claude"):
+        qa = ConversationalRetrievalChain.from_llm(
+            ChatAnthropic(
+                model=model, anthropic_api_key=anthropic_api_key, temperature=temperature, max_tokens_to_sample=max_tokens), vector_store.as_retriever(), memory=memory, verbose=True, max_tokens_limit=102400)
+
+    history.append(("user", question))
+    model_response = qa({"question": question})
+    history.append(("assistant", model_response["answer"]))
+
+    return {"history": history}
+
+@app.get("/")
+async def root():
+    return {"message": "Hello World"}

v2/common.py

@@ -0,0 +1,52 @@
+from typing import Optional
+from fastapi import UploadFile
+from langchain.schema import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+# from stats import add_usage
+import asyncio
+import os
+import tempfile
+import time
+from utils import compute_sha1_from_file, compute_sha1_from_content
+
+async def process_file(vector_store, file: UploadFile, loader_class, stats_db: Optional = None):
+    documents = []
+    file_sha = ""
+    file_name = file.filename
+    file_size = file.file._file.tell()  # Getting the size of the file
+    dateshort = time.strftime("%Y%m%d")
+    
+    # Here, we're writing the uploaded file to a temporary file, so we can use it with your existing code.
+    with tempfile.NamedTemporaryFile(delete=False, suffix=file.filename) as tmp_file:
+        await file.seek(0)
+        content = await file.read()
+        tmp_file.write(content)
+        tmp_file.flush()
+        
+        loader = loader_class(tmp_file.name)
+        documents = loader.load()
+        file_sha1 = compute_sha1_from_file(tmp_file.name)  # Ensure this function works with FastAPI
+
+    os.remove(tmp_file.name)
+    chunk_size = 500
+    chunk_overlap = 0
+    
+    text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    
+    print(documents)
+    documents = text_splitter.split_documents(documents)
+
+
+    # Add the document sha1 as metadata to each document
+    docs_with_metadata = [Document(page_content=doc.page_content, metadata={"file_sha1": file_sha1, "file_size":file_size , "file_name": file_name, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort}) for doc in documents]
+    
+    vector_store.add_documents(docs_with_metadata)
+    # if stats_db:
+    #     add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,"file_type": file.filename, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
+    return
+
+async def file_already_exists(supabase, file):
+    file_content = await file.read()
+    file_sha1 = compute_sha1_from_content(file_content)
+    response = supabase.table("documents").select("id").eq("metadata->>file_sha1", file_sha1).execute()
+    return len(response.data) > 0

v2/txt.py

@@ -0,0 +1,6 @@
+from common import process_file
+from langchain.document_loaders import TextLoader
+from fastapi import UploadFile
+
+async def process_txt(vector_store, file: UploadFile, stats_db):
+    return await process_file(vector_store, file, TextLoader, stats_db=stats_db)

v2/utils.py

@@ -0,0 +1,11 @@
+import hashlib
+
+def compute_sha1_from_file(file_path):
+    with open(file_path, "rb") as file:
+        bytes = file.read() 
+        readable_hash = hashlib.sha1(bytes).hexdigest()
+    return readable_hash
+
+def compute_sha1_from_content(content):
+    readable_hash = hashlib.sha1(content).hexdigest()
+    return readable_hash

website/package.json

@@ -10,10 +10,12 @@
   },
   "dependencies": {
     "@vercel/analytics": "^1.0.1",
+    "axios": "^1.4.0",
     "eslint": "8.39.0",
     "eslint-config-next": "13.4.1",
     "next": "13.4.1",
     "react": "18.2.0",
-    "react-dom": "18.2.0"
+    "react-dom": "18.2.0",
+    "react-dropzone": "^14.2.3"
   }
 }

website/yarn.lock

@@ -320,6 +320,16 @@ ast-types-flow@^0.0.7:
   resolved "https://registry.yarnpkg.com/ast-types-flow/-/ast-types-flow-0.0.7.tgz#f70b735c6bca1a5c9c22d982c3e39e7feba3bdad"
   integrity sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==
 
+asynckit@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
+  integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==
+
+attr-accept@^2.2.2:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/attr-accept/-/attr-accept-2.2.2.tgz#646613809660110749e92f2c10833b70968d929b"
+  integrity sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg==
+
 available-typed-arrays@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7"
@@ -330,6 +340,15 @@ axe-core@^4.6.2:
   resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.7.0.tgz#34ba5a48a8b564f67e103f0aa5768d76e15bbbbf"
   integrity sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==
 
+axios@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.4.0.tgz#38a7bf1224cd308de271146038b551d725f0be1f"
+  integrity sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
 axobject-query@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-3.1.1.tgz#3b6e5c6d4e43ca7ba51c5babf99d22a9c68485e1"
@@ -426,6 +445,13 @@ color-name@~1.1.4:
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
   integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
 
+combined-stream@^1.0.8:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
+  integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==
+  dependencies:
+    delayed-stream "~1.0.0"
+
 concat-map@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -519,6 +545,11 @@ define-properties@^1.1.3, define-properties@^1.1.4, define-properties@^1.2.0:
     has-property-descriptors "^1.0.0"
     object-keys "^1.1.1"
 
+delayed-stream@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
+  integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==
+
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
@@ -914,6 +945,13 @@ file-entry-cache@^6.0.1:
   dependencies:
     flat-cache "^3.0.4"
 
+file-selector@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/file-selector/-/file-selector-0.6.0.tgz#fa0a8d9007b829504db4d07dd4de0310b65287dc"
+  integrity sha512-QlZ5yJC0VxHxQQsQhXvBaC7VRJ2uaxTf+Tfpu4Z/OcVQJVpZO+DGU0rkoVW5ce2SccxugvpBJoMvUs59iILYdw==
+  dependencies:
+    tslib "^2.4.0"
+
 fill-range@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -942,6 +980,11 @@ flatted@^3.1.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.7.tgz#609f39207cb614b89d0765b477cb2d437fbf9787"
   integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==
 
+follow-redirects@^1.15.0:
+  version "1.15.2"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
+  integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
+
 for-each@^0.3.3:
   version "0.3.3"
   resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e"
@@ -949,6 +992,15 @@ for-each@^0.3.3:
   dependencies:
     is-callable "^1.1.3"
 
+form-data@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
+  integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -1490,6 +1542,18 @@ micromatch@^4.0.4:
     braces "^3.0.2"
     picomatch "^2.3.1"
 
+mime-db@1.52.0:
+  version "1.52.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
+  integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
+
+mime-types@^2.1.12:
+  version "2.1.35"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
+  integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
+  dependencies:
+    mime-db "1.52.0"
+
 mimic-fn@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
@@ -1764,6 +1828,11 @@ prop-types@^15.8.1:
     object-assign "^4.1.1"
     react-is "^16.13.1"
 
+proxy-from-env@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
+  integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
+
 punycode@^2.1.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f"
@@ -1782,6 +1851,15 @@ react-dom@18.2.0:
     loose-envify "^1.1.0"
     scheduler "^0.23.0"
 
+react-dropzone@^14.2.3:
+  version "14.2.3"
+  resolved "https://registry.yarnpkg.com/react-dropzone/-/react-dropzone-14.2.3.tgz#0acab68308fda2d54d1273a1e626264e13d4e84b"
+  integrity sha512-O3om8I+PkFKbxCukfIR3QAGftYXDZfOE2N1mr/7qebQJHs7U+/RSL/9xomJNpRg9kM5h9soQSdf0Gc7OHF5Fug==
+  dependencies:
+    attr-accept "^2.2.2"
+    file-selector "^0.6.0"
+    prop-types "^15.8.1"
+
 react-is@^16.13.1:
   version "16.13.1"
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4"