mirror of
https://github.com/QuivrHQ/quivr.git
synced 2024-08-17 02:10:31 +03:00
feat(api): new api init
This commit is contained in:
parent
a0ea793ff3
commit
58630f7207
3
frontend/.eslintrc.json
Normal file
3
frontend/.eslintrc.json
Normal file
@ -0,0 +1,3 @@
|
||||
{
|
||||
"extends": "next/core-web-vitals"
|
||||
}
|
35
frontend/.gitignore
vendored
Normal file
35
frontend/.gitignore
vendored
Normal file
@ -0,0 +1,35 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.js
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# local env files
|
||||
.env*.local
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
34
frontend/README.md
Normal file
34
frontend/README.md
Normal file
@ -0,0 +1,34 @@
|
||||
This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
|
||||
|
||||
## Getting Started
|
||||
|
||||
First, run the development server:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
# or
|
||||
yarn dev
|
||||
# or
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
|
||||
|
||||
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
|
||||
|
||||
This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
|
||||
|
||||
## Learn More
|
||||
|
||||
To learn more about Next.js, take a look at the following resources:
|
||||
|
||||
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
|
||||
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
|
||||
|
||||
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
|
||||
|
||||
## Deploy on Vercel
|
||||
|
||||
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
|
||||
|
||||
Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.
|
41
frontend/app/chat/page.tsx
Normal file
41
frontend/app/chat/page.tsx
Normal file
@ -0,0 +1,41 @@
|
||||
'use client';
|
||||
import { useState } from 'react';
|
||||
import axios from 'axios';
|
||||
|
||||
export default function ChatPage() {
|
||||
const [question, setQuestion] = useState('');
|
||||
const [history, setHistory] = useState([['User', 'Hello!']]);
|
||||
|
||||
const askQuestion = async () => {
|
||||
const response = await axios.post('http://localhost:8000/chat/', {
|
||||
model: 'gpt-3.5-turbo',
|
||||
question,
|
||||
history
|
||||
});
|
||||
setHistory(response.data.history);
|
||||
setQuestion('');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col items-center justify-center min-h-screen bg-gray-200 py-2">
|
||||
<div className="px-4 py-5 bg-white shadow rounded-lg w-full md:w-1/2">
|
||||
{history.map(([speaker, text], idx) => (
|
||||
<p key={idx} className="mb-4 text-black">
|
||||
<b className="mr-2 text-black">{speaker}:</b> {text}
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
<div className="mt-6">
|
||||
<input
|
||||
type="text"
|
||||
value={question}
|
||||
onChange={(e) => setQuestion(e.target.value)}
|
||||
className="px-3 py-2 border border-gray-300 shadow-sm rounded-md focus:outline-none text-black focus:ring-2 focus:ring-indigo-600"
|
||||
/>
|
||||
<button onClick={askQuestion} className="ml-2 px-4 py-2 bg-indigo-600 text-white rounded-md shadow-sm hover:bg-indigo-500 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
|
||||
Ask
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
BIN
frontend/app/favicon.ico
Normal file
BIN
frontend/app/favicon.ico
Normal file
Binary file not shown.
After Width: | Height: | Size: 25 KiB |
33
frontend/app/globals.css
Normal file
33
frontend/app/globals.css
Normal file
@ -0,0 +1,33 @@
|
||||
@import 'tailwindcss/base';
|
||||
@import 'tailwindcss/components';
|
||||
@import 'tailwindcss/utilities';
|
||||
|
||||
:root {
|
||||
--foreground-rgb: 0, 0, 0;
|
||||
--background-start-rgb: 214, 219, 220;
|
||||
--background-end-rgb: 255, 255, 255;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--foreground-rgb: 255, 255, 255;
|
||||
--background-start-rgb: 0, 0, 0;
|
||||
--background-end-rgb: 0, 0, 0;
|
||||
}
|
||||
}
|
||||
|
||||
body {
|
||||
color: rgb(var(--foreground-rgb));
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
transparent,
|
||||
rgb(var(--background-end-rgb))
|
||||
)
|
||||
rgb(var(--background-start-rgb));
|
||||
}
|
||||
|
||||
/* Tailwind's text-white is a utility class that sets the color of the text to white.
|
||||
This will override it to use the foreground color from your root variables. */
|
||||
.text-white {
|
||||
color: rgb(var(--foreground-rgb)) !important;
|
||||
}
|
21
frontend/app/layout.tsx
Normal file
21
frontend/app/layout.tsx
Normal file
@ -0,0 +1,21 @@
|
||||
import './globals.css'
|
||||
import { Inter } from 'next/font/google'
|
||||
|
||||
const inter = Inter({ subsets: ['latin'] })
|
||||
|
||||
export const metadata = {
|
||||
title: 'Create Next App',
|
||||
description: 'Generated by create next app',
|
||||
}
|
||||
|
||||
export default function RootLayout({
|
||||
children,
|
||||
}: {
|
||||
children: React.ReactNode
|
||||
}) {
|
||||
return (
|
||||
<html lang="en">
|
||||
<body className={inter.className}>{children}</body>
|
||||
</html>
|
||||
)
|
||||
}
|
24
frontend/app/page.tsx
Normal file
24
frontend/app/page.tsx
Normal file
@ -0,0 +1,24 @@
|
||||
'use client';
|
||||
import Link from 'next/link';
|
||||
|
||||
export default function HomePage() {
|
||||
return (
|
||||
<div className="flex flex-col items-center justify-center min-h-screen bg-gray-100">
|
||||
<div className="m-4 p-6 max-w-md mx-auto bg-white rounded-xl shadow-md flex items-center space-x-4">
|
||||
<h1 className="mb-4 text-xl font-bold text-gray-900">Welcome!</h1>
|
||||
<div className="flex flex-col space-y-4">
|
||||
<Link href="/chat">
|
||||
<a className="px-4 py-2 border border-transparent text-sm font-medium rounded-md text-white bg-indigo-600 hover:bg-indigo-700">
|
||||
Go to Chat
|
||||
</a>
|
||||
</Link>
|
||||
<Link href="/upload">
|
||||
<a className="px-4 py-2 border border-transparent text-sm font-medium rounded-md text-white bg-indigo-600 hover:bg-indigo-700">
|
||||
Go to Upload
|
||||
</a>
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
49
frontend/app/upload/page.tsx
Normal file
49
frontend/app/upload/page.tsx
Normal file
@ -0,0 +1,49 @@
|
||||
'use client';
|
||||
import { useCallback, useState } from 'react';
|
||||
import { useDropzone } from 'react-dropzone';
|
||||
import axios from 'axios';
|
||||
|
||||
export default function UploadPage() {
|
||||
const [message, setMessage] = useState(null);
|
||||
|
||||
const onDrop = useCallback(async (acceptedFiles) => {
|
||||
const file = acceptedFiles[0];
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
try {
|
||||
const response = await axios.post('http://localhost:8000/upload', formData);
|
||||
setMessage({
|
||||
type: 'success',
|
||||
text: 'File uploaded successfully: ' + JSON.stringify(response.data)
|
||||
});
|
||||
} catch (error) {
|
||||
setMessage({
|
||||
type: 'error',
|
||||
text: 'Failed to upload file: ' + error.toString()
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop });
|
||||
|
||||
return (
|
||||
<div
|
||||
{...getRootProps()}
|
||||
className="flex flex-col items-center justify-center h-screen bg-gray-100"
|
||||
>
|
||||
<input {...getInputProps()} />
|
||||
<div className="mt-2 p-6 max-w-sm mx-auto bg-white rounded-xl shadow-md flex items-center space-x-4">
|
||||
{
|
||||
isDragActive
|
||||
? <p className="text-blue-600">Drop the files here...</p>
|
||||
: <p className="text-gray-500">Drag 'n' drop some files here, or click to select files</p>
|
||||
}
|
||||
</div>
|
||||
{message && (
|
||||
<div className={`mt-4 p-2 rounded ${message.type === 'success' ? 'bg-green-500' : 'bg-red-500'}`}>
|
||||
<p className="text-white">{message.text}</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
4
frontend/next.config.js
Normal file
4
frontend/next.config.js
Normal file
@ -0,0 +1,4 @@
|
||||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {}
|
||||
|
||||
module.exports = nextConfig
|
4277
frontend/package-lock.json
generated
Normal file
4277
frontend/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
27
frontend/package.json
Normal file
27
frontend/package.json
Normal file
@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "frontend",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/node": "20.1.7",
|
||||
"@types/react": "18.2.6",
|
||||
"@types/react-dom": "18.2.4",
|
||||
"autoprefixer": "10.4.14",
|
||||
"axios": "^1.4.0",
|
||||
"eslint": "8.40.0",
|
||||
"eslint-config-next": "13.4.2",
|
||||
"next": "13.4.2",
|
||||
"postcss": "8.4.23",
|
||||
"react": "18.2.0",
|
||||
"react-dom": "18.2.0",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"tailwindcss": "3.3.2",
|
||||
"typescript": "5.0.4"
|
||||
}
|
||||
}
|
3
frontend/pages/about.tsx
Normal file
3
frontend/pages/about.tsx
Normal file
@ -0,0 +1,3 @@
|
||||
export default function About() {
|
||||
return <div>About</div>;
|
||||
}
|
6
frontend/postcss.config.js
Normal file
6
frontend/postcss.config.js
Normal file
@ -0,0 +1,6 @@
|
||||
module.exports = {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
1
frontend/public/next.svg
Normal file
1
frontend/public/next.svg
Normal file
@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
|
After Width: | Height: | Size: 1.3 KiB |
1
frontend/public/vercel.svg
Normal file
1
frontend/public/vercel.svg
Normal file
@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>
|
After Width: | Height: | Size: 629 B |
18
frontend/tailwind.config.js
Normal file
18
frontend/tailwind.config.js
Normal file
@ -0,0 +1,18 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
module.exports = {
|
||||
content: [
|
||||
'./pages/**/*.{js,ts,jsx,tsx,mdx}',
|
||||
'./components/**/*.{js,ts,jsx,tsx,mdx}',
|
||||
'./app/**/*.{js,ts,jsx,tsx,mdx}',
|
||||
],
|
||||
theme: {
|
||||
extend: {
|
||||
backgroundImage: {
|
||||
'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
|
||||
'gradient-conic':
|
||||
'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
|
||||
},
|
||||
},
|
||||
},
|
||||
plugins: [],
|
||||
}
|
28
frontend/tsconfig.json
Normal file
28
frontend/tsconfig.json
Normal file
@ -0,0 +1,28 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es5",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "preserve",
|
||||
"incremental": true,
|
||||
"plugins": [
|
||||
{
|
||||
"name": "next"
|
||||
}
|
||||
],
|
||||
"paths": {
|
||||
"@/*": ["./*"]
|
||||
}
|
||||
},
|
||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
2671
frontend/yarn.lock
Normal file
2671
frontend/yarn.lock
Normal file
File diff suppressed because it is too large
Load Diff
@ -9,3 +9,6 @@ supabase==1.0.3
|
||||
tiktoken==0.4.0
|
||||
unstructured==0.6.5
|
||||
anthropic==0.2.8
|
||||
fastapi==0.95.2
|
||||
python-multipart==0.0.6
|
||||
uvicorn==0.22.0
|
105
test/api.py
Normal file
105
test/api.py
Normal file
@ -0,0 +1,105 @@
|
||||
from fastapi import FastAPI, UploadFile, File, HTTPException
|
||||
import os
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Tuple
|
||||
from supabase import create_client, Client
|
||||
from langchain.embeddings.openai import OpenAIEmbeddings
|
||||
from langchain.memory import ConversationBufferMemory
|
||||
from langchain.vectorstores import SupabaseVectorStore
|
||||
from langchain.chains import ConversationalRetrievalChain
|
||||
from langchain.llms import OpenAI
|
||||
|
||||
|
||||
from common import file_already_exists
|
||||
from txt import process_txt
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
origins = [
|
||||
"http://localhost.tiangolo.com",
|
||||
"https://localhost.tiangolo.com",
|
||||
"http://localhost",
|
||||
"http://localhost:3000",
|
||||
"http://localhost:8080",
|
||||
]
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
anthropic_api_key = ""
|
||||
supabase: Client = create_client(supabase_url, supabase_key)
|
||||
embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
|
||||
vector_store = SupabaseVectorStore(
|
||||
supabase, embeddings, table_name="documents")
|
||||
memory = ConversationBufferMemory(
|
||||
memory_key="chat_history", return_messages=True)
|
||||
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
model: str
|
||||
question: str
|
||||
history: List[Tuple[str, str]] # A list of tuples where each tuple is (speaker, text)
|
||||
|
||||
file_processors = {
|
||||
".txt": process_txt,
|
||||
}
|
||||
|
||||
async def filter_file(file: UploadFile, supabase, vector_store):
|
||||
if await file_already_exists(supabase, file):
|
||||
return f"😎 {file.filename} is already in the database."
|
||||
elif file.file._file.tell() < 1:
|
||||
return f"💨 {file.filename} is empty."
|
||||
else:
|
||||
file_extension = os.path.splitext(file.filename)[-1]
|
||||
if file_extension in file_processors:
|
||||
await file_processors[file_extension](vector_store, file, stats_db=None)
|
||||
return f"✅ {file.filename} "
|
||||
else:
|
||||
return f"❌ {file.filename} is not a valid file type."
|
||||
|
||||
@app.post("/upload")
|
||||
async def upload_file(file: UploadFile):
|
||||
# Modify your code to work with FastAPI
|
||||
# Here we assume that you have some way to get `supabase`, `openai_key`, and `vector_store`
|
||||
|
||||
print(f"Received file: {file.filename}")
|
||||
|
||||
message = await filter_file(file, supabase, vector_store)
|
||||
|
||||
return {"message": message}
|
||||
|
||||
@app.post("/chat/")
|
||||
async def chat_endpoint(chat_message: ChatMessage):
|
||||
model = chat_message.model
|
||||
question = chat_message.question
|
||||
history = chat_message.history
|
||||
temperature = 0
|
||||
max_tokens = 100
|
||||
|
||||
# Logic from your Streamlit app goes here. For example:
|
||||
qa = None
|
||||
if model.startswith("gpt"):
|
||||
qa = ConversationalRetrievalChain.from_llm(
|
||||
OpenAI(
|
||||
model_name=model, openai_api_key=openai_api_key, temperature=temperature, max_tokens=max_tokens), vector_store.as_retriever(), memory=memory, verbose=True)
|
||||
elif anthropic_api_key and model.startswith("claude"):
|
||||
qa = ConversationalRetrievalChain.from_llm(
|
||||
ChatAnthropic(
|
||||
model=model, anthropic_api_key=anthropic_api_key, temperature=temperature, max_tokens_to_sample=max_tokens), vector_store.as_retriever(), memory=memory, verbose=True, max_tokens_limit=102400)
|
||||
|
||||
history.append(("user", question))
|
||||
model_response = qa({"question": question})
|
||||
history.append(("assistant", model_response["answer"]))
|
||||
|
||||
return {"history": history}
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
return {"message": "Hello World"}
|
52
test/common.py
Normal file
52
test/common.py
Normal file
@ -0,0 +1,52 @@
|
||||
from typing import Optional
|
||||
from fastapi import UploadFile
|
||||
from langchain.schema import Document
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
# from stats import add_usage
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from utils import compute_sha1_from_file, compute_sha1_from_content
|
||||
|
||||
async def process_file(vector_store, file: UploadFile, loader_class, stats_db: Optional = None):
|
||||
documents = []
|
||||
file_sha = ""
|
||||
file_name = file.filename
|
||||
file_size = file.file._file.tell() # Getting the size of the file
|
||||
dateshort = time.strftime("%Y%m%d")
|
||||
|
||||
# Here, we're writing the uploaded file to a temporary file, so we can use it with your existing code.
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=file.filename) as tmp_file:
|
||||
await file.seek(0)
|
||||
content = await file.read()
|
||||
tmp_file.write(content)
|
||||
tmp_file.flush()
|
||||
|
||||
loader = loader_class(tmp_file.name)
|
||||
documents = loader.load()
|
||||
file_sha1 = compute_sha1_from_file(tmp_file.name) # Ensure this function works with FastAPI
|
||||
|
||||
os.remove(tmp_file.name)
|
||||
chunk_size = 500
|
||||
chunk_overlap = 0
|
||||
|
||||
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
||||
|
||||
print(documents)
|
||||
documents = text_splitter.split_documents(documents)
|
||||
|
||||
|
||||
# Add the document sha1 as metadata to each document
|
||||
docs_with_metadata = [Document(page_content=doc.page_content, metadata={"file_sha1": file_sha1, "file_size":file_size , "file_name": file_name, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort}) for doc in documents]
|
||||
|
||||
vector_store.add_documents(docs_with_metadata)
|
||||
# if stats_db:
|
||||
# add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,"file_type": file.filename, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
|
||||
return
|
||||
|
||||
async def file_already_exists(supabase, file):
|
||||
file_content = await file.read()
|
||||
file_sha1 = compute_sha1_from_content(file_content)
|
||||
response = supabase.table("documents").select("id").eq("metadata->>file_sha1", file_sha1).execute()
|
||||
return len(response.data) > 0
|
6
test/txt.py
Normal file
6
test/txt.py
Normal file
@ -0,0 +1,6 @@
|
||||
from common import process_file
|
||||
from langchain.document_loaders import TextLoader
|
||||
from fastapi import UploadFile
|
||||
|
||||
async def process_txt(vector_store, file: UploadFile, stats_db):
|
||||
return await process_file(vector_store, file, TextLoader, stats_db=stats_db)
|
11
test/utils.py
Normal file
11
test/utils.py
Normal file
@ -0,0 +1,11 @@
|
||||
import hashlib
|
||||
|
||||
def compute_sha1_from_file(file_path):
|
||||
with open(file_path, "rb") as file:
|
||||
bytes = file.read()
|
||||
readable_hash = hashlib.sha1(bytes).hexdigest()
|
||||
return readable_hash
|
||||
|
||||
def compute_sha1_from_content(content):
|
||||
readable_hash = hashlib.sha1(content).hexdigest()
|
||||
return readable_hash
|
@ -10,10 +10,12 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@vercel/analytics": "^1.0.1",
|
||||
"axios": "^1.4.0",
|
||||
"eslint": "8.39.0",
|
||||
"eslint-config-next": "13.4.1",
|
||||
"next": "13.4.1",
|
||||
"react": "18.2.0",
|
||||
"react-dom": "18.2.0"
|
||||
"react-dom": "18.2.0",
|
||||
"react-dropzone": "^14.2.3"
|
||||
}
|
||||
}
|
||||
|
@ -320,6 +320,16 @@ ast-types-flow@^0.0.7:
|
||||
resolved "https://registry.yarnpkg.com/ast-types-flow/-/ast-types-flow-0.0.7.tgz#f70b735c6bca1a5c9c22d982c3e39e7feba3bdad"
|
||||
integrity sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==
|
||||
|
||||
asynckit@^0.4.0:
|
||||
version "0.4.0"
|
||||
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
|
||||
integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==
|
||||
|
||||
attr-accept@^2.2.2:
|
||||
version "2.2.2"
|
||||
resolved "https://registry.yarnpkg.com/attr-accept/-/attr-accept-2.2.2.tgz#646613809660110749e92f2c10833b70968d929b"
|
||||
integrity sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg==
|
||||
|
||||
available-typed-arrays@^1.0.5:
|
||||
version "1.0.5"
|
||||
resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7"
|
||||
@ -330,6 +340,15 @@ axe-core@^4.6.2:
|
||||
resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.7.0.tgz#34ba5a48a8b564f67e103f0aa5768d76e15bbbbf"
|
||||
integrity sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==
|
||||
|
||||
axios@^1.4.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/axios/-/axios-1.4.0.tgz#38a7bf1224cd308de271146038b551d725f0be1f"
|
||||
integrity sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==
|
||||
dependencies:
|
||||
follow-redirects "^1.15.0"
|
||||
form-data "^4.0.0"
|
||||
proxy-from-env "^1.1.0"
|
||||
|
||||
axobject-query@^3.1.1:
|
||||
version "3.1.1"
|
||||
resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-3.1.1.tgz#3b6e5c6d4e43ca7ba51c5babf99d22a9c68485e1"
|
||||
@ -426,6 +445,13 @@ color-name@~1.1.4:
|
||||
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
|
||||
integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
|
||||
|
||||
combined-stream@^1.0.8:
|
||||
version "1.0.8"
|
||||
resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
|
||||
integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==
|
||||
dependencies:
|
||||
delayed-stream "~1.0.0"
|
||||
|
||||
concat-map@0.0.1:
|
||||
version "0.0.1"
|
||||
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
|
||||
@ -519,6 +545,11 @@ define-properties@^1.1.3, define-properties@^1.1.4, define-properties@^1.2.0:
|
||||
has-property-descriptors "^1.0.0"
|
||||
object-keys "^1.1.1"
|
||||
|
||||
delayed-stream@~1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
|
||||
integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==
|
||||
|
||||
dir-glob@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
|
||||
@ -914,6 +945,13 @@ file-entry-cache@^6.0.1:
|
||||
dependencies:
|
||||
flat-cache "^3.0.4"
|
||||
|
||||
file-selector@^0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/file-selector/-/file-selector-0.6.0.tgz#fa0a8d9007b829504db4d07dd4de0310b65287dc"
|
||||
integrity sha512-QlZ5yJC0VxHxQQsQhXvBaC7VRJ2uaxTf+Tfpu4Z/OcVQJVpZO+DGU0rkoVW5ce2SccxugvpBJoMvUs59iILYdw==
|
||||
dependencies:
|
||||
tslib "^2.4.0"
|
||||
|
||||
fill-range@^7.0.1:
|
||||
version "7.0.1"
|
||||
resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
|
||||
@ -942,6 +980,11 @@ flatted@^3.1.0:
|
||||
resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.7.tgz#609f39207cb614b89d0765b477cb2d437fbf9787"
|
||||
integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==
|
||||
|
||||
follow-redirects@^1.15.0:
|
||||
version "1.15.2"
|
||||
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
|
||||
integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
|
||||
|
||||
for-each@^0.3.3:
|
||||
version "0.3.3"
|
||||
resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e"
|
||||
@ -949,6 +992,15 @@ for-each@^0.3.3:
|
||||
dependencies:
|
||||
is-callable "^1.1.3"
|
||||
|
||||
form-data@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
|
||||
integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
|
||||
dependencies:
|
||||
asynckit "^0.4.0"
|
||||
combined-stream "^1.0.8"
|
||||
mime-types "^2.1.12"
|
||||
|
||||
fs.realpath@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
|
||||
@ -1490,6 +1542,18 @@ micromatch@^4.0.4:
|
||||
braces "^3.0.2"
|
||||
picomatch "^2.3.1"
|
||||
|
||||
mime-db@1.52.0:
|
||||
version "1.52.0"
|
||||
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
|
||||
integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
|
||||
|
||||
mime-types@^2.1.12:
|
||||
version "2.1.35"
|
||||
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
|
||||
integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
|
||||
dependencies:
|
||||
mime-db "1.52.0"
|
||||
|
||||
mimic-fn@^2.1.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
|
||||
@ -1764,6 +1828,11 @@ prop-types@^15.8.1:
|
||||
object-assign "^4.1.1"
|
||||
react-is "^16.13.1"
|
||||
|
||||
proxy-from-env@^1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
|
||||
integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
|
||||
|
||||
punycode@^2.1.0:
|
||||
version "2.3.0"
|
||||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f"
|
||||
@ -1782,6 +1851,15 @@ react-dom@18.2.0:
|
||||
loose-envify "^1.1.0"
|
||||
scheduler "^0.23.0"
|
||||
|
||||
react-dropzone@^14.2.3:
|
||||
version "14.2.3"
|
||||
resolved "https://registry.yarnpkg.com/react-dropzone/-/react-dropzone-14.2.3.tgz#0acab68308fda2d54d1273a1e626264e13d4e84b"
|
||||
integrity sha512-O3om8I+PkFKbxCukfIR3QAGftYXDZfOE2N1mr/7qebQJHs7U+/RSL/9xomJNpRg9kM5h9soQSdf0Gc7OHF5Fug==
|
||||
dependencies:
|
||||
attr-accept "^2.2.2"
|
||||
file-selector "^0.6.0"
|
||||
prop-types "^15.8.1"
|
||||
|
||||
react-is@^16.13.1:
|
||||
version "16.13.1"
|
||||
resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4"
|
||||
|
Loading…
Reference in New Issue
Block a user