mirror of
https://github.com/QuivrHQ/quivr.git
synced 2024-09-11 14:36:35 +03:00
feat(url): moved to own file
This commit is contained in:
parent
ea5883ddb9
commit
766a13ed9b
40
files.py
40
files.py
@ -8,6 +8,7 @@ from loaders.markdown import process_markdown
|
|||||||
from loaders.html import process_html
|
from loaders.html import process_html
|
||||||
from utils import compute_sha1_from_content
|
from utils import compute_sha1_from_content
|
||||||
from loaders.pdf import process_pdf
|
from loaders.pdf import process_pdf
|
||||||
|
from loaders.html import get_html, create_html_file, delete_tempfile
|
||||||
import requests
|
import requests
|
||||||
import re
|
import re
|
||||||
import unicodedata
|
import unicodedata
|
||||||
@ -30,7 +31,7 @@ file_processors = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def file_uploader(supabase, openai_key, vector_store):
|
def file_uploader(supabase, openai_key, vector_store):
|
||||||
files = st.file_uploader("Upload a file", accept_multiple_files=True, type=list(file_processors.keys()))
|
files = st.file_uploader("**Upload a file**", accept_multiple_files=True, type=list(file_processors.keys()))
|
||||||
if st.button("Add to Database"):
|
if st.button("Add to Database"):
|
||||||
if files is not None:
|
if files is not None:
|
||||||
for file in files:
|
for file in files:
|
||||||
@ -60,8 +61,8 @@ def filter_file(file, supabase, vector_store):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def url_uploader(supabase, openai_key, vector_store):
|
def url_uploader(supabase, openai_key, vector_store):
|
||||||
url = st.text_input("## Add an url",placeholder="https://www.quiver.app")
|
url = st.text_area("## Add an url",placeholder="https://www.quivr.app")
|
||||||
button = st.button("Add the website page to the database")
|
button = st.button("Add the URL to the database")
|
||||||
if button:
|
if button:
|
||||||
html = get_html(url)
|
html = get_html(url)
|
||||||
if html:
|
if html:
|
||||||
@ -72,36 +73,3 @@ def url_uploader(supabase, openai_key, vector_store):
|
|||||||
else:
|
else:
|
||||||
st.write(f"❌ Failed to access to {url} .")
|
st.write(f"❌ Failed to access to {url} .")
|
||||||
|
|
||||||
def get_html(url):
|
|
||||||
response = requests.get(url)
|
|
||||||
if response.status_code == 200:
|
|
||||||
return response.text
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
def create_html_file(url, content):
|
|
||||||
file_name = slugify(url) + ".html"
|
|
||||||
temp_file_path = os.path.join(tempfile.gettempdir(), file_name)
|
|
||||||
with open(temp_file_path, 'w') as temp_file:
|
|
||||||
temp_file.write(content)
|
|
||||||
|
|
||||||
record = UploadedFileRec(id=None, name=file_name, type='text/html', data=open(temp_file_path, 'rb').read())
|
|
||||||
uploaded_file = UploadedFile(record)
|
|
||||||
|
|
||||||
return uploaded_file, temp_file_path
|
|
||||||
|
|
||||||
def delete_tempfile(temp_file_path, url, ret):
|
|
||||||
try:
|
|
||||||
os.remove(temp_file_path)
|
|
||||||
if ret:
|
|
||||||
st.write(f"✅ Content saved... {url} ")
|
|
||||||
except OSError as e:
|
|
||||||
print(f"Error while deleting the temporary file: {str(e)}")
|
|
||||||
if ret:
|
|
||||||
st.write(f"❌ Error while saving content... {url} ")
|
|
||||||
|
|
||||||
def slugify(text):
|
|
||||||
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')
|
|
||||||
text = re.sub(r'[^\w\s-]', '', text).strip().lower()
|
|
||||||
text = re.sub(r'[-\s]+', '-', text)
|
|
||||||
return text
|
|
@ -1,5 +1,47 @@
|
|||||||
from .common import process_file
|
from .common import process_file
|
||||||
from langchain.document_loaders import UnstructuredHTMLLoader
|
from langchain.document_loaders import UnstructuredHTMLLoader
|
||||||
|
import requests
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
import tempfile
|
||||||
|
import os
|
||||||
|
import streamlit as st
|
||||||
|
from streamlit.runtime.uploaded_file_manager import UploadedFileRec, UploadedFile
|
||||||
|
|
||||||
def process_html(vector_store, file):
|
def process_html(vector_store, file):
|
||||||
return process_file(vector_store, file, UnstructuredHTMLLoader, ".html")
|
return process_file(vector_store, file, UnstructuredHTMLLoader, ".html")
|
||||||
|
|
||||||
|
|
||||||
|
def get_html(url):
|
||||||
|
response = requests.get(url)
|
||||||
|
if response.status_code == 200:
|
||||||
|
return response.text
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def create_html_file(url, content):
|
||||||
|
file_name = slugify(url) + ".html"
|
||||||
|
temp_file_path = os.path.join(tempfile.gettempdir(), file_name)
|
||||||
|
with open(temp_file_path, 'w') as temp_file:
|
||||||
|
temp_file.write(content)
|
||||||
|
|
||||||
|
record = UploadedFileRec(id=None, name=file_name, type='text/html', data=open(temp_file_path, 'rb').read())
|
||||||
|
uploaded_file = UploadedFile(record)
|
||||||
|
|
||||||
|
return uploaded_file, temp_file_path
|
||||||
|
|
||||||
|
def delete_tempfile(temp_file_path, url, ret):
|
||||||
|
try:
|
||||||
|
os.remove(temp_file_path)
|
||||||
|
if ret:
|
||||||
|
st.write(f"✅ Content saved... {url} ")
|
||||||
|
except OSError as e:
|
||||||
|
print(f"Error while deleting the temporary file: {str(e)}")
|
||||||
|
if ret:
|
||||||
|
st.write(f"❌ Error while saving content... {url} ")
|
||||||
|
|
||||||
|
def slugify(text):
|
||||||
|
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')
|
||||||
|
text = re.sub(r'[^\w\s-]', '', text).strip().lower()
|
||||||
|
text = re.sub(r'[-\s]+', '-', text)
|
||||||
|
return text
|
10
main.py
10
main.py
@ -63,8 +63,14 @@ if user_choice == 'Add Knowledge':
|
|||||||
"Select Chunk Size", 100, 1000, st.session_state['chunk_size'], 50)
|
"Select Chunk Size", 100, 1000, st.session_state['chunk_size'], 50)
|
||||||
st.session_state['chunk_overlap'] = st.sidebar.slider(
|
st.session_state['chunk_overlap'] = st.sidebar.slider(
|
||||||
"Select Chunk Overlap", 0, 100, st.session_state['chunk_overlap'], 10)
|
"Select Chunk Overlap", 0, 100, st.session_state['chunk_overlap'], 10)
|
||||||
file_uploader(supabase, openai_api_key, vector_store)
|
|
||||||
url_uploader(supabase, openai_api_key, vector_store)
|
# Create two columns for the file uploader and URL uploader
|
||||||
|
col1, col2 = st.columns(2)
|
||||||
|
|
||||||
|
with col1:
|
||||||
|
file_uploader(supabase, openai_api_key, vector_store)
|
||||||
|
with col2:
|
||||||
|
url_uploader(supabase, openai_api_key, vector_store)
|
||||||
elif user_choice == 'Chat with your Brain':
|
elif user_choice == 'Chat with your Brain':
|
||||||
# Display model and temperature selection only when asking questions
|
# Display model and temperature selection only when asking questions
|
||||||
st.sidebar.title("Configuration")
|
st.sidebar.title("Configuration")
|
||||||
|
Loading…
Reference in New Issue
Block a user