mirror of
https://github.com/StanGirard/quivr.git
synced 2024-09-11 13:15:41 +03:00
feat(sql): merged all into one and made migration fault tolerant
This commit is contained in:
parent
583e4d6378
commit
537efc834d
12
README.md
12
README.md
@ -101,17 +101,9 @@ cp .frontend_env.example frontend/.env
|
|||||||
|
|
||||||
- **Step 4**: Run the following migration scripts on the Supabase database via the web interface (SQL Editor -> `New query`)
|
- **Step 4**: Run the following migration scripts on the Supabase database via the web interface (SQL Editor -> `New query`)
|
||||||
|
|
||||||
[Migration Script 1](scripts/supabase_new_store_documents.sql)
|
[Creation Script 1](scripts/tables.sql)
|
||||||
|
|
||||||
[Migration Script 2](scripts/supabase_usage_table.sql)
|
> _If you come from an old version of Quivr, you can use the [migration script](scripts/20230606131110_add_uuid_user_id.sql) to migrate your data to the new version_
|
||||||
|
|
||||||
[Migration Script 3](scripts/supabase_vector_store_summary.sql)
|
|
||||||
|
|
||||||
[Migrations Script 4](scripts/supabase_users_table.sql)
|
|
||||||
|
|
||||||
[Migration Script 5](scripts/supabase_chats_table.sql)
|
|
||||||
|
|
||||||
[Migration Script 6](supabase/migrations/20230606131110_add_uuid_user_id.sql)
|
|
||||||
|
|
||||||
- **Step 5**: Launch the app
|
- **Step 5**: Launch the app
|
||||||
|
|
||||||
|
@ -100,11 +100,9 @@ cp .frontend_env.example frontend/.env
|
|||||||
|
|
||||||
[DB Table Creation Script 1](https://github.com/StanGirard/quivr/tree/main/scripts/supabase_new_store_documents.sql)
|
[DB Table Creation Script 1](https://github.com/StanGirard/quivr/tree/main/scripts/supabase_new_store_documents.sql)
|
||||||
|
|
||||||
[DB Table Creation Script 2](https://github.com/StanGirard/quivr/tree/main/scripts/supabase_usage_table.sql)
|
[Creation Script 1](https://github.com/StanGirard/quivr/tree/main/scripts/tables.sql)
|
||||||
|
|
||||||
[DB Table Creation Script 3](https://github.com/StanGirard/quivr/tree/main/scripts/supabase_vector_store_summary.sql)
|
> _If you come from an old version of Quivr, you can use the [migration script](https://github.com/StanGirard/quivr/tree/main/scripts/20230606131110_add_uuid_user_id.sql) to migrate your data to the new version_
|
||||||
|
|
||||||
[DB Table Creation Script 4](https://github.com/StanGirard/quivr/tree/main/scripts/supabase_users_table.sql)
|
|
||||||
|
|
||||||
- **Step 5**: Launch the app
|
- **Step 5**: Launch the app
|
||||||
|
|
||||||
|
33
scripts/20230606131110_add_uuid_user_id.sql
Normal file
33
scripts/20230606131110_add_uuid_user_id.sql
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- Function to check if column exists in a table
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
-- Check if email column doesn't exist, then add it
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'email') THEN
|
||||||
|
ALTER TABLE users ADD COLUMN email TEXT;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- Copy user_id to email column only if user_id column exists
|
||||||
|
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'user_id') THEN
|
||||||
|
UPDATE users SET email = user_id;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- Check if user_id column exists, then drop it
|
||||||
|
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'user_id') THEN
|
||||||
|
ALTER TABLE users DROP COLUMN user_id;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- Check if new user_id column doesn't exist, then add it
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'user_id') THEN
|
||||||
|
ALTER TABLE users ADD COLUMN user_id UUID DEFAULT gen_random_uuid();
|
||||||
|
ALTER TABLE users ADD PRIMARY KEY (user_id);
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
EXCEPTION WHEN others THEN
|
||||||
|
-- Exception block to catch errors
|
||||||
|
RAISE NOTICE 'An error occurred during migration.';
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
COMMIT;
|
@ -1,7 +0,0 @@
|
|||||||
create table if not exists chats(
|
|
||||||
chat_id uuid default uuid_generate_v4() primary key,
|
|
||||||
user_id uuid references users(user_id),
|
|
||||||
creation_time timestamp default current_timestamp,
|
|
||||||
history jsonb,
|
|
||||||
chat_name text
|
|
||||||
);
|
|
@ -1,40 +0,0 @@
|
|||||||
create extension if not exists vector;
|
|
||||||
|
|
||||||
-- Create a table to store your documents
|
|
||||||
create table if not exists vectors (
|
|
||||||
id bigserial primary key,
|
|
||||||
user_id text, -- new column added here
|
|
||||||
content text, -- corresponds to Document.pageContent
|
|
||||||
metadata jsonb, -- corresponds to Document.metadata
|
|
||||||
embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE OR REPLACE FUNCTION match_vectors(query_embedding vector(1536), match_count int, p_user_id text) -- user_id changed to p_user_id here
|
|
||||||
RETURNS TABLE(
|
|
||||||
id bigint,
|
|
||||||
user_id text, -- new column added here
|
|
||||||
content text,
|
|
||||||
metadata jsonb,
|
|
||||||
-- we return matched vectors to enable maximal marginal relevance searches
|
|
||||||
embedding vector(1536),
|
|
||||||
similarity float)
|
|
||||||
LANGUAGE plpgsql
|
|
||||||
AS $$
|
|
||||||
# variable_conflict use_column
|
|
||||||
BEGIN
|
|
||||||
RETURN query
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
user_id, -- new column added here
|
|
||||||
content,
|
|
||||||
metadata,
|
|
||||||
embedding,
|
|
||||||
1 -(vectors.embedding <=> query_embedding) AS similarity
|
|
||||||
FROM
|
|
||||||
vectors
|
|
||||||
WHERE vectors.user_id = p_user_id -- filter changed here
|
|
||||||
ORDER BY
|
|
||||||
vectors.embedding <=> query_embedding
|
|
||||||
LIMIT match_count;
|
|
||||||
END;
|
|
||||||
$$;
|
|
@ -1,12 +0,0 @@
|
|||||||
create table
|
|
||||||
stats (
|
|
||||||
-- A column called "time" with data type "timestamp"
|
|
||||||
time timestamp,
|
|
||||||
-- A column called "details" with data type "text"
|
|
||||||
chat boolean,
|
|
||||||
embedding boolean,
|
|
||||||
details text,
|
|
||||||
metadata jsonb,
|
|
||||||
-- An "integer" primary key column called "id" that is generated always as identity
|
|
||||||
id integer primary key generated always as identity
|
|
||||||
);
|
|
@ -1,6 +0,0 @@
|
|||||||
create table if not exists users(
|
|
||||||
user_id uuid,
|
|
||||||
email text,
|
|
||||||
date text,
|
|
||||||
requests_count int
|
|
||||||
);
|
|
@ -1,38 +0,0 @@
|
|||||||
-- Create a table to store your summaries
|
|
||||||
create table if not exists summaries (
|
|
||||||
id bigserial primary key,
|
|
||||||
document_id bigint references vectors(id),
|
|
||||||
content text, -- corresponds to the summarized content
|
|
||||||
metadata jsonb, -- corresponds to Document.metadata
|
|
||||||
embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE OR REPLACE FUNCTION match_summaries(query_embedding vector(1536), match_count int, match_threshold float)
|
|
||||||
RETURNS TABLE(
|
|
||||||
id bigint,
|
|
||||||
document_id bigint,
|
|
||||||
content text,
|
|
||||||
metadata jsonb,
|
|
||||||
-- we return matched vectors to enable maximal marginal relevance searches
|
|
||||||
embedding vector(1536),
|
|
||||||
similarity float)
|
|
||||||
LANGUAGE plpgsql
|
|
||||||
AS $$
|
|
||||||
# variable_conflict use_column
|
|
||||||
BEGIN
|
|
||||||
RETURN query
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
document_id,
|
|
||||||
content,
|
|
||||||
metadata,
|
|
||||||
embedding,
|
|
||||||
1 -(summaries.embedding <=> query_embedding) AS similarity
|
|
||||||
FROM
|
|
||||||
summaries
|
|
||||||
WHERE 1 - (summaries.embedding <=> query_embedding) > match_threshold
|
|
||||||
ORDER BY
|
|
||||||
summaries.embedding <=> query_embedding
|
|
||||||
LIMIT match_count;
|
|
||||||
END;
|
|
||||||
$$;
|
|
105
scripts/tables.sql
Normal file
105
scripts/tables.sql
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
-- Create users table
|
||||||
|
CREATE TABLE IF NOT EXISTS users(
|
||||||
|
user_id UUID DEFAULT gen_random_uuid() PRIMARY KEY,
|
||||||
|
email TEXT,
|
||||||
|
date TEXT,
|
||||||
|
requests_count INT
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Create chats table
|
||||||
|
CREATE TABLE IF NOT EXISTS chats(
|
||||||
|
chat_id UUID DEFAULT uuid_generate_v4() PRIMARY KEY,
|
||||||
|
user_id UUID REFERENCES users(user_id),
|
||||||
|
creation_time TIMESTAMP DEFAULT current_timestamp,
|
||||||
|
history JSONB,
|
||||||
|
chat_name TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Create vector extension
|
||||||
|
CREATE EXTENSION IF NOT EXISTS vector;
|
||||||
|
|
||||||
|
-- Create vectors table
|
||||||
|
CREATE TABLE IF NOT EXISTS vectors (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
user_id TEXT,
|
||||||
|
content TEXT,
|
||||||
|
metadata JSONB,
|
||||||
|
embedding VECTOR(1536)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Create function to match vectors
|
||||||
|
CREATE OR REPLACE FUNCTION match_vectors(query_embedding VECTOR(1536), match_count INT, p_user_id TEXT)
|
||||||
|
RETURNS TABLE(
|
||||||
|
id BIGINT,
|
||||||
|
user_id TEXT,
|
||||||
|
content TEXT,
|
||||||
|
metadata JSONB,
|
||||||
|
embedding VECTOR(1536),
|
||||||
|
similarity FLOAT
|
||||||
|
) LANGUAGE plpgsql AS $$
|
||||||
|
#variable_conflict use_column
|
||||||
|
BEGIN
|
||||||
|
RETURN QUERY
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
user_id,
|
||||||
|
content,
|
||||||
|
metadata,
|
||||||
|
embedding,
|
||||||
|
1 - (vectors.embedding <=> query_embedding) AS similarity
|
||||||
|
FROM
|
||||||
|
vectors
|
||||||
|
WHERE vectors.user_id = p_user_id
|
||||||
|
ORDER BY
|
||||||
|
vectors.embedding <=> query_embedding
|
||||||
|
LIMIT match_count;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
-- Create stats table
|
||||||
|
CREATE TABLE stats (
|
||||||
|
time TIMESTAMP,
|
||||||
|
chat BOOLEAN,
|
||||||
|
embedding BOOLEAN,
|
||||||
|
details TEXT,
|
||||||
|
metadata JSONB,
|
||||||
|
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Create summaries table
|
||||||
|
CREATE TABLE IF NOT EXISTS summaries (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
document_id BIGINT REFERENCES vectors(id),
|
||||||
|
content TEXT,
|
||||||
|
metadata JSONB,
|
||||||
|
embedding VECTOR(1536)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Create function to match summaries
|
||||||
|
CREATE OR REPLACE FUNCTION match_summaries(query_embedding VECTOR(1536), match_count INT, match_threshold FLOAT)
|
||||||
|
RETURNS TABLE(
|
||||||
|
id BIGINT,
|
||||||
|
document_id BIGINT,
|
||||||
|
content TEXT,
|
||||||
|
metadata JSONB,
|
||||||
|
embedding VECTOR(1536),
|
||||||
|
similarity FLOAT
|
||||||
|
) LANGUAGE plpgsql AS $$
|
||||||
|
#variable_conflict use_column
|
||||||
|
BEGIN
|
||||||
|
RETURN QUERY
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
document_id,
|
||||||
|
content,
|
||||||
|
metadata,
|
||||||
|
embedding,
|
||||||
|
1 - (summaries.embedding <=> query_embedding) AS similarity
|
||||||
|
FROM
|
||||||
|
summaries
|
||||||
|
WHERE 1 - (summaries.embedding <=> query_embedding) > match_threshold
|
||||||
|
ORDER BY
|
||||||
|
summaries.embedding <=> query_embedding
|
||||||
|
LIMIT match_count;
|
||||||
|
END;
|
||||||
|
$$;
|
@ -1,13 +0,0 @@
|
|||||||
BEGIN;
|
|
||||||
|
|
||||||
-- Create a new column for email and copy the current user_id to it
|
|
||||||
ALTER TABLE users ADD COLUMN email text;
|
|
||||||
UPDATE users SET email = user_id;
|
|
||||||
|
|
||||||
-- Drop the current user_id column
|
|
||||||
ALTER TABLE users DROP COLUMN user_id;
|
|
||||||
|
|
||||||
-- Create a new UUID user_id column and set it as the primary key
|
|
||||||
ALTER TABLE users ADD COLUMN user_id UUID DEFAULT gen_random_uuid() PRIMARY KEY;
|
|
||||||
|
|
||||||
COMMIT;
|
|
Loading…
Reference in New Issue
Block a user