From ed5de8b80c6c25fea743a35091237de10efc5dc7 Mon Sep 17 00:00:00 2001 From: Zineb El Bachiri <100568984+gozineb@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:57:25 +0100 Subject: [PATCH] feat: supabase vault (#1605) # Description https://github.com/StanGirard/quivr/issues/1551 ## Checklist before requesting a review Please delete options that are not relevant. - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my code - [ ] I have commented hard-to-understand areas - [ ] I have ideally added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged ## Screenshots (if appropriate): --- .../external_api_secret/__init__.py | 3 ++ .../external_api_secret/create_secret.py | 21 ++++++++ .../external_api_secret/delete_secret.py | 18 +++++++ .../external_api_secret/read_secret.py | 20 +++++++ .../repository/external_api_secret/utils.py | 5 ++ scripts/20231107104700_setup_vault.sql | 53 +++++++++++++++++++ .../20230606131110_add_uuid_user_id.sql | 0 .../20230620170840_add_vectors_brains.sql | 0 .../20230620183620_use_supabase_user_id.sql | 0 .../20230627151100_update_match_vectors.sql | 0 ...629143400_add_file_sha1_brains_vectors.sql | 0 .../20230701180101_add_prompts_table.sql | 0 ...030_add_subscription_invitations_table.sql | 0 ...202307111517031_change_vectors_id_type.sql | 0 ...17164900_add_get_user_email_by_user_id.sql | 0 ...17173000_add_get_user_id_by_user_email.sql | 0 .../202307241530031_add_fields_to_brain.sql | 0 ...20230731172400_add_user_identity_table.sql | 0 .../20230802120700_add_prompt_id_to_brain.sql | 0 ...ompt_id_brain_id_to_chat_history_table.sql | 0 .../202308181004030_rename_users_table.sql | 0 ...8217004800_add_public_prompts_examples.sql | 0 ...20230906151400_add_notifications_table.sql | 0 .../202309127004032_add_user_limits.sql | 0 .../20230913110420_add_storage_bucket.sql | 0 .../202309151054032_add_knowledge_tables.sql | 0 .../202309157004032_add_sha1_column.sql | 0 ...1160000_add_last_update_field_to_brain.sql | 0 .../202309307004032_change_user_settings.sql | 0 .../20231004150000_add_onboarding_table.sql | 0 ...0_add_onboarding_a_to_onboarding_table.sql | 0 ...00_add_create_user_onboarding_function.sql | 0 ...add_creation_time_to_onboardings_table.sql | 0 .../20231023140000_add_stripe_wrapper.sql | 0 ...160000_copy_auth_users_to_public_users.sql | 0 scripts/pg_tables.sql | 50 ++++++++++++++++- scripts/tables.sql | 52 ++++++++++++++++-- 37 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 backend/repository/external_api_secret/__init__.py create mode 100644 backend/repository/external_api_secret/create_secret.py create mode 100644 backend/repository/external_api_secret/delete_secret.py create mode 100644 backend/repository/external_api_secret/read_secret.py create mode 100644 backend/repository/external_api_secret/utils.py create mode 100644 scripts/20231107104700_setup_vault.sql rename scripts/{ => old_migrations/migrations_06_2023}/20230606131110_add_uuid_user_id.sql (100%) rename scripts/{ => old_migrations/migrations_06_2023}/20230620170840_add_vectors_brains.sql (100%) rename scripts/{ => old_migrations/migrations_06_2023}/20230620183620_use_supabase_user_id.sql (100%) rename scripts/{ => old_migrations/migrations_06_2023}/20230627151100_update_match_vectors.sql (100%) rename scripts/{ => old_migrations/migrations_06_2023}/20230629143400_add_file_sha1_brains_vectors.sql (100%) rename scripts/{ => old_migrations/migrations_07_2023}/20230701180101_add_prompts_table.sql (100%) rename scripts/{ => old_migrations/migrations_07_2023}/202307111517030_add_subscription_invitations_table.sql (100%) rename scripts/{ => old_migrations/migrations_07_2023}/202307111517031_change_vectors_id_type.sql (100%) rename scripts/{ => old_migrations/migrations_07_2023}/20230717164900_add_get_user_email_by_user_id.sql (100%) rename scripts/{ => old_migrations/migrations_07_2023}/20230717173000_add_get_user_id_by_user_email.sql (100%) rename scripts/{ => old_migrations/migrations_07_2023}/202307241530031_add_fields_to_brain.sql (100%) rename scripts/{ => old_migrations/migrations_07_2023}/20230731172400_add_user_identity_table.sql (100%) rename scripts/{ => old_migrations/migrations_08_2023}/20230802120700_add_prompt_id_to_brain.sql (100%) rename scripts/{ => old_migrations/migrations_08_2023}/20230809154300_add_prompt_id_brain_id_to_chat_history_table.sql (100%) rename scripts/{ => old_migrations/migrations_08_2023}/202308181004030_rename_users_table.sql (100%) rename scripts/{ => old_migrations/migrations_08_2023}/202308217004800_add_public_prompts_examples.sql (100%) rename scripts/{ => old_migrations/migrations_09_2023}/20230906151400_add_notifications_table.sql (100%) rename scripts/{ => old_migrations/migrations_09_2023}/202309127004032_add_user_limits.sql (100%) rename scripts/{ => old_migrations/migrations_09_2023}/20230913110420_add_storage_bucket.sql (100%) rename scripts/{ => old_migrations/migrations_09_2023}/202309151054032_add_knowledge_tables.sql (100%) rename scripts/{ => old_migrations/migrations_09_2023}/202309157004032_add_sha1_column.sql (100%) rename scripts/{ => old_migrations/migrations_09_2023}/20230921160000_add_last_update_field_to_brain.sql (100%) rename scripts/{ => old_migrations/migrations_09_2023}/202309307004032_change_user_settings.sql (100%) rename scripts/{ => old_migrations/migrations_10_2023}/20231004150000_add_onboarding_table.sql (100%) rename scripts/{ => old_migrations/migrations_10_2023}/20231005170000_add_onboarding_a_to_onboarding_table.sql (100%) rename scripts/{ => old_migrations/migrations_10_2023}/20231010120000_add_create_user_onboarding_function.sql (100%) rename scripts/{ => old_migrations/migrations_10_2023}/20231012150000_add_creation_time_to_onboardings_table.sql (100%) rename scripts/{ => old_migrations/migrations_10_2023}/20231023140000_add_stripe_wrapper.sql (100%) rename scripts/{ => old_migrations/migrations_10_2023}/20231023160000_copy_auth_users_to_public_users.sql (100%) diff --git a/backend/repository/external_api_secret/__init__.py b/backend/repository/external_api_secret/__init__.py new file mode 100644 index 000000000..3c7e3b446 --- /dev/null +++ b/backend/repository/external_api_secret/__init__.py @@ -0,0 +1,3 @@ +from .create_secret import create_secret +from .delete_secret import delete_secret +from .read_secret import read_secret diff --git a/backend/repository/external_api_secret/create_secret.py b/backend/repository/external_api_secret/create_secret.py new file mode 100644 index 000000000..0b902d139 --- /dev/null +++ b/backend/repository/external_api_secret/create_secret.py @@ -0,0 +1,21 @@ +from uuid import UUID + +from models import get_supabase_client +from utils import build_secret_unique_name + + +def create_secret( + user_id: UUID, brain_id: UUID, secret_name: str, secret_value +) -> UUID | None: + supabase_client = get_supabase_client() + response = supabase_client.rpc( + "insert_secret", + { + "name": build_secret_unique_name( + user_id=user_id, brain_id=brain_id, secret_name=secret_name + ), + "secret": secret_value, + }, + ).execute() + + return response.data diff --git a/backend/repository/external_api_secret/delete_secret.py b/backend/repository/external_api_secret/delete_secret.py new file mode 100644 index 000000000..74a931604 --- /dev/null +++ b/backend/repository/external_api_secret/delete_secret.py @@ -0,0 +1,18 @@ +from uuid import UUID + +from models import get_supabase_client +from utils import build_secret_unique_name + + +def delete_secret(user_id: UUID, brain_id: UUID, secret_name: str) -> bool: + supabase_client = get_supabase_client() + response = supabase_client.rpc( + "delete_secret", + { + "name": build_secret_unique_name( + user_id=user_id, brain_id=brain_id, secret_name=secret_name + ), + }, + ).execute() + + return response.data diff --git a/backend/repository/external_api_secret/read_secret.py b/backend/repository/external_api_secret/read_secret.py new file mode 100644 index 000000000..9fa3a6481 --- /dev/null +++ b/backend/repository/external_api_secret/read_secret.py @@ -0,0 +1,20 @@ +from uuid import UUID + +from models import get_supabase_client +from utils import build_secret_unique_name + + +def read_secret( + user_id: UUID, brain_id: UUID, secret_name: str, secret_value +) -> UUID | None: + supabase_client = get_supabase_client() + response = supabase_client.rpc( + "read_secret", + { + "secret_name": build_secret_unique_name( + user_id=user_id, brain_id=brain_id, secret_name=secret_name + ), + }, + ).execute() + + return response.data diff --git a/backend/repository/external_api_secret/utils.py b/backend/repository/external_api_secret/utils.py new file mode 100644 index 000000000..8db943a1e --- /dev/null +++ b/backend/repository/external_api_secret/utils.py @@ -0,0 +1,5 @@ +from uuid import UUID + + +def build_secret_unique_name(user_id: UUID, brain_id: UUID, secret_name: str): + return f"{user_id}-{brain_id}-{secret_name}" diff --git a/scripts/20231107104700_setup_vault.sql b/scripts/20231107104700_setup_vault.sql new file mode 100644 index 000000000..1b3ac91a1 --- /dev/null +++ b/scripts/20231107104700_setup_vault.sql @@ -0,0 +1,53 @@ +CREATE OR REPLACE FUNCTION insert_secret(name text, secret text) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +begin + return vault.create_secret(secret, name); +end; +$$; + + +create or replace function read_secret(secret_name text) +returns text +language plpgsql +security definer set search_path = public +as $$ +declare + secret text; +begin + select decrypted_secret from vault.decrypted_secrets where name = + secret_name into secret; + return secret; +end; +$$; + +create or replace function delete_secret(secret_name text) +returns text +language plpgsql +security definer set search_path = public +as $$ +declare + deleted_rows int; +begin + delete from vault.decrypted_secrets where name = secret_name; + get diagnostics deleted_rows = row_count; + if deleted_rows = 0 then + return false; + else + return true; + end if; +end; +$$; + +-- Insert a migration record if it doesn't exist +INSERT INTO migrations (name) +SELECT '20231107104700_setup_vault' +WHERE NOT EXISTS ( + SELECT 1 FROM migrations WHERE name = '20231107104700_setup_vault' +); + +-- Commit the changes +COMMIT; diff --git a/scripts/20230606131110_add_uuid_user_id.sql b/scripts/old_migrations/migrations_06_2023/20230606131110_add_uuid_user_id.sql similarity index 100% rename from scripts/20230606131110_add_uuid_user_id.sql rename to scripts/old_migrations/migrations_06_2023/20230606131110_add_uuid_user_id.sql diff --git a/scripts/20230620170840_add_vectors_brains.sql b/scripts/old_migrations/migrations_06_2023/20230620170840_add_vectors_brains.sql similarity index 100% rename from scripts/20230620170840_add_vectors_brains.sql rename to scripts/old_migrations/migrations_06_2023/20230620170840_add_vectors_brains.sql diff --git a/scripts/20230620183620_use_supabase_user_id.sql b/scripts/old_migrations/migrations_06_2023/20230620183620_use_supabase_user_id.sql similarity index 100% rename from scripts/20230620183620_use_supabase_user_id.sql rename to scripts/old_migrations/migrations_06_2023/20230620183620_use_supabase_user_id.sql diff --git a/scripts/20230627151100_update_match_vectors.sql b/scripts/old_migrations/migrations_06_2023/20230627151100_update_match_vectors.sql similarity index 100% rename from scripts/20230627151100_update_match_vectors.sql rename to scripts/old_migrations/migrations_06_2023/20230627151100_update_match_vectors.sql diff --git a/scripts/20230629143400_add_file_sha1_brains_vectors.sql b/scripts/old_migrations/migrations_06_2023/20230629143400_add_file_sha1_brains_vectors.sql similarity index 100% rename from scripts/20230629143400_add_file_sha1_brains_vectors.sql rename to scripts/old_migrations/migrations_06_2023/20230629143400_add_file_sha1_brains_vectors.sql diff --git a/scripts/20230701180101_add_prompts_table.sql b/scripts/old_migrations/migrations_07_2023/20230701180101_add_prompts_table.sql similarity index 100% rename from scripts/20230701180101_add_prompts_table.sql rename to scripts/old_migrations/migrations_07_2023/20230701180101_add_prompts_table.sql diff --git a/scripts/202307111517030_add_subscription_invitations_table.sql b/scripts/old_migrations/migrations_07_2023/202307111517030_add_subscription_invitations_table.sql similarity index 100% rename from scripts/202307111517030_add_subscription_invitations_table.sql rename to scripts/old_migrations/migrations_07_2023/202307111517030_add_subscription_invitations_table.sql diff --git a/scripts/202307111517031_change_vectors_id_type.sql b/scripts/old_migrations/migrations_07_2023/202307111517031_change_vectors_id_type.sql similarity index 100% rename from scripts/202307111517031_change_vectors_id_type.sql rename to scripts/old_migrations/migrations_07_2023/202307111517031_change_vectors_id_type.sql diff --git a/scripts/20230717164900_add_get_user_email_by_user_id.sql b/scripts/old_migrations/migrations_07_2023/20230717164900_add_get_user_email_by_user_id.sql similarity index 100% rename from scripts/20230717164900_add_get_user_email_by_user_id.sql rename to scripts/old_migrations/migrations_07_2023/20230717164900_add_get_user_email_by_user_id.sql diff --git a/scripts/20230717173000_add_get_user_id_by_user_email.sql b/scripts/old_migrations/migrations_07_2023/20230717173000_add_get_user_id_by_user_email.sql similarity index 100% rename from scripts/20230717173000_add_get_user_id_by_user_email.sql rename to scripts/old_migrations/migrations_07_2023/20230717173000_add_get_user_id_by_user_email.sql diff --git a/scripts/202307241530031_add_fields_to_brain.sql b/scripts/old_migrations/migrations_07_2023/202307241530031_add_fields_to_brain.sql similarity index 100% rename from scripts/202307241530031_add_fields_to_brain.sql rename to scripts/old_migrations/migrations_07_2023/202307241530031_add_fields_to_brain.sql diff --git a/scripts/20230731172400_add_user_identity_table.sql b/scripts/old_migrations/migrations_07_2023/20230731172400_add_user_identity_table.sql similarity index 100% rename from scripts/20230731172400_add_user_identity_table.sql rename to scripts/old_migrations/migrations_07_2023/20230731172400_add_user_identity_table.sql diff --git a/scripts/20230802120700_add_prompt_id_to_brain.sql b/scripts/old_migrations/migrations_08_2023/20230802120700_add_prompt_id_to_brain.sql similarity index 100% rename from scripts/20230802120700_add_prompt_id_to_brain.sql rename to scripts/old_migrations/migrations_08_2023/20230802120700_add_prompt_id_to_brain.sql diff --git a/scripts/20230809154300_add_prompt_id_brain_id_to_chat_history_table.sql b/scripts/old_migrations/migrations_08_2023/20230809154300_add_prompt_id_brain_id_to_chat_history_table.sql similarity index 100% rename from scripts/20230809154300_add_prompt_id_brain_id_to_chat_history_table.sql rename to scripts/old_migrations/migrations_08_2023/20230809154300_add_prompt_id_brain_id_to_chat_history_table.sql diff --git a/scripts/202308181004030_rename_users_table.sql b/scripts/old_migrations/migrations_08_2023/202308181004030_rename_users_table.sql similarity index 100% rename from scripts/202308181004030_rename_users_table.sql rename to scripts/old_migrations/migrations_08_2023/202308181004030_rename_users_table.sql diff --git a/scripts/202308217004800_add_public_prompts_examples.sql b/scripts/old_migrations/migrations_08_2023/202308217004800_add_public_prompts_examples.sql similarity index 100% rename from scripts/202308217004800_add_public_prompts_examples.sql rename to scripts/old_migrations/migrations_08_2023/202308217004800_add_public_prompts_examples.sql diff --git a/scripts/20230906151400_add_notifications_table.sql b/scripts/old_migrations/migrations_09_2023/20230906151400_add_notifications_table.sql similarity index 100% rename from scripts/20230906151400_add_notifications_table.sql rename to scripts/old_migrations/migrations_09_2023/20230906151400_add_notifications_table.sql diff --git a/scripts/202309127004032_add_user_limits.sql b/scripts/old_migrations/migrations_09_2023/202309127004032_add_user_limits.sql similarity index 100% rename from scripts/202309127004032_add_user_limits.sql rename to scripts/old_migrations/migrations_09_2023/202309127004032_add_user_limits.sql diff --git a/scripts/20230913110420_add_storage_bucket.sql b/scripts/old_migrations/migrations_09_2023/20230913110420_add_storage_bucket.sql similarity index 100% rename from scripts/20230913110420_add_storage_bucket.sql rename to scripts/old_migrations/migrations_09_2023/20230913110420_add_storage_bucket.sql diff --git a/scripts/202309151054032_add_knowledge_tables.sql b/scripts/old_migrations/migrations_09_2023/202309151054032_add_knowledge_tables.sql similarity index 100% rename from scripts/202309151054032_add_knowledge_tables.sql rename to scripts/old_migrations/migrations_09_2023/202309151054032_add_knowledge_tables.sql diff --git a/scripts/202309157004032_add_sha1_column.sql b/scripts/old_migrations/migrations_09_2023/202309157004032_add_sha1_column.sql similarity index 100% rename from scripts/202309157004032_add_sha1_column.sql rename to scripts/old_migrations/migrations_09_2023/202309157004032_add_sha1_column.sql diff --git a/scripts/20230921160000_add_last_update_field_to_brain.sql b/scripts/old_migrations/migrations_09_2023/20230921160000_add_last_update_field_to_brain.sql similarity index 100% rename from scripts/20230921160000_add_last_update_field_to_brain.sql rename to scripts/old_migrations/migrations_09_2023/20230921160000_add_last_update_field_to_brain.sql diff --git a/scripts/202309307004032_change_user_settings.sql b/scripts/old_migrations/migrations_09_2023/202309307004032_change_user_settings.sql similarity index 100% rename from scripts/202309307004032_change_user_settings.sql rename to scripts/old_migrations/migrations_09_2023/202309307004032_change_user_settings.sql diff --git a/scripts/20231004150000_add_onboarding_table.sql b/scripts/old_migrations/migrations_10_2023/20231004150000_add_onboarding_table.sql similarity index 100% rename from scripts/20231004150000_add_onboarding_table.sql rename to scripts/old_migrations/migrations_10_2023/20231004150000_add_onboarding_table.sql diff --git a/scripts/20231005170000_add_onboarding_a_to_onboarding_table.sql b/scripts/old_migrations/migrations_10_2023/20231005170000_add_onboarding_a_to_onboarding_table.sql similarity index 100% rename from scripts/20231005170000_add_onboarding_a_to_onboarding_table.sql rename to scripts/old_migrations/migrations_10_2023/20231005170000_add_onboarding_a_to_onboarding_table.sql diff --git a/scripts/20231010120000_add_create_user_onboarding_function.sql b/scripts/old_migrations/migrations_10_2023/20231010120000_add_create_user_onboarding_function.sql similarity index 100% rename from scripts/20231010120000_add_create_user_onboarding_function.sql rename to scripts/old_migrations/migrations_10_2023/20231010120000_add_create_user_onboarding_function.sql diff --git a/scripts/20231012150000_add_creation_time_to_onboardings_table.sql b/scripts/old_migrations/migrations_10_2023/20231012150000_add_creation_time_to_onboardings_table.sql similarity index 100% rename from scripts/20231012150000_add_creation_time_to_onboardings_table.sql rename to scripts/old_migrations/migrations_10_2023/20231012150000_add_creation_time_to_onboardings_table.sql diff --git a/scripts/20231023140000_add_stripe_wrapper.sql b/scripts/old_migrations/migrations_10_2023/20231023140000_add_stripe_wrapper.sql similarity index 100% rename from scripts/20231023140000_add_stripe_wrapper.sql rename to scripts/old_migrations/migrations_10_2023/20231023140000_add_stripe_wrapper.sql diff --git a/scripts/20231023160000_copy_auth_users_to_public_users.sql b/scripts/old_migrations/migrations_10_2023/20231023160000_copy_auth_users_to_public_users.sql similarity index 100% rename from scripts/20231023160000_copy_auth_users_to_public_users.sql rename to scripts/old_migrations/migrations_10_2023/20231023160000_copy_auth_users_to_public_users.sql diff --git a/scripts/pg_tables.sql b/scripts/pg_tables.sql index 79cd702b6..99df3d329 100644 --- a/scripts/pg_tables.sql +++ b/scripts/pg_tables.sql @@ -167,13 +167,59 @@ CREATE TABLE IF NOT EXISTS brain_subscription_invitations ( FOREIGN KEY (brain_id) REFERENCES brains (brain_id) ); +-- Create functions for secrets in vault +CREATE OR REPLACE FUNCTION insert_secret(name text, secret text) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +begin + return vault.create_secret(secret, name); +end; +$$; + + +create or replace function read_secret(secret_name text) +returns text +language plpgsql +security definer set search_path = public +as $$ +declare + secret text; +begin + select decrypted_secret from vault.decrypted_secrets where name = + secret_name into secret; + return secret; +end; +$$; + +create or replace function delete_secret(secret_name text) +returns text +language plpgsql +security definer set search_path = public +as $$ +declare + deleted_rows int; +begin + delete from vault.decrypted_secrets where name = secret_name; + get diagnostics deleted_rows = row_count; + if deleted_rows = 0 then + return false; + else + return true; + end if; +end; +$$; + + CREATE TABLE IF NOT EXISTS migrations ( name VARCHAR(255) PRIMARY KEY, executed_at TIMESTAMPTZ DEFAULT current_timestamp ); INSERT INTO migrations (name) -SELECT '202307111517031_change_vectors_id_type' +SELECT '20231107104700_setup_vault' WHERE NOT EXISTS ( - SELECT 1 FROM migrations WHERE name = '202307111517031_change_vectors_id_type' + SELECT 1 FROM migrations WHERE name = '20231107104700_setup_vault' ); \ No newline at end of file diff --git a/scripts/tables.sql b/scripts/tables.sql index 4ae46c60a..24876dc81 100644 --- a/scripts/tables.sql +++ b/scripts/tables.sql @@ -398,10 +398,54 @@ CREATE POLICY "Access Quivr Storage 1jccrwz_2" ON storage.objects FOR UPDATE TO CREATE POLICY "Access Quivr Storage 1jccrwz_3" ON storage.objects FOR DELETE TO anon USING (bucket_id = 'quivr'); +-- Create functions for secrets in vault +CREATE OR REPLACE FUNCTION insert_secret(name text, secret text) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +begin + return vault.create_secret(secret, name); +end; +$$; + + +create or replace function read_secret(secret_name text) +returns text +language plpgsql +security definer set search_path = public +as $$ +declare + secret text; +begin + select decrypted_secret from vault.decrypted_secrets where name = + secret_name into secret; + return secret; +end; +$$; + +create or replace function delete_secret(secret_name text) +returns text +language plpgsql +security definer set search_path = public +as $$ +declare + deleted_rows int; +begin + delete from vault.decrypted_secrets where name = secret_name; + get diagnostics deleted_rows = row_count; + if deleted_rows = 0 then + return false; + else + return true; + end if; +end; +$$; + + INSERT INTO migrations (name) -SELECT '20231106110000_add_field_brain_type_to_brain_table' +SELECT '20231107104700_setup_vault' WHERE NOT EXISTS ( - SELECT 1 FROM migrations WHERE name = '20231106110000_add_field_brain_type_to_brain_table' + SELECT 1 FROM migrations WHERE name = '20231107104700_setup_vault' ); - -