QuivrHQ/quivr
1
1
Fork 0
mirror of https://github.com/QuivrHQ/quivr.git synced 2024-12-14 17:03:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feat/handle_azure_sites (#2921)

Browse Source 
# Description

Fix get all sites
This commit is contained in:
Chloé Daems 2024-07-29 15:49:04 +02:00 committed by GitHub
parent 237a1c0d0a
commit 206e25ffd2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 27 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
backend/api/quivr_api/modules/sync/utils/sync.py
View File

@ -322,8 +322,9 @@ class AzureDriveSync(BaseSync):
return credentials return credentials
def get_files(
def get_files(self, credentials, site_folder_id=None, recursive=False) -> List[SyncFile]: self, credentials, site_folder_id=None, recursive=False
) -> List[SyncFile]:
def fetch_files(endpoint, headers, max_retries=1): def fetch_files(endpoint, headers, max_retries=1):
logger.debug(f"fetching files from {endpoint}.") logger.debug(f"fetching files from {endpoint}.")
@ -365,14 +366,13 @@ class AzureDriveSync(BaseSync):
if not folder_id and not site_id: if not folder_id and not site_id:
# Fetch the sites # Fetch the sites
endpoint = "https://graph.microsoft.com/v1.0/me/followedSites" #endpoint = "https://graph.microsoft.com/v1.0/me/followedSites"
endpoint = "https://graph.microsoft.com/v1.0/sites?search=*"
elif site_id == "root": elif site_id == "root":
if not folder_id: if not folder_id:
endpoint = "https://graph.microsoft.com/v1.0/me/drive/root/children" endpoint = "https://graph.microsoft.com/v1.0/me/drive/root/children"
else: else:
endpoint = ( endpoint = f"https://graph.microsoft.com/v1.0/me/drive/items/{folder_id}/children"
f"https://graph.microsoft.com/v1.0/me/drive/items/{folder_id}/children"
)
else: else:
if not folder_id: if not folder_id:
endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/root/children" endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/root/children"
@ -380,10 +380,9 @@ class AzureDriveSync(BaseSync):
endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/items/{folder_id}/children" endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/items/{folder_id}/children"
items = fetch_files(endpoint, headers) items = fetch_files(endpoint, headers)
if not folder_id and len(items) == 0: if not folder_id and not site_id and len(items) == 0:
logger.debug("No sites found in Azure Drive, files : %s", items) logger.debug("No sites found in Azure Drive, files : %s", items)
return self.get_files(credentials, "root", recursive) return self.get_files(credentials, "root:", recursive)
if not items: if not items:
logger.info("No files found in Azure Drive") logger.info("No files found in Azure Drive")
@ -395,7 +394,9 @@ class AzureDriveSync(BaseSync):
for item in items: for item in items:
file_data = SyncFile( file_data = SyncFile(
name=item.get("name") if site_folder_id else item.get("displayName"), name=item.get("name") if site_folder_id else item.get("displayName"),
id=f'{item.get("sharepointIds", {}).get("siteId")}:' if not site_folder_id else f'{site_id}:{item.get("id")}', id=f'{item.get("id", {}).split(",")[1]}:'
if not site_folder_id
else f'{site_id}:{item.get("id")}',
is_folder="folder" in item or not site_folder_id, is_folder="folder" in item or not site_folder_id,
last_modified=item.get("lastModifiedDateTime"), last_modified=item.get("lastModifiedDateTime"),
mime_type=item.get("file", {}).get("mimeType", "folder"), mime_type=item.get("file", {}).get("mimeType", "folder"),
@ -411,14 +412,16 @@ class AzureDriveSync(BaseSync):
files.extend(folder_files) files.extend(folder_files)
if not folder_id and not site_id: if not folder_id and not site_id:
files.append(SyncFile( files.append(
SyncFile(
name="My Drive", name="My Drive",
id="root:", id="root:",
is_folder=True, is_folder=True,
last_modified="", last_modified="",
mime_type="folder", mime_type="folder",
web_view_link="https://onedrive.live.com" web_view_link="https://onedrive.live.com",
)) )
)
for file in files: for file in files:
file.name = remove_special_characters(file.name) file.name = remove_special_characters(file.name)
logger.info("Azure Drive files retrieved successfully: %s", len(files)) logger.info("Azure Drive files retrieved successfully: %s", len(files))
@ -448,7 +451,8 @@ class AzureDriveSync(BaseSync):
folder_id = None folder_id = None
if site_id == "root": if site_id == "root":
endpoint = ( endpoint = (
f"https://graph.microsoft.com/v1.0/me/drive/items/{folder_id}") f"https://graph.microsoft.com/v1.0/me/drive/items/{folder_id}"
)
else: else:
endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/items/{folder_id}" endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/items/{folder_id}"
response = requests.get(endpoint, headers=headers) response = requests.get(endpoint, headers=headers)
@ -496,9 +500,7 @@ class AzureDriveSync(BaseSync):
f"https://graph.microsoft.com/v1.0/me/drive/items/{file_id}/content" f"https://graph.microsoft.com/v1.0/me/drive/items/{file_id}/content"
) )
else: else:
download_endpoint = ( download_endpoint = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/items/{folder_id}/content"
f"https://graph.microsoft.com/v1.0/sites/{site_id}/drive/items/{folder_id}/content"
)
logger.info("Downloading file: %s", file_name) logger.info("Downloading file: %s", file_name)
download_response = requests.get( download_response = requests.get(
download_endpoint, headers=headers, stream=True download_endpoint, headers=headers, stream=True