QuivrHQ/quivr
1
1
Fork 0
mirror of https://github.com/QuivrHQ/quivr.git synced 2024-09-11 14:36:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

notion sync wip

Browse Source
This commit is contained in:
aminediro 2024-08-16 10:21:32 +02:00
parent fe366d82d5
commit f0acc92a70
3 changed files with 40 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/api/quivr_api/modules/sync/controller/notion_sync_routes.py
View File

@ -137,7 +137,7 @@ def oauth2callback_notion(request: Request, background_tasks: BackgroundTasks):
account_id = owner_info["id"]
result: dict[str, str] = {
"access_token": oauth_result["access_token"],
"access_token": access_token,
"refresh_token": oauth_result.get("refresh_token", ""),
"account_id": account_id,
"expires_in": oauth_result.get("expires_in", ""),

66
backend/api/quivr_api/modules/sync/service/sync_notion.py
View File

@ -37,7 +37,9 @@ class SyncNotionService(BaseService[NotionRepository]):
else None
),
name=f'{page["properties"]["title"]["title"][0]["text"]["content"]}.md',
icon=page["icon"]["emoji"] if page["icon"] else None,
icon=page["icon"]["emoji"]
if "icon" in page and "emoji" in page["icon"]
else None,
mime_type="md",
web_view_link=page["url"],
is_folder=True,
@ -77,7 +79,9 @@ class SyncNotionService(BaseService[NotionRepository]):
notion_id=page["id"],
parent_id=page["parent"][parent_type],
name=f'{page["properties"]["title"]["title"][0]["text"]["content"]}.md',
icon=page["icon"]["emoji"] if page["icon"] else None,
icon=page["icon"]["emoji"]
if "icon" in page and "emoji" in page["icon"]
else None,
mime_type="md",
web_view_link=page["url"],
is_folder=True,
@ -106,9 +110,9 @@ class SyncNotionService(BaseService[NotionRepository]):
or child_notion_page["in_trash"]
):
pages_to_delete.append(child.notion_id)
except:
logger.info(
f"Page {child.notion_id} is in trash or archived, we are deleting it"
except Exception:
logger.error(
f"Page {child.notion_id} is in trash or archived, we are deleting it."
)
pages_to_delete.append(child.notion_id)
@ -186,17 +190,38 @@ async def store_notion_pages(
def fetch_notion_pages(
notion_client: Client, notion_sync: dict[str, Any] | None = None
notion_client: Client,
last_sync_time: datetime = datetime.now() - timedelta(hours=6),
) -> List[dict[str, Any]]:
all_search_result = []
last_sync_time = datetime.now() - timedelta(hours=6)
search_result = notion_client.search(
query="",
filter={"property": "object", "value": "page"},
sort={"direction": "descending", "timestamp": "last_edited_time"},
)
last_edited_time: datetime | None = None
if notion_sync:
for page in search_result["results"]:
last_edited_time = datetime.strptime(
page["last_edited_time"], "%Y-%m-%dT%H:%M:%S.%fZ"
)
if last_edited_time > last_sync_time:
all_search_result.append(page)
if last_edited_time and last_edited_time < last_sync_time:
# We check if the last element of the search result is older than 6 hours, if it is, we stop the search
return all_search_result
while search_result["has_more"]: # type: ignore
logger.debug("Next cursor: %s", search_result["next_cursor"]) # type: ignore
search_result = notion_client.search(
query="",
filter={"property": "object", "value": "page"},
sort={"direction": "descending", "timestamp": "last_edited_time"},
start_cursor=search_result["next_cursor"], # type: ignore
)
for page in search_result["results"]:
last_edited_time = datetime.strptime(
page["last_edited_time"], "%Y-%m-%dT%H:%M:%S.%fZ"
@ -208,29 +233,4 @@ def fetch_notion_pages(
# We check if the last element of the search result is older than 6 hours, if it is, we stop the search
return all_search_result
while search_result["has_more"]: # type: ignore
logger.debug("Next cursor: %s", search_result["next_cursor"]) # type: ignore
search_result = notion_client.search(
query="",
filter={"property": "object", "value": "page"},
sort={"direction": "descending", "timestamp": "last_edited_time"},
start_cursor=search_result["next_cursor"], # type: ignore
)
end_sync_time = datetime.strptime(
search_result["results"][-1]["last_edited_time"],
"%Y-%m-%dT%H:%M:%S.%fZ", # type: ignore
)
if notion_sync:
for page in search_result["results"]:
last_edited_time = datetime.strptime(
page["last_edited_time"], "%Y-%m-%dT%H:%M:%S.%fZ"
)
if last_edited_time > last_sync_time:
all_search_result.append(page)
if last_edited_time and last_edited_time < last_sync_time:
# We check if the last element of the search result is older than 6 hours, if it is, we stop the search
return all_search_result
return all_search_result

7
backend/worker/quivr_worker/syncs/store_notion.py
View File

@ -1,3 +1,4 @@
from datetime import datetime
from uuid import UUID
from notion_client import Client
@ -23,7 +24,11 @@ async def fetch_and_store_notion_files_async(
notion_repository = NotionRepository(session)
notion_service = SyncNotionService(notion_repository)
notion_client = Client(auth=access_token)
all_search_result = fetch_notion_pages(notion_client)
all_search_result = fetch_notion_pages(
notion_client,
last_sync_time=datetime(1970, 1, 1, 0, 0, 0), # UNIX EPOCH
)
logger.debug(f"Notion fetched {len(all_search_result)} pages")
pages = await store_notion_pages(all_search_result, notion_service, user_id)
if pages:
logger.info(f"stored {len(pages)} from notion for {user_id}")