mirror of
https://github.com/QuivrHQ/quivr.git
synced 2024-09-11 14:36:35 +03:00
notion sync wip
This commit is contained in:
parent
fe366d82d5
commit
f0acc92a70
@ -137,7 +137,7 @@ def oauth2callback_notion(request: Request, background_tasks: BackgroundTasks):
|
||||
account_id = owner_info["id"]
|
||||
|
||||
result: dict[str, str] = {
|
||||
"access_token": oauth_result["access_token"],
|
||||
"access_token": access_token,
|
||||
"refresh_token": oauth_result.get("refresh_token", ""),
|
||||
"account_id": account_id,
|
||||
"expires_in": oauth_result.get("expires_in", ""),
|
||||
|
@ -37,7 +37,9 @@ class SyncNotionService(BaseService[NotionRepository]):
|
||||
else None
|
||||
),
|
||||
name=f'{page["properties"]["title"]["title"][0]["text"]["content"]}.md',
|
||||
icon=page["icon"]["emoji"] if page["icon"] else None,
|
||||
icon=page["icon"]["emoji"]
|
||||
if "icon" in page and "emoji" in page["icon"]
|
||||
else None,
|
||||
mime_type="md",
|
||||
web_view_link=page["url"],
|
||||
is_folder=True,
|
||||
@ -77,7 +79,9 @@ class SyncNotionService(BaseService[NotionRepository]):
|
||||
notion_id=page["id"],
|
||||
parent_id=page["parent"][parent_type],
|
||||
name=f'{page["properties"]["title"]["title"][0]["text"]["content"]}.md',
|
||||
icon=page["icon"]["emoji"] if page["icon"] else None,
|
||||
icon=page["icon"]["emoji"]
|
||||
if "icon" in page and "emoji" in page["icon"]
|
||||
else None,
|
||||
mime_type="md",
|
||||
web_view_link=page["url"],
|
||||
is_folder=True,
|
||||
@ -106,9 +110,9 @@ class SyncNotionService(BaseService[NotionRepository]):
|
||||
or child_notion_page["in_trash"]
|
||||
):
|
||||
pages_to_delete.append(child.notion_id)
|
||||
except:
|
||||
logger.info(
|
||||
f"Page {child.notion_id} is in trash or archived, we are deleting it"
|
||||
except Exception:
|
||||
logger.error(
|
||||
f"Page {child.notion_id} is in trash or archived, we are deleting it."
|
||||
)
|
||||
pages_to_delete.append(child.notion_id)
|
||||
|
||||
@ -186,17 +190,38 @@ async def store_notion_pages(
|
||||
|
||||
|
||||
def fetch_notion_pages(
|
||||
notion_client: Client, notion_sync: dict[str, Any] | None = None
|
||||
notion_client: Client,
|
||||
last_sync_time: datetime = datetime.now() - timedelta(hours=6),
|
||||
) -> List[dict[str, Any]]:
|
||||
all_search_result = []
|
||||
last_sync_time = datetime.now() - timedelta(hours=6)
|
||||
|
||||
search_result = notion_client.search(
|
||||
query="",
|
||||
filter={"property": "object", "value": "page"},
|
||||
sort={"direction": "descending", "timestamp": "last_edited_time"},
|
||||
)
|
||||
last_edited_time: datetime | None = None
|
||||
if notion_sync:
|
||||
|
||||
for page in search_result["results"]:
|
||||
last_edited_time = datetime.strptime(
|
||||
page["last_edited_time"], "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
)
|
||||
if last_edited_time > last_sync_time:
|
||||
all_search_result.append(page)
|
||||
|
||||
if last_edited_time and last_edited_time < last_sync_time:
|
||||
# We check if the last element of the search result is older than 6 hours, if it is, we stop the search
|
||||
return all_search_result
|
||||
|
||||
while search_result["has_more"]: # type: ignore
|
||||
logger.debug("Next cursor: %s", search_result["next_cursor"]) # type: ignore
|
||||
|
||||
search_result = notion_client.search(
|
||||
query="",
|
||||
filter={"property": "object", "value": "page"},
|
||||
sort={"direction": "descending", "timestamp": "last_edited_time"},
|
||||
start_cursor=search_result["next_cursor"], # type: ignore
|
||||
)
|
||||
for page in search_result["results"]:
|
||||
last_edited_time = datetime.strptime(
|
||||
page["last_edited_time"], "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
@ -208,29 +233,4 @@ def fetch_notion_pages(
|
||||
# We check if the last element of the search result is older than 6 hours, if it is, we stop the search
|
||||
return all_search_result
|
||||
|
||||
while search_result["has_more"]: # type: ignore
|
||||
logger.debug("Next cursor: %s", search_result["next_cursor"]) # type: ignore
|
||||
|
||||
search_result = notion_client.search(
|
||||
query="",
|
||||
filter={"property": "object", "value": "page"},
|
||||
sort={"direction": "descending", "timestamp": "last_edited_time"},
|
||||
start_cursor=search_result["next_cursor"], # type: ignore
|
||||
)
|
||||
end_sync_time = datetime.strptime(
|
||||
search_result["results"][-1]["last_edited_time"],
|
||||
"%Y-%m-%dT%H:%M:%S.%fZ", # type: ignore
|
||||
)
|
||||
if notion_sync:
|
||||
for page in search_result["results"]:
|
||||
last_edited_time = datetime.strptime(
|
||||
page["last_edited_time"], "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
)
|
||||
if last_edited_time > last_sync_time:
|
||||
all_search_result.append(page)
|
||||
|
||||
if last_edited_time and last_edited_time < last_sync_time:
|
||||
# We check if the last element of the search result is older than 6 hours, if it is, we stop the search
|
||||
return all_search_result
|
||||
|
||||
return all_search_result
|
||||
|
@ -1,3 +1,4 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from notion_client import Client
|
||||
@ -23,7 +24,11 @@ async def fetch_and_store_notion_files_async(
|
||||
notion_repository = NotionRepository(session)
|
||||
notion_service = SyncNotionService(notion_repository)
|
||||
notion_client = Client(auth=access_token)
|
||||
all_search_result = fetch_notion_pages(notion_client)
|
||||
all_search_result = fetch_notion_pages(
|
||||
notion_client,
|
||||
last_sync_time=datetime(1970, 1, 1, 0, 0, 0), # UNIX EPOCH
|
||||
)
|
||||
logger.debug(f"Notion fetched {len(all_search_result)} pages")
|
||||
pages = await store_notion_pages(all_search_result, notion_service, user_id)
|
||||
if pages:
|
||||
logger.info(f"stored {len(pages)} from notion for {user_id}")
|
||||
|
Loading…
Reference in New Issue
Block a user