QuivrHQ/quivr
1
1
Fork 0
mirror of https://github.com/QuivrHQ/quivr.git synced 2024-12-14 17:03:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix: added chunk_size in tika processor (#3466)

Browse Source
This commit is contained in:
AmineDiro 2024-11-09 15:34:12 +01:00 committed by GitHub
parent 190d971bd7
commit 063bbd323d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/quivr_core/processor/implementations/tika_processor.py
View File

@ -1,3 +1,4 @@
import tiktoken
import logging import logging
import os import os
from typing import AsyncIterable from typing import AsyncIterable
@ -39,6 +40,7 @@ class TikaProcessor(ProcessorBase):
self.max_retries = max_retries self.max_retries = max_retries
self._client = httpx.AsyncClient(timeout=timeout) self._client = httpx.AsyncClient(timeout=timeout)
self.enc = tiktoken.get_encoding("cl100k_base")
self.splitter_config = splitter_config self.splitter_config = splitter_config
if splitter: if splitter:
@ -73,5 +75,7 @@ class TikaProcessor(ProcessorBase):
txt = await self._send_parse_tika(f) txt = await self._send_parse_tika(f)
document = Document(page_content=txt) document = Document(page_content=txt)
docs = self.text_splitter.split_documents([document]) docs = self.text_splitter.split_documents([document])
for doc in docs:
doc.metadata = {"chunk_size": len(self.enc.encode(doc.page_content))}
return docs return docs