mirror of
https://github.com/QuivrHQ/quivr.git
synced 2024-08-17 02:10:31 +03:00
feat(RAGAS): added one test case
This commit is contained in:
Stan Girard
parent
2172fe47f5
commit
b2617afbbc
1
.gitignore
vendored
1
.gitignore
vendored
@ -82,3 +82,4 @@ application.log
|
||||
backend/celerybeat-schedule.db
|
||||
|
||||
backend/application.log.*
|
||||
backend/score.json
|
||||
|
||||
2
Pipfile
2
Pipfile
@ -55,6 +55,8 @@ unstructured = {extras = ["all-docs"], version = "*"}
|
||||
llama-parse = "*"
|
||||
llama-index = "*"
|
||||
lxml = {extras = ["html_clean"], version = "*"}
|
||||
ragas = "*"
|
||||
datasets = "*"
|
||||
|
||||
[dev-packages]
|
||||
black = "*"
|
||||
|
||||
465
Pipfile.lock
generated
465
Pipfile.lock
generated
@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "3cd313bc27c5b87ecc1455049348b02b69e1d6d6b9a5499ed4d9779e30bc8fc2"
|
||||
"sha256": "0f19a2edadec711ea3586769d93663f24ddcae77a66f0f5bd4cb7c4ea43b59f1"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@ -136,6 +136,13 @@
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==4.3.0"
|
||||
},
|
||||
"appdirs": {
|
||||
"hashes": [
|
||||
"sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
|
||||
"sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
|
||||
],
|
||||
"version": "==1.4.4"
|
||||
},
|
||||
"astor": {
|
||||
"hashes": [
|
||||
"sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5",
|
||||
@ -244,49 +251,49 @@
|
||||
},
|
||||
"black": {
|
||||
"hashes": [
|
||||
"sha256:2818cf72dfd5d289e48f37ccfa08b460bf469e67fb7c4abb07edc2e9f16fb63f",
|
||||
"sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93",
|
||||
"sha256:4acf672def7eb1725f41f38bf6bf425c8237248bb0804faa3965c036f7672d11",
|
||||
"sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0",
|
||||
"sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9",
|
||||
"sha256:56f52cfbd3dabe2798d76dbdd299faa046a901041faf2cf33288bc4e6dae57b5",
|
||||
"sha256:65b76c275e4c1c5ce6e9870911384bff5ca31ab63d19c76811cb1fb162678213",
|
||||
"sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d",
|
||||
"sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7",
|
||||
"sha256:79dcf34b33e38ed1b17434693763301d7ccbd1c5860674a8f871bd15139e7837",
|
||||
"sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f",
|
||||
"sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395",
|
||||
"sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995",
|
||||
"sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f",
|
||||
"sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597",
|
||||
"sha256:b5991d523eee14756f3c8d5df5231550ae8993e2286b8014e2fdea7156ed0959",
|
||||
"sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5",
|
||||
"sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb",
|
||||
"sha256:c7ed6668cbbfcd231fa0dc1b137d3e40c04c7f786e626b405c62bcd5db5857e4",
|
||||
"sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7",
|
||||
"sha256:e19cb1c6365fd6dc38a6eae2dcb691d7d83935c10215aef8e6c38edee3f77abd",
|
||||
"sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"
|
||||
"sha256:1bb9ca06e556a09f7f7177bc7cb604e5ed2d2df1e9119e4f7d2f1f7071c32e5d",
|
||||
"sha256:21f9407063ec71c5580b8ad975653c66508d6a9f57bd008bb8691d273705adcd",
|
||||
"sha256:4396ca365a4310beef84d446ca5016f671b10f07abdba3e4e4304218d2c71d33",
|
||||
"sha256:44d99dfdf37a2a00a6f7a8dcbd19edf361d056ee51093b2445de7ca09adac965",
|
||||
"sha256:5cd5b4f76056cecce3e69b0d4c228326d2595f506797f40b9233424e2524c070",
|
||||
"sha256:64578cf99b6b46a6301bc28bdb89f9d6f9b592b1c5837818a177c98525dbe397",
|
||||
"sha256:64e60a7edd71fd542a10a9643bf369bfd2644de95ec71e86790b063aa02ff745",
|
||||
"sha256:652e55bb722ca026299eb74e53880ee2315b181dfdd44dca98e43448620ddec1",
|
||||
"sha256:6644f97a7ef6f401a150cca551a1ff97e03c25d8519ee0bbc9b0058772882665",
|
||||
"sha256:6ad001a9ddd9b8dfd1b434d566be39b1cd502802c8d38bbb1ba612afda2ef436",
|
||||
"sha256:71d998b73c957444fb7c52096c3843875f4b6b47a54972598741fe9a7f737fcb",
|
||||
"sha256:74eb9b5420e26b42c00a3ff470dc0cd144b80a766128b1771d07643165e08d0e",
|
||||
"sha256:75a2d0b4f5eb81f7eebc31f788f9830a6ce10a68c91fbe0fade34fff7a2836e6",
|
||||
"sha256:7852b05d02b5b9a8c893ab95863ef8986e4dda29af80bbbda94d7aee1abf8702",
|
||||
"sha256:7f2966b9b2b3b7104fca9d75b2ee856fe3fdd7ed9e47c753a4bb1a675f2caab8",
|
||||
"sha256:8e5537f456a22cf5cfcb2707803431d2feeb82ab3748ade280d6ccd0b40ed2e8",
|
||||
"sha256:d4e71cdebdc8efeb6deaf5f2deb28325f8614d48426bed118ecc2dcaefb9ebf3",
|
||||
"sha256:dae79397f367ac8d7adb6c779813328f6d690943f64b32983e896bcccd18cbad",
|
||||
"sha256:e3a3a092b8b756c643fe45f4624dbd5a389f770a4ac294cf4d0fce6af86addaf",
|
||||
"sha256:eb949f56a63c5e134dfdca12091e98ffb5fd446293ebae123d10fc1abad00b9e",
|
||||
"sha256:f07b69fda20578367eaebbd670ff8fc653ab181e1ff95d84497f9fa20e7d0641",
|
||||
"sha256:f95cece33329dc4aa3b0e1a771c41075812e46cf3d6e3f1dfe3d91ff09826ed2"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==24.3.0"
|
||||
"version": "==24.4.0"
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:33cf93f6de5176f1188c923f4de1ae149ed723b89ed12e434f2b2f628491769e",
|
||||
"sha256:9733ce811bd82feab506ad9309e375a79cabe8c6149061971c17754ce8997551"
|
||||
"sha256:7a02f44af32095946587d748ebeb39c3fa15b9d7275307ff612a6760ead47e04",
|
||||
"sha256:91e6343474173e9b82f603076856e1d5b7b68f44247bdd556250857a3f16b37b"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.34.83"
|
||||
"version": "==1.34.84"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:0a3fbbe018416aeefa8978454fb0b8129adbaf556647b72269bf02e4bf1f4161",
|
||||
"sha256:0f302aa76283d4df62b4fbb6d3d20115c1a8957fc02171257fc93904d69d5636"
|
||||
"sha256:a2b309bf5594f0eb6f63f355ade79ba575ce8bf672e52e91da1a7933caa245e6",
|
||||
"sha256:da1ae0a912e69e10daee2a34dafd6c6c106450d20b8623665feceb2d96c173eb"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.34.83"
|
||||
"version": "==1.34.84"
|
||||
},
|
||||
"celery": {
|
||||
"extras": [
|
||||
@ -638,6 +645,15 @@
|
||||
"markers": "python_version >= '3.7' and python_version < '4.0'",
|
||||
"version": "==0.5.11"
|
||||
},
|
||||
"datasets": {
|
||||
"hashes": [
|
||||
"sha256:cdf8b8c6abf7316377ba4f49f9589a4c74556d6b481afd0abd2284f3d69185cb",
|
||||
"sha256:f1bbf0e2896917a914de01cbd37075b14deea3837af87ad0d9f697388ccaeb50"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_full_version >= '3.8.0'",
|
||||
"version": "==2.18.0"
|
||||
},
|
||||
"debugpy": {
|
||||
"hashes": [
|
||||
"sha256:016a9fcfc2c6b57f939673c874310d8581d51a0fe0858e7fac4e240c5eb743cb",
|
||||
@ -689,6 +705,14 @@
|
||||
],
|
||||
"version": "==2.1.0"
|
||||
},
|
||||
"dill": {
|
||||
"hashes": [
|
||||
"sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca",
|
||||
"sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==0.3.8"
|
||||
},
|
||||
"dirtyjson": {
|
||||
"hashes": [
|
||||
"sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53",
|
||||
@ -1015,12 +1039,15 @@
|
||||
"version": "==1.4.1"
|
||||
},
|
||||
"fsspec": {
|
||||
"extras": [
|
||||
"http"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:918d18d41bf73f0e2b261824baeb1b124bcf771767e3a26425cd7dec3332f512",
|
||||
"sha256:f39780e282d7d117ffb42bb96992f8a90795e4d0fb0f661a70ca39fe9c43ded9"
|
||||
"sha256:817f969556fa5916bc682e02ca2045f96ff7f586d45110fcb76022063ad2c7d8",
|
||||
"sha256:b6ad1a679f760dda52b1168c859d01b7b80648ea6f7f7c7f5a8a91dc3f3ecb84"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==2024.3.1"
|
||||
"version": "==2024.2.0"
|
||||
},
|
||||
"gitdb": {
|
||||
"hashes": [
|
||||
@ -1505,20 +1532,20 @@
|
||||
},
|
||||
"langfuse": {
|
||||
"hashes": [
|
||||
"sha256:74ddb34303f43b5c6a7b1feedb8b4722bc16f050f1ba329a68da2319e2a1b618",
|
||||
"sha256:931c70b26f14b948faa4d668c93834ac9d23b4c2598b8eb4be1118aa7538afdd"
|
||||
"sha256:a95aab75d483614eb7ad8fd9f6e54078479c805e39098980caa676d9d42db54f",
|
||||
"sha256:cbfb03028df42d5e1894860d889a5cdf7cbfbf56ef9deb974c6691a08c689692"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
|
||||
"version": "==2.25.0"
|
||||
"version": "==2.26.1"
|
||||
},
|
||||
"langsmith": {
|
||||
"hashes": [
|
||||
"sha256:5a5b7fafe767fa28826c925f175875c09bf5368bfdb141286381a94bf737e6ef",
|
||||
"sha256:713206107df636db1edf30867d64b92495afb1f09d2fee0857a77b7a8ee083d5"
|
||||
"sha256:17b0a908b8d39b6da3ecff658c8c00304b0b62f59945a5e16c2da5a254ea21a6",
|
||||
"sha256:f5ddd17628baa03a775525c5547a543a559313e425cdb2bf23579ffcf6056a76"
|
||||
],
|
||||
"markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
|
||||
"version": "==0.1.45"
|
||||
"version": "==0.1.47"
|
||||
},
|
||||
"layoutparser": {
|
||||
"extras": [
|
||||
@ -1534,21 +1561,21 @@
|
||||
},
|
||||
"litellm": {
|
||||
"hashes": [
|
||||
"sha256:062e5be75196da7348ae0c4f60d396f0b23ee874708ed81c40f7675161213385",
|
||||
"sha256:686ee040154d7062b0078d882fa6399c5c7cc5ec9b5266490dee68f1b8905a36"
|
||||
"sha256:4545d4af8cd177c1539968335920253ad44c38e09420854f142ffff9cdb3fff7",
|
||||
"sha256:ffc7ad5550534f737097e8e33cef575dc806e5ff4c624aaec2b400e28c2b4ea3"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8'",
|
||||
"version": "==1.35.2"
|
||||
"version": "==1.35.5"
|
||||
},
|
||||
"llama-index": {
|
||||
"hashes": [
|
||||
"sha256:2ecf9d6da489cc3f99b1adaea50b1549efa55e04cd050f332da784a87c2db846",
|
||||
"sha256:2fe2857af6c2d9cefdd978931ebd09f78265a57df2d9d094d0c78869dd0715b8"
|
||||
"sha256:df61c04c05ddf0488aa87085c464e52ff215338933de771de46e2e4965719eaa",
|
||||
"sha256:f83b826007a8b4570a165f5db8ce770cf186adf7686ec0ddbeccf1887fcd5394"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
|
||||
"version": "==0.10.28"
|
||||
"version": "==0.10.29"
|
||||
},
|
||||
"llama-index-agent-openai": {
|
||||
"hashes": [
|
||||
@ -1568,11 +1595,11 @@
|
||||
},
|
||||
"llama-index-core": {
|
||||
"hashes": [
|
||||
"sha256:73e2ec86ea42a0efdc73dcfe46e03ba39ab7725699dcd791464c160d24a02cb1",
|
||||
"sha256:b454640396daa2678b638299e2a5e2539fa3548afd280d45d224fd3b303c20f4"
|
||||
"sha256:105eace45d877f31ff330e9afe7e86bd212b776b96d594862f24c60e03ab99b9",
|
||||
"sha256:65188c9917ec0847005ebd4343d7eaf9ddc80cea50d876a02eccbb16f1684371"
|
||||
],
|
||||
"markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
|
||||
"version": "==0.10.28"
|
||||
"version": "==0.10.29"
|
||||
},
|
||||
"llama-index-embeddings-openai": {
|
||||
"hashes": [
|
||||
@ -1632,11 +1659,11 @@
|
||||
},
|
||||
"llama-index-readers-file": {
|
||||
"hashes": [
|
||||
"sha256:1a3f3bc1f09c6894318a5392edc43b118fabc2f2820f446c51cad73676662f73",
|
||||
"sha256:7f25b8fddaf1a91bac06c66727b913b48e26ef68714fcd5197051b3f710f4cfa"
|
||||
"sha256:725fbce501eb7bdbb630b625e88985c51d55b99006b8b3342af4cefcb49bf51e",
|
||||
"sha256:91c9871e80d7c911a4e2bb265ee46c3db6a93d681ba6ddf731752cd7764b368b"
|
||||
],
|
||||
"markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
|
||||
"version": "==0.1.16"
|
||||
"version": "==0.1.18"
|
||||
},
|
||||
"llama-index-readers-llama-parse": {
|
||||
"hashes": [
|
||||
@ -1648,20 +1675,20 @@
|
||||
},
|
||||
"llama-parse": {
|
||||
"hashes": [
|
||||
"sha256:52b450c9bb94746c5f86867f54fd2e74670d89b385dcb879eb93490a42fda628",
|
||||
"sha256:ee521e9422db48dbdf00303cb6e90f1066131960b5ca4e14f5b5a6db6b7917e2"
|
||||
"sha256:2c08962b66791c61fc360ae2042f953729c7b8decc3590d01fea5a98ca1f6676",
|
||||
"sha256:d723af84d6a1fc99eb431915d21865d20b76d8a246dbaa124d1f96c956a644f7"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version < '4.0' and python_full_version >= '3.8.1'",
|
||||
"version": "==0.4.0"
|
||||
"version": "==0.4.1"
|
||||
},
|
||||
"llamaindex-py-client": {
|
||||
"hashes": [
|
||||
"sha256:979d7c8307442f4208e2793f8e8f896827d4ac315af2df51c27bedbeb7ba8f07",
|
||||
"sha256:cbfa65006abe248a6c17bfb3703f884d1d395c3609583ff60783b43ab5f7b0d6"
|
||||
"sha256:091ee49a92592e3894777ade12516c2137093f9d6441a549f406461917ce9b7e",
|
||||
"sha256:5417e41666504a77ecf5bdd9b403ffff1d714880ee30d49e234fb7686177eeeb"
|
||||
],
|
||||
"markers": "python_version >= '3.8' and python_version < '4'",
|
||||
"version": "==0.1.17"
|
||||
"version": "==0.1.18"
|
||||
},
|
||||
"lxml": {
|
||||
"extras": [
|
||||
@ -2074,6 +2101,24 @@
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==6.0.5"
|
||||
},
|
||||
"multiprocess": {
|
||||
"hashes": [
|
||||
"sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41",
|
||||
"sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1",
|
||||
"sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a",
|
||||
"sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee",
|
||||
"sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3",
|
||||
"sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435",
|
||||
"sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a",
|
||||
"sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054",
|
||||
"sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02",
|
||||
"sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec",
|
||||
"sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a",
|
||||
"sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==0.70.16"
|
||||
},
|
||||
"mypy-extensions": {
|
||||
"hashes": [
|
||||
"sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d",
|
||||
@ -2247,12 +2292,12 @@
|
||||
},
|
||||
"openai": {
|
||||
"hashes": [
|
||||
"sha256:72464cdb0602a57af87acb4888b1a48a1c02182cc9f09d2f2f3200b185223d5f",
|
||||
"sha256:72e6758cec080a3e5a9daf843178c975fed656fe0831919f4dd89bb62431724f"
|
||||
"sha256:c5a909e70e2b4cd04ef4076b68497c4eb894704cf8ab23d32d6552931fda6e5b",
|
||||
"sha256:d42e0d7a692c7b78cbae21598df5ded396abecf6c285325635ec62621e6c82f1"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_full_version >= '3.7.1'",
|
||||
"version": "==1.17.0"
|
||||
"version": "==1.17.1"
|
||||
},
|
||||
"opencv-python": {
|
||||
"hashes": [
|
||||
@ -2609,11 +2654,11 @@
|
||||
},
|
||||
"postgrest": {
|
||||
"hashes": [
|
||||
"sha256:6c5c8e53cdcede8b6654ddfc7505e5af0c41ce56c6935f7b1d05545bb899d8b8",
|
||||
"sha256:cf89106d0877ac2c7b070ad136f78350eb89dbdd998cd83d6852010e0bcdb878"
|
||||
"sha256:30c8fb54fd37cec929531fc43d05e12df318830f572a1b93491411fe411c8cbd",
|
||||
"sha256:fd3f4646d17cf5321049d00b7d0cdea5e84285cb28bd6acdab99487081f68794"
|
||||
],
|
||||
"markers": "python_version >= '3.8' and python_version < '4.0'",
|
||||
"version": "==0.16.2"
|
||||
"version": "==0.16.3"
|
||||
},
|
||||
"posthog": {
|
||||
"hashes": [
|
||||
@ -2785,6 +2830,56 @@
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
|
||||
"version": "==1.11.0"
|
||||
},
|
||||
"pyarrow": {
|
||||
"hashes": [
|
||||
"sha256:033b7cad32198754d93465dcfb71d0ba7cb7cd5c9afd7052cab7214676eec38b",
|
||||
"sha256:06c2bb2a98bc792f040bef31ad3e9be6a63d0cb39189227c08a7d955db96816e",
|
||||
"sha256:23c6753ed4f6adb8461e7c383e418391b8d8453c5d67e17f416c3a5d5709afbd",
|
||||
"sha256:248723e4ed3255fcd73edcecc209744d58a9ca852e4cf3d2577811b6d4b59818",
|
||||
"sha256:25335e6f1f07fdaa026a61c758ee7d19ce824a866b27bba744348fa73bb5a440",
|
||||
"sha256:28f3016958a8e45a1069303a4a4f6a7d4910643fc08adb1e2e4a7ff056272ad3",
|
||||
"sha256:290e36a59a0993e9a5224ed2fb3e53375770f07379a0ea03ee2fce2e6d30b423",
|
||||
"sha256:29850d050379d6e8b5a693098f4de7fd6a2bea4365bfd073d7c57c57b95041ee",
|
||||
"sha256:2d4f905209de70c0eb5b2de6763104d5a9a37430f137678edfb9a675bac9cd98",
|
||||
"sha256:3a4f240852b302a7af4646c8bfe9950c4691a419847001178662a98915fd7ee7",
|
||||
"sha256:3e6d459c0c22f0b9c810a3917a1de3ee704b021a5fb8b3bacf968eece6df098f",
|
||||
"sha256:3ff3bdfe6f1b81ca5b73b70a8d482d37a766433823e0c21e22d1d7dde76ca33f",
|
||||
"sha256:4e7d9cfb5a1e648e172428c7a42b744610956f3b70f524aa3a6c02a448ba853e",
|
||||
"sha256:58922e4bfece8b02abf7159f1f53a8f4d9f8e08f2d988109126c17c3bb261f22",
|
||||
"sha256:5f8bc839ea36b1f99984c78e06e7a06054693dc2af8920f6fb416b5bca9944e4",
|
||||
"sha256:6669799a1d4ca9da9c7e06ef48368320f5856f36f9a4dd31a11839dda3f6cc8c",
|
||||
"sha256:7167107d7fb6dcadb375b4b691b7e316f4368f39f6f45405a05535d7ad5e5058",
|
||||
"sha256:88b340f0a1d05b5ccc3d2d986279045655b1fe8e41aba6ca44ea28da0d1455d8",
|
||||
"sha256:89722cb64286ab3d4daf168386f6968c126057b8c7ec3ef96302e81d8cdb8ae4",
|
||||
"sha256:8bd2baa5fe531571847983f36a30ddbf65261ef23e496862ece83bdceb70420d",
|
||||
"sha256:8c1faf2482fb89766e79745670cbca04e7018497d85be9242d5350cba21357e1",
|
||||
"sha256:90adb99e8ce5f36fbecbbc422e7dcbcbed07d985eed6062e459e23f9e71fd197",
|
||||
"sha256:90f19e976d9c3d8e73c80be84ddbe2f830b6304e4c576349d9360e335cd627fc",
|
||||
"sha256:9c9bc803cb3b7bfacc1e96ffbfd923601065d9d3f911179d81e72d99fd74a3d9",
|
||||
"sha256:a22366249bf5fd40ddacc4f03cd3160f2d7c247692945afb1899bab8a140ddfb",
|
||||
"sha256:ad2459bf1f22b6a5cdcc27ebfd99307d5526b62d217b984b9f5c974651398832",
|
||||
"sha256:adccc81d3dc0478ea0b498807b39a8d41628fa9210729b2f718b78cb997c7c91",
|
||||
"sha256:b116e7fd7889294cbd24eb90cd9bdd3850be3738d61297855a71ac3b8124ee38",
|
||||
"sha256:c2a335198f886b07e4b5ea16d08ee06557e07db54a8400cc0d03c7f6a22f785f",
|
||||
"sha256:cd0ba387705044b3ac77b1b317165c0498299b08261d8122c96051024f953cd5",
|
||||
"sha256:e85241b44cc3d365ef950432a1b3bd44ac54626f37b2e3a0cc89c20e45dfd8bf",
|
||||
"sha256:eaa8f96cecf32da508e6c7f69bb8401f03745c050c1dd42ec2596f2e98deecac",
|
||||
"sha256:f3d77463dee7e9f284ef42d341689b459a63ff2e75cee2b9302058d0d98fe142",
|
||||
"sha256:f5e81dfb4e519baa6b4c80410421528c214427e77ca0ea9461eb4097c328fa33",
|
||||
"sha256:f639c059035011db8c0497e541a8a45d98a58dbe34dc8fadd0ef128f2cee46e5",
|
||||
"sha256:f7a197f3670606a960ddc12adbe8075cea5f707ad7bf0dffa09637fdbb89f76c"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==15.0.2"
|
||||
},
|
||||
"pyarrow-hotfix": {
|
||||
"hashes": [
|
||||
"sha256:79d3e030f7ff890d408a100ac16d6f00b14d44a502d7897cd9fc3e3a534e9945",
|
||||
"sha256:dcc9ae2d220dff0083be6a9aa8e0cdee5182ad358d4931fce825c545e5c89178"
|
||||
],
|
||||
"markers": "python_version >= '3.5'",
|
||||
"version": "==0.6"
|
||||
},
|
||||
"pyasn1": {
|
||||
"hashes": [
|
||||
"sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c",
|
||||
@ -2986,55 +3081,6 @@
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"pymupdf": {
|
||||
"hashes": [
|
||||
"sha256:02a6586979df2ad958b524ba42955beaa67fd21661616a0ed04ac07db009474c",
|
||||
"sha256:0edda1024ada67603e5888f31656048d3fd53167c8b0d56f435b986eb507df8f",
|
||||
"sha256:12078c0bee337de969dbd6d89ef446312794d74db365cb9ac14902b863b35414",
|
||||
"sha256:38728bb6aab9e3879aa8ac4d337be8fe838d33973f43e3b7805b86265c24f349",
|
||||
"sha256:38e6101dab2ff86c4e2444fcec8a04377ae1d6db1bef0f7a1ddab3ac6abe4d41",
|
||||
"sha256:468a8bb2b95828e0f6739fbfe509700cc0dac600f756d8cb6316316e1eba9689",
|
||||
"sha256:6427aee313e24447f57edbfc7a28aa6bbca007fe0ad77603f54a371c6c510eeb",
|
||||
"sha256:6832f1d9332810760b587ad375eb84d64ec8d8f29395995b463cb5f30533a413",
|
||||
"sha256:6952d47f0f05cf9338470dda078e4533ddb876368b199ebfa2f9e6076311898b",
|
||||
"sha256:6d412da9f9a73f66973eea4284776f292135906700a06c39122e862a1e3ccf58",
|
||||
"sha256:73d07e127936948a29a7dbd4c831e9eb45a60b495d72e604d454fd040fd08c5f",
|
||||
"sha256:73f86eefd7f3878f112fa10791aa2e63934cf59a4c024dd54cd6fe94443c352c",
|
||||
"sha256:7df966954ff0edbcd5d743c5f6fb68b3203e67534747e8753691b8ffedeaa518",
|
||||
"sha256:825c62367b01e61b4bce0cc96d45b0ec336475422cfa36de6f441b4d3389a26e",
|
||||
"sha256:8489df092473d590fb14903433bd99a07dc3d2924f5a5c8ead615795f2d65a65",
|
||||
"sha256:8a81106a8bc229823736487d2492fd3af724a94521a1cd9b67849dd04b9c31ed",
|
||||
"sha256:8eb292d16671166acdaa280e98cac4368298f32556f2de2ee690782a635df8ee",
|
||||
"sha256:95a54611abb7322f5b10b44cbf19b605ed172df2c4c7995ad78854bc8423dd9c",
|
||||
"sha256:9a3b21c8fc274ff42855ca2da65961e2319b05b75ef9e2caf25c04f9083ec79c",
|
||||
"sha256:b8a5247d0cec87765481c38d2b8602f0264bf7ca6b5dc3013caf64ce46ad4d5e",
|
||||
"sha256:c54ff927257b432ffd39dc6a0a46bd1120e85d192100efca021f27d4b881cfd6",
|
||||
"sha256:caf6ceb1dbebe9f70bf7dd683cc91b896604a7c62873e5b50089f9e85e85c517",
|
||||
"sha256:d1078ea265635e962693d7298bd39be64af7d1dd2c6dc663a8562e75f547f948",
|
||||
"sha256:d2b4f8956d0ca7564604491db8b29cd7872a2b4d65f1d7e16a1bccfecf84bb56",
|
||||
"sha256:dbc5d67dfd07123293993eb93bee35d329fce0bc8134b9cd5514ef75c68ffee8",
|
||||
"sha256:de5b6c4db4a2a9f28937e79135f732827c424f7444c12767cc1081c8006f0430",
|
||||
"sha256:e3f7a101a14d742c93b660b7586ab4c1491caea9062a5de9c308578a7a4f8b69",
|
||||
"sha256:e47504391908e2d721c743aed36196310a5e15355a85459c1c4ddcf8f2002fbe",
|
||||
"sha256:ee9cfac470aeb6b5b7deb4f6472b7796c3132856849c635c8e56c7a371e40238",
|
||||
"sha256:f775bb56391629e81b5f870fc3dec0a0fb44cb34a92b4696b9207b31234711df",
|
||||
"sha256:f7b7f2011fa522a57fb3d6a7a58bcdcf01ee59bdad536ef9eb5c3fdf1e04e6c3"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.24.1"
|
||||
},
|
||||
"pymupdfb": {
|
||||
"hashes": [
|
||||
"sha256:01c8b7f0ce9166310eb28c7aebcb8d5fe12a4bc082f9b00d580095eebeaf0af5",
|
||||
"sha256:0fbcc0d2a9ce79fa38eb4e8bb5c959b582f7a49938874e9f61d1a6f5eeb1e4b8",
|
||||
"sha256:17444ea7d6897c27759880ad76af537d19779f901de82ae9548598a70f614558",
|
||||
"sha256:37179e363bf69ce9be637937c5469957b96968341dabe3ce8f4b690a82e9ad92",
|
||||
"sha256:490f7fff4dbe362bc895cefdfc5030d712311d024d357a1388d64816eb215d34",
|
||||
"sha256:ae67736058882cdd9459810a4aae9ac2b2e89ac2e916cb5fefb0f651c9739e9e"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.24.1"
|
||||
},
|
||||
"pypandoc": {
|
||||
"hashes": [
|
||||
"sha256:31652073c7960c2b03570bd1e94f602ca9bc3e70099df5ead4cea98ff5151c1e",
|
||||
@ -3089,6 +3135,13 @@
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.1.358"
|
||||
},
|
||||
"pysbd": {
|
||||
"hashes": [
|
||||
"sha256:cd838939b7b0b185fcf86b0baf6636667dfb6e474743beeff878e9f42e022953"
|
||||
],
|
||||
"markers": "python_version >= '3'",
|
||||
"version": "==0.3.4"
|
||||
},
|
||||
"pytesseract": {
|
||||
"hashes": [
|
||||
"sha256:8f22cc98f765bf13517ead0c70effedb46c153540d25783e04014f28b55a5fc6",
|
||||
@ -3261,6 +3314,14 @@
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==6.0.1"
|
||||
},
|
||||
"ragas": {
|
||||
"hashes": [
|
||||
"sha256:abe02b40a8d11842c42e222226901287858beb70203f1227a403a9261d0bb684",
|
||||
"sha256:db857262dda63fc01a7eef837cbba166202084b5d535b2e8ad408c63a66f9319"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.1.7"
|
||||
},
|
||||
"rapidfuzz": {
|
||||
"hashes": [
|
||||
"sha256:00b5ee47b387fa3805f4038362a085ec58149135dc5bc640ca315a9893a16f9e",
|
||||
@ -3359,11 +3420,11 @@
|
||||
},
|
||||
"realtime": {
|
||||
"hashes": [
|
||||
"sha256:1a39b5dcdb345b4cc7fd43bc035feb38ca915c9248962f20d264625bc8eb2c4e",
|
||||
"sha256:809b99a1c09390a4580ca2d37d84c85dffacb1804f80c6f5a4491d312c20e6e3"
|
||||
"sha256:a9095f60121a365e84656c582e6ccd8dc8b3a732ddddb2ccd26cc3d32b77bdf6",
|
||||
"sha256:b06bea001985f089167320bda1e91c6b2d866f56ca810bb8d768ee3cf695ee21"
|
||||
],
|
||||
"markers": "python_version >= '3.8' and python_version < '4.0'",
|
||||
"version": "==1.0.3"
|
||||
"version": "==1.0.4"
|
||||
},
|
||||
"redis": {
|
||||
"hashes": [
|
||||
@ -3679,11 +3740,11 @@
|
||||
},
|
||||
"setuptools": {
|
||||
"hashes": [
|
||||
"sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e",
|
||||
"sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"
|
||||
"sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987",
|
||||
"sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==69.2.0"
|
||||
"version": "==69.5.1"
|
||||
},
|
||||
"sgmllib3k": {
|
||||
"hashes": [
|
||||
@ -3813,12 +3874,12 @@
|
||||
},
|
||||
"supabase": {
|
||||
"hashes": [
|
||||
"sha256:8b95744ce4ad24245ec23c090f273dfc9c2d9a53e3a80186959903947dbe1ed6",
|
||||
"sha256:a7dec0586f8931f378a45b2ffb28d8e37b3719f979c17f541b0156019144e645"
|
||||
"sha256:04e7095c3458e94a83f2b77493f788e730d148b2bc0c5d899de6f0bfb43d1ea6",
|
||||
"sha256:ce5de33e56eeccd92caa529dc29f484f0d5de073fd543b7dfd969e8cc57f5f62"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8' and python_version < '4.0'",
|
||||
"version": "==2.4.1"
|
||||
"version": "==2.4.2"
|
||||
},
|
||||
"supafunc": {
|
||||
"hashes": [
|
||||
@ -4188,7 +4249,7 @@
|
||||
"sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
|
||||
"sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"markers": "python_version >= '3.10'",
|
||||
"version": "==2.2.1"
|
||||
},
|
||||
"uvicorn": {
|
||||
@ -4429,6 +4490,120 @@
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"xxhash": {
|
||||
"hashes": [
|
||||
"sha256:00f2fdef6b41c9db3d2fc0e7f94cb3db86693e5c45d6de09625caad9a469635b",
|
||||
"sha256:0379d6cf1ff987cd421609a264ce025e74f346e3e145dd106c0cc2e3ec3f99a9",
|
||||
"sha256:0aac5010869240e95f740de43cd6a05eae180c59edd182ad93bf12ee289484fa",
|
||||
"sha256:0c786a6cd74e8765c6809892a0d45886e7c3dc54de4985b4a5eb8b630f3b8e3b",
|
||||
"sha256:0e041ce5714f95251a88670c114b748bca3bf80cc72400e9f23e6d0d59cf2681",
|
||||
"sha256:10e0a619cdd1c0980e25eb04e30fe96cf8f4324758fa497080af9c21a6de573f",
|
||||
"sha256:11f11357c86d83e53719c592021fd524efa9cf024dc7cb1dfb57bbbd0d8713f2",
|
||||
"sha256:1d03f1c0d16d24ea032e99f61c552cb2b77d502e545187338bea461fde253583",
|
||||
"sha256:1d0ae4c2e7698adef58710d6e7a32ff518b66b98854b1c68e70eee504ad061d8",
|
||||
"sha256:200a5a3ad9c7c0c02ed1484a1d838b63edcf92ff538770ea07456a3732c577f4",
|
||||
"sha256:2070b6d5bbef5ee031666cf21d4953c16e92c2f8a24a94b5c240f8995ba3b1d0",
|
||||
"sha256:21287bcdd299fdc3328cc0fbbdeaa46838a1c05391264e51ddb38a3f5b09611f",
|
||||
"sha256:23cfd9ca09acaf07a43e5a695143d9a21bf00f5b49b15c07d5388cadf1f9ce11",
|
||||
"sha256:248d3e83d119770f96003271fe41e049dd4ae52da2feb8f832b7a20e791d2920",
|
||||
"sha256:25dc66be3db54f8a2d136f695b00cfe88018e59ccff0f3b8f545869f376a8a46",
|
||||
"sha256:2a8ba6181514681c2591840d5632fcf7356ab287d4aff1c8dea20f3c78097088",
|
||||
"sha256:2be491723405e15cc099ade1280133ccfbf6322d2ef568494fb7d07d280e7eee",
|
||||
"sha256:312eba88ffe0a05e332e3a6f9788b73883752be63f8588a6dc1261a3eaaaf2b2",
|
||||
"sha256:36ad4457644c91a966f6fe137d7467636bdc51a6ce10a1d04f365c70d6a16d7e",
|
||||
"sha256:3b685fab18876b14a8f94813fa2ca80cfb5ab6a85d31d5539b7cd749ce9e3624",
|
||||
"sha256:4178f78d70e88f1c4a89ff1ffe9f43147185930bb962ee3979dba15f2b1cc799",
|
||||
"sha256:419ffe34c17ae2df019a4685e8d3934d46b2e0bbe46221ab40b7e04ed9f11137",
|
||||
"sha256:41ddeae47cf2828335d8d991f2d2b03b0bdc89289dc64349d712ff8ce59d0647",
|
||||
"sha256:431625fad7ab5649368c4849d2b49a83dc711b1f20e1f7f04955aab86cd307bc",
|
||||
"sha256:43984c0a92f06cac434ad181f329a1445017c33807b7ae4f033878d860a4b0f2",
|
||||
"sha256:450401f42bbd274b519d3d8dcf3c57166913381a3d2664d6609004685039f9d3",
|
||||
"sha256:4603a0f642a1e8d7f3ba5c4c25509aca6a9c1cc16f85091004a7028607ead663",
|
||||
"sha256:4c76a77dbd169450b61c06fd2d5d436189fc8ab7c1571d39265d4822da16df22",
|
||||
"sha256:4cb11d8debab1626181633d184b2372aaa09825bde709bf927704ed72765bed1",
|
||||
"sha256:543c7fcbc02bbb4840ea9915134e14dc3dc15cbd5a30873a7a5bf66039db97ec",
|
||||
"sha256:562d8b8f783c6af969806aaacf95b6c7b776929ae26c0cd941d54644ea7ef51e",
|
||||
"sha256:58c49083801885273e262c0f5bbeac23e520564b8357fbb18fb94ff09d3d3ea5",
|
||||
"sha256:595b252943b3552de491ff51e5bb79660f84f033977f88f6ca1605846637b7c6",
|
||||
"sha256:5bef2a7dc7b4f4beb45a1edbba9b9194c60a43a89598a87f1a0226d183764189",
|
||||
"sha256:5dab508ac39e0ab988039bc7f962c6ad021acd81fd29145962b068df4148c476",
|
||||
"sha256:6066d88c9329ab230e18998daec53d819daeee99d003955c8db6fc4971b45ca3",
|
||||
"sha256:6127813abc1477f3a83529b6bbcfeddc23162cece76fa69aee8f6a8a97720562",
|
||||
"sha256:64da57d5ed586ebb2ecdde1e997fa37c27fe32fe61a656b77fabbc58e6fbff6e",
|
||||
"sha256:665a65c2a48a72068fcc4d21721510df5f51f1142541c890491afc80451636d2",
|
||||
"sha256:672b273040d5d5a6864a36287f3514efcd1d4b1b6a7480f294c4b1d1ee1b8de0",
|
||||
"sha256:696b4e18b7023527d5c50ed0626ac0520edac45a50ec7cf3fc265cd08b1f4c03",
|
||||
"sha256:6a9ff50a3cf88355ca4731682c168049af1ca222d1d2925ef7119c1a78e95b3b",
|
||||
"sha256:6d3472fd4afef2a567d5f14411d94060099901cd8ce9788b22b8c6f13c606a93",
|
||||
"sha256:6d42b24d1496deb05dee5a24ed510b16de1d6c866c626c2beb11aebf3be278b9",
|
||||
"sha256:6e66df260fed01ed8ea790c2913271641c58481e807790d9fca8bfd5a3c13844",
|
||||
"sha256:6fa45e8cbfbadb40a920fe9ca40c34b393e0b067082d94006f7f64e70c7490a6",
|
||||
"sha256:719a378930504ab159f7b8e20fa2aa1896cde050011af838af7e7e3518dd82de",
|
||||
"sha256:71be94265b6c6590f0018bbf73759d21a41c6bda20409782d8117e76cd0dfa8b",
|
||||
"sha256:743612da4071ff9aa4d055f3f111ae5247342931dedb955268954ef7201a71ff",
|
||||
"sha256:74fb5cb9406ccd7c4dd917f16630d2e5e8cbbb02fc2fca4e559b2a47a64f4940",
|
||||
"sha256:7688d7c02149a90a3d46d55b341ab7ad1b4a3f767be2357e211b4e893efbaaf6",
|
||||
"sha256:7a97322e9a7440bf3c9805cbaac090358b43f650516486746f7fa482672593df",
|
||||
"sha256:8106d88da330f6535a58a8195aa463ef5281a9aa23b04af1848ff715c4398fb4",
|
||||
"sha256:8c59f3e46e7daf4c589e8e853d700ef6607afa037bfad32c390175da28127e8c",
|
||||
"sha256:8cc07256eff0795e0f642df74ad096f8c5d23fe66bc138b83970b50fc7f7f6c5",
|
||||
"sha256:911035345932a153c427107397c1518f8ce456f93c618dd1c5b54ebb22e73747",
|
||||
"sha256:91dbfa55346ad3e18e738742236554531a621042e419b70ad8f3c1d9c7a16e7f",
|
||||
"sha256:92693c487e39523a80474b0394645b393f0ae781d8db3474ccdcead0559ccf45",
|
||||
"sha256:93805bc3233ad89abf51772f2ed3355097a5dc74e6080de19706fc447da99cd3",
|
||||
"sha256:961d948b7b1c1b6c08484bbce3d489cdf153e4122c3dfb07c2039621243d8795",
|
||||
"sha256:9804b9eb254d4b8cc83ab5a2002128f7d631dd427aa873c8727dba7f1f0d1c2b",
|
||||
"sha256:9c0f7b2d547d72c7eda7aa817acf8791f0146b12b9eba1d4432c531fb0352228",
|
||||
"sha256:9ecb6c987b62437c2f99c01e97caf8d25660bf541fe79a481d05732e5236719c",
|
||||
"sha256:9f3025a0d5d8cf406a9313cd0d5789c77433ba2004b1c75439b67678e5136537",
|
||||
"sha256:9fd28a9da300e64e434cfc96567a8387d9a96e824a9be1452a1e7248b7763b78",
|
||||
"sha256:a15cbf3a9c40672523bdb6ea97ff74b443406ba0ab9bca10ceccd9546414bd84",
|
||||
"sha256:a162840cf4de8a7cd8720ff3b4417fbc10001eefdd2d21541a8226bb5556e3bb",
|
||||
"sha256:a55e0506fdb09640a82ec4f44171273eeabf6f371a4ec605633adb2837b5d9d5",
|
||||
"sha256:a8b4977963926f60b0d4f830941c864bed16aa151206c01ad5c531636da5708e",
|
||||
"sha256:a90356ead70d715fe64c30cd0969072de1860e56b78adf7c69d954b43e29d9fa",
|
||||
"sha256:aabf37fb8fa27430d50507deeab2ee7b1bcce89910dd10657c38e71fee835594",
|
||||
"sha256:ac56eebb364e44c85e1d9e9cc5f6031d78a34f0092fea7fc80478139369a8b4a",
|
||||
"sha256:b2746035f518f0410915e247877f7df43ef3372bf36cfa52cc4bc33e85242641",
|
||||
"sha256:b29728cff2c12f3d9f1d940528ee83918d803c0567866e062683f300d1d2eff3",
|
||||
"sha256:b41edaf05734092f24f48c0958b3c6cbaaa5b7e024880692078c6b1f8247e2fc",
|
||||
"sha256:b526015a973bfbe81e804a586b703f163861da36d186627e27524f5427b0d520",
|
||||
"sha256:b5beb1c6a72fdc7584102f42c4d9df232ee018ddf806e8c90906547dfb43b2da",
|
||||
"sha256:b736a2a2728ba45017cb67785e03125a79d246462dfa892d023b827007412c52",
|
||||
"sha256:b9097af00ebf429cc7c0e7d2fdf28384e4e2e91008130ccda8d5ae653db71e54",
|
||||
"sha256:bb11628470a6004dc71a09fe90c2f459ff03d611376c1debeec2d648f44cb693",
|
||||
"sha256:bbe750d512982ee7d831838a5dee9e9848f3fb440e4734cca3f298228cc957a6",
|
||||
"sha256:c09c49473212d9c87261d22c74370457cfff5db2ddfc7fd1e35c80c31a8c14ce",
|
||||
"sha256:c44d584afdf3c4dbb3277e32321d1a7b01d6071c1992524b6543025fb8f4206f",
|
||||
"sha256:c4bbba9b182697a52bc0c9f8ec0ba1acb914b4937cd4a877ad78a3b3eeabefb3",
|
||||
"sha256:c9e1b646af61f1fc7083bb7b40536be944f1ac67ef5e360bca2d73430186971a",
|
||||
"sha256:ca7783b20e3e4f3f52f093538895863f21d18598f9a48211ad757680c3bd006f",
|
||||
"sha256:d6322c4291c3ff174dcd104fae41500e75dad12be6f3085d119c2c8a80956c51",
|
||||
"sha256:d699b921af0dcde50ab18be76c0d832f803034d80470703700cb7df0fbec2832",
|
||||
"sha256:d77d09a1113899fad5f354a1eb4f0a9afcf58cefff51082c8ad643ff890e30cf",
|
||||
"sha256:dd59ed668801c3fae282f8f4edadf6dc7784db6d18139b584b6d9677ddde1b6b",
|
||||
"sha256:dfd7a6cc483e20b4ad90224aeb589e64ec0f31e5610ab9957ff4314270b2bf31",
|
||||
"sha256:e01226b6b6a1ffe4e6bd6d08cfcb3ca708b16f02eb06dd44f3c6e53285f03e4f",
|
||||
"sha256:e17032f5a4fea0a074717fe33477cb5ee723a5f428de7563e75af64bfc1b1e10",
|
||||
"sha256:e867f68a8f381ea12858e6d67378c05359d3a53a888913b5f7d35fbf68939d5f",
|
||||
"sha256:e9f749999ed80f3955a4af0eb18bb43993f04939350b07b8dd2f44edc98ffee9",
|
||||
"sha256:ebbb1616435b4a194ce3466d7247df23499475c7ed4eb2681a1fa42ff766aff6",
|
||||
"sha256:ef2e194262f5db16075caea7b3f7f49392242c688412f386d3c7b07c7733a70a",
|
||||
"sha256:ef73a53fe90558a4096e3256752268a8bdc0322f4692ed928b6cd7ce06ad4fe3",
|
||||
"sha256:f1d7c69a1e9ca5faa75546fdd267f214f63f52f12692f9b3a2f6467c9e67d5e7",
|
||||
"sha256:f31ce76489f8601cc7b8713201ce94b4bd7b7ce90ba3353dccce7e9e1fee71fa",
|
||||
"sha256:f3ff8dbd0ec97aec842476cb8ccc3e17dd288cd6ce3c8ef38bff83d6eb927817",
|
||||
"sha256:fa122124d2e3bd36581dd78c0efa5f429f5220313479fb1072858188bc2d5ff1",
|
||||
"sha256:faec30437919555b039a8bdbaba49c013043e8f76c999670aef146d33e05b3a0",
|
||||
"sha256:fc6dbd5fc3c9886a9e041848508b7fb65fd82f94cc793253990f81617b61fe49",
|
||||
"sha256:fc860d887c5cb2f524899fb8338e1bb3d5789f75fac179101920d9afddef284b",
|
||||
"sha256:fd79d4087727daf4d5b8afe594b37d611ab95dc8e29fe1a7517320794837eb7d",
|
||||
"sha256:fd7bddb3a5b86213cc3f2c61500c16945a1b80ecd572f3078ddbbe68f9dabdfb",
|
||||
"sha256:fe0a98d990e433013f41827b62be9ab43e3cf18e08b1483fcc343bda0d691182"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==3.4.1"
|
||||
},
|
||||
"yarl": {
|
||||
"hashes": [
|
||||
"sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51",
|
||||
@ -4537,31 +4712,31 @@
|
||||
"develop": {
|
||||
"black": {
|
||||
"hashes": [
|
||||
"sha256:2818cf72dfd5d289e48f37ccfa08b460bf469e67fb7c4abb07edc2e9f16fb63f",
|
||||
"sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93",
|
||||
"sha256:4acf672def7eb1725f41f38bf6bf425c8237248bb0804faa3965c036f7672d11",
|
||||
"sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0",
|
||||
"sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9",
|
||||
"sha256:56f52cfbd3dabe2798d76dbdd299faa046a901041faf2cf33288bc4e6dae57b5",
|
||||
"sha256:65b76c275e4c1c5ce6e9870911384bff5ca31ab63d19c76811cb1fb162678213",
|
||||
"sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d",
|
||||
"sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7",
|
||||
"sha256:79dcf34b33e38ed1b17434693763301d7ccbd1c5860674a8f871bd15139e7837",
|
||||
"sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f",
|
||||
"sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395",
|
||||
"sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995",
|
||||
"sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f",
|
||||
"sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597",
|
||||
"sha256:b5991d523eee14756f3c8d5df5231550ae8993e2286b8014e2fdea7156ed0959",
|
||||
"sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5",
|
||||
"sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb",
|
||||
"sha256:c7ed6668cbbfcd231fa0dc1b137d3e40c04c7f786e626b405c62bcd5db5857e4",
|
||||
"sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7",
|
||||
"sha256:e19cb1c6365fd6dc38a6eae2dcb691d7d83935c10215aef8e6c38edee3f77abd",
|
||||
"sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"
|
||||
"sha256:1bb9ca06e556a09f7f7177bc7cb604e5ed2d2df1e9119e4f7d2f1f7071c32e5d",
|
||||
"sha256:21f9407063ec71c5580b8ad975653c66508d6a9f57bd008bb8691d273705adcd",
|
||||
"sha256:4396ca365a4310beef84d446ca5016f671b10f07abdba3e4e4304218d2c71d33",
|
||||
"sha256:44d99dfdf37a2a00a6f7a8dcbd19edf361d056ee51093b2445de7ca09adac965",
|
||||
"sha256:5cd5b4f76056cecce3e69b0d4c228326d2595f506797f40b9233424e2524c070",
|
||||
"sha256:64578cf99b6b46a6301bc28bdb89f9d6f9b592b1c5837818a177c98525dbe397",
|
||||
"sha256:64e60a7edd71fd542a10a9643bf369bfd2644de95ec71e86790b063aa02ff745",
|
||||
"sha256:652e55bb722ca026299eb74e53880ee2315b181dfdd44dca98e43448620ddec1",
|
||||
"sha256:6644f97a7ef6f401a150cca551a1ff97e03c25d8519ee0bbc9b0058772882665",
|
||||
"sha256:6ad001a9ddd9b8dfd1b434d566be39b1cd502802c8d38bbb1ba612afda2ef436",
|
||||
"sha256:71d998b73c957444fb7c52096c3843875f4b6b47a54972598741fe9a7f737fcb",
|
||||
"sha256:74eb9b5420e26b42c00a3ff470dc0cd144b80a766128b1771d07643165e08d0e",
|
||||
"sha256:75a2d0b4f5eb81f7eebc31f788f9830a6ce10a68c91fbe0fade34fff7a2836e6",
|
||||
"sha256:7852b05d02b5b9a8c893ab95863ef8986e4dda29af80bbbda94d7aee1abf8702",
|
||||
"sha256:7f2966b9b2b3b7104fca9d75b2ee856fe3fdd7ed9e47c753a4bb1a675f2caab8",
|
||||
"sha256:8e5537f456a22cf5cfcb2707803431d2feeb82ab3748ade280d6ccd0b40ed2e8",
|
||||
"sha256:d4e71cdebdc8efeb6deaf5f2deb28325f8614d48426bed118ecc2dcaefb9ebf3",
|
||||
"sha256:dae79397f367ac8d7adb6c779813328f6d690943f64b32983e896bcccd18cbad",
|
||||
"sha256:e3a3a092b8b756c643fe45f4624dbd5a389f770a4ac294cf4d0fce6af86addaf",
|
||||
"sha256:eb949f56a63c5e134dfdca12091e98ffb5fd446293ebae123d10fc1abad00b9e",
|
||||
"sha256:f07b69fda20578367eaebbd670ff8fc653ab181e1ff95d84497f9fa20e7d0641",
|
||||
"sha256:f95cece33329dc4aa3b0e1a771c41075812e46cf3d6e3f1dfe3d91ff09826ed2"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==24.3.0"
|
||||
"version": "==24.4.0"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
|
||||
@ -41,6 +41,8 @@ async def process_file(
|
||||
if file.documents is not None:
|
||||
for doc in file.documents: # pyright: ignore reportPrivateUsage=none
|
||||
new_metadata = metadata.copy()
|
||||
# Add filename at beginning of page content
|
||||
doc.page_content = f"Filename: {new_metadata['original_file_name']} Content: {doc.page_content}"
|
||||
len_chunk = len(enc.encode(doc.page_content))
|
||||
page_content_encoded = doc.page_content.encode("unicode_escape").decode(
|
||||
"ascii", "replace"
|
||||
|
||||
@ -7,6 +7,7 @@ amqp==5.2.0; python_version >= '3.6'
|
||||
annotated-types==0.6.0; python_version >= '3.8'
|
||||
antlr4-python3-runtime==4.9.3
|
||||
anyio==4.3.0; python_version >= '3.8'
|
||||
appdirs==1.4.4
|
||||
astor==0.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
async-generator==1.10; python_version >= '3.5'
|
||||
async-timeout==4.0.3; python_full_version < '3.12.0'
|
||||
@ -15,9 +16,9 @@ attrs==23.2.0; python_version >= '3.7'
|
||||
backoff==2.2.1; python_version >= '3.7' and python_version < '4.0'
|
||||
beautifulsoup4==4.12.3; python_full_version >= '3.6.0'
|
||||
billiard==4.2.0; python_version >= '3.7'
|
||||
black==24.3.0; python_version >= '3.8'
|
||||
boto3==1.34.83; python_version >= '3.8'
|
||||
botocore==1.34.83; python_version >= '3.8'
|
||||
black==24.4.0; python_version >= '3.8'
|
||||
boto3==1.34.84; python_version >= '3.8'
|
||||
botocore==1.34.84; python_version >= '3.8'
|
||||
celery[redis,sqs]==5.3.6; python_version >= '3.8'
|
||||
certifi==2024.2.2; python_version >= '3.6'
|
||||
cffi==1.16.0; platform_python_implementation != 'PyPy'
|
||||
@ -35,10 +36,12 @@ cssselect==1.2.0; python_version >= '3.7'
|
||||
cycler==0.12.1; python_version >= '3.8'
|
||||
dataclasses-json==0.6.4; python_version >= '3.7' and python_version < '4.0'
|
||||
dataclasses-json-speakeasy==0.5.11; python_version >= '3.7' and python_version < '4.0'
|
||||
datasets==2.18.0; python_full_version >= '3.8.0'
|
||||
debugpy==1.8.1; python_version >= '3.8'
|
||||
decorator==5.1.1; python_version >= '3.5'
|
||||
deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
deprecation==2.1.0
|
||||
dill==0.3.8; python_version >= '3.8'
|
||||
dirtyjson==1.0.8
|
||||
distro==1.9.0; python_version >= '3.6'
|
||||
docker==7.0.0; python_version >= '3.8'
|
||||
@ -60,7 +63,7 @@ flatbuffers==24.3.25
|
||||
flower==2.0.1; python_version >= '3.7'
|
||||
fonttools==4.51.0; python_version >= '3.8'
|
||||
frozenlist==1.4.1; python_version >= '3.8'
|
||||
fsspec==2024.3.1; python_version >= '3.8'
|
||||
fsspec[http]==2024.2.0; python_version >= '3.8'
|
||||
gitdb==4.0.11; python_version >= '3.7'
|
||||
gitpython==3.1.43; python_version >= '3.7'
|
||||
gotrue==2.4.2; python_version >= '3.8' and python_version < '4.0'
|
||||
@ -92,14 +95,14 @@ langchain-core==0.1.42; python_version < '4.0' and python_full_version >= '3.8.1
|
||||
langchain-openai==0.1.3; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
langchain-text-splitters==0.0.1; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
langdetect==1.0.9
|
||||
langfuse==2.25.0; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
langsmith==0.1.45; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
langfuse==2.26.1; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
langsmith==0.1.47; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
layoutparser[layoutmodels,tesseract]==0.3.4; python_version >= '3.6'
|
||||
litellm==1.35.2; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8'
|
||||
llama-index==0.10.28; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
litellm==1.35.5; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8'
|
||||
llama-index==0.10.29; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-agent-openai==0.2.2; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-cli==0.1.11; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-core==0.10.28; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-core==0.10.29; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-embeddings-openai==0.1.7; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-indices-managed-llama-cloud==0.1.5; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-legacy==0.9.48; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
@ -107,10 +110,10 @@ llama-index-llms-openai==0.1.15; python_version < '4.0' and python_full_version
|
||||
llama-index-multi-modal-llms-openai==0.1.5; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-program-openai==0.1.5; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-question-gen-openai==0.1.3; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-readers-file==0.1.16; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-readers-file==0.1.18; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-index-readers-llama-parse==0.1.4; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llama-parse==0.4.0; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llamaindex-py-client==0.1.17; python_version >= '3.8' and python_version < '4'
|
||||
llama-parse==0.4.1; python_version < '4.0' and python_full_version >= '3.8.1'
|
||||
llamaindex-py-client==0.1.18; python_version >= '3.8' and python_version < '4'
|
||||
lxml[html_clean]==5.2.1; python_version >= '3.6'
|
||||
lxml-html-clean==0.1.1
|
||||
markdown==3.6
|
||||
@ -122,6 +125,7 @@ monotonic==1.6
|
||||
mpmath==1.3.0
|
||||
msg-parser==1.2.0
|
||||
multidict==6.0.5; python_version >= '3.7'
|
||||
multiprocess==0.70.16; python_version >= '3.8'
|
||||
mypy-extensions==1.0.0; python_version >= '3.5'
|
||||
nest-asyncio==1.6.0; python_version >= '3.5'
|
||||
networkx==3.3
|
||||
@ -133,7 +137,7 @@ olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2,
|
||||
omegaconf==2.3.0; python_version >= '3.6'
|
||||
onnx==1.16.0
|
||||
onnxruntime==1.15.1
|
||||
openai==1.17.0; python_full_version >= '3.7.1'
|
||||
openai==1.17.1; python_full_version >= '3.7.1'
|
||||
opencv-python==4.9.0.80; python_version >= '3.6'
|
||||
openpyxl==3.1.2
|
||||
orjson==3.10.0; python_version >= '3.8'
|
||||
@ -150,7 +154,7 @@ pillow-heif==0.16.0
|
||||
platformdirs==4.2.0; python_version >= '3.8'
|
||||
pluggy==1.4.0; python_version >= '3.8'
|
||||
portalocker==2.8.2; python_version >= '3.8'
|
||||
postgrest==0.16.2; python_version >= '3.8' and python_version < '4.0'
|
||||
postgrest==0.16.3; python_version >= '3.8' and python_version < '4.0'
|
||||
posthog==3.5.0
|
||||
prometheus-client==0.20.0; python_version >= '3.8'
|
||||
prompt-toolkit==3.0.43; python_full_version >= '3.7.0'
|
||||
@ -159,6 +163,8 @@ psutil==5.9.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2,
|
||||
psycopg2==2.9.9; python_version >= '3.7'
|
||||
psycopg2-binary==2.9.9; python_version >= '3.7'
|
||||
py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pyarrow==15.0.2; python_version >= '3.8'
|
||||
pyarrow-hotfix==0.6; python_version >= '3.5'
|
||||
pyasn1==0.6.0; python_version >= '3.8'
|
||||
pycocotools==2.0.7; python_version >= '3.5'
|
||||
pycodestyle==2.11.1; python_version >= '3.8'
|
||||
@ -168,13 +174,12 @@ pydantic==2.7.0; python_version >= '3.8'
|
||||
pydantic-core==2.18.1; python_version >= '3.8'
|
||||
pydantic-settings==2.2.1; python_version >= '3.8'
|
||||
pyflakes==3.2.0; python_version >= '3.8'
|
||||
pymupdf==1.24.1; python_version >= '3.8'
|
||||
pymupdfb==1.24.1; python_version >= '3.8'
|
||||
pypandoc==1.13; python_version >= '3.6'
|
||||
pyparsing==3.1.2; python_full_version >= '3.6.8'
|
||||
pypdf==4.2.0; python_version >= '3.6'
|
||||
pypdfium2==4.29.0; python_version >= '3.6'
|
||||
pyright==1.1.358; python_version >= '3.7'
|
||||
pysbd==0.3.4; python_version >= '3'
|
||||
pytesseract==0.3.10; python_version >= '3.7'
|
||||
pytest==8.1.1; python_version >= '3.8'
|
||||
pytest-celery==1.0.0; python_version >= '3.8' and python_version < '4.0'
|
||||
@ -190,8 +195,9 @@ python-multipart==0.0.9; python_version >= '3.8'
|
||||
python-pptx==0.6.23
|
||||
pytz==2024.1
|
||||
pyyaml==6.0.1; python_version >= '3.6'
|
||||
ragas==0.1.7
|
||||
rapidfuzz==3.8.1; python_version >= '3.8'
|
||||
realtime==1.0.3; python_version >= '3.8' and python_version < '4.0'
|
||||
realtime==1.0.4; python_version >= '3.8' and python_version < '4.0'
|
||||
redis==5.0.3; python_version >= '3.7'
|
||||
regex==2023.12.25; python_version >= '3.7'
|
||||
requests==2.31.0; python_version >= '3.7'
|
||||
@ -203,7 +209,7 @@ s3transfer==0.10.1; python_version >= '3.8'
|
||||
safetensors==0.4.2; python_version >= '3.7'
|
||||
scipy==1.13.0; python_version >= '3.9'
|
||||
sentry-sdk[fastapi]==1.45.0
|
||||
setuptools==69.2.0; python_version >= '3.8'
|
||||
setuptools==69.5.1; python_version >= '3.8'
|
||||
sgmllib3k==1.0.0
|
||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
smmap==5.0.1; python_version >= '3.7'
|
||||
@ -214,7 +220,7 @@ starlette==0.37.2; python_version >= '3.8'
|
||||
storage3==0.7.4; python_version >= '3.8' and python_version < '4.0'
|
||||
strenum==0.4.15
|
||||
striprtf==0.0.26
|
||||
supabase==2.4.1; python_version >= '3.8' and python_version < '4.0'
|
||||
supabase==2.4.2; python_version >= '3.8' and python_version < '4.0'
|
||||
supafunc==0.4.5; python_version >= '3.8' and python_version < '4.0'
|
||||
sympy==1.12; python_version >= '3.8'
|
||||
tabulate==0.9.0; python_version >= '3.7'
|
||||
@ -236,7 +242,7 @@ unstructured[all-docs]==0.13.2; python_version < '3.12' and python_full_version
|
||||
unstructured-client==0.18.0; python_version >= '3.8'
|
||||
unstructured-inference==0.7.25
|
||||
unstructured.pytesseract==0.3.12
|
||||
urllib3==2.2.1; python_version >= '3.8'
|
||||
urllib3==2.2.1; python_version >= '3.10'
|
||||
uvicorn==0.29.0; python_version >= '3.8'
|
||||
vine==5.1.0; python_version >= '3.6'
|
||||
watchdog==4.0.0; python_version >= '3.8'
|
||||
@ -246,5 +252,6 @@ websockets==12.0; python_version >= '3.8'
|
||||
wrapt==1.16.0; python_version >= '3.6'
|
||||
xlrd==2.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
xlsxwriter==3.2.0; python_version >= '3.6'
|
||||
xxhash==3.4.1; python_version >= '3.7'
|
||||
yarl==1.9.4; python_version >= '3.7'
|
||||
zipp==3.18.1; python_version >= '3.8'
|
||||
|
||||
BIN
backend/tests/input/Plan Artemis Sept 2020.pdf
Normal file
BIN
backend/tests/input/Plan Artemis Sept 2020.pdf
Normal file
Binary file not shown.
@ -1,38 +1,41 @@
|
||||
import argparse
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
import sys
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Add the current directory to the Python path
|
||||
sys.path.append(os.getcwd())
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
load_dotenv(verbose=True, override=True)
|
||||
|
||||
import glob
|
||||
import uuid
|
||||
|
||||
import pandas as pd
|
||||
import uuid
|
||||
import glob
|
||||
import ragas
|
||||
from datasets import Dataset
|
||||
|
||||
from celery_worker import process_file_and_notify
|
||||
from repository.files.upload_file import upload_file_storage
|
||||
from datasets import Dataset
|
||||
from langchain_core.runnables.base import RunnableSerializable
|
||||
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
|
||||
from modules.brain.rags.quivr_rag import QuivrRAG
|
||||
from modules.brain.service.brain_service import BrainService
|
||||
from modules.knowledge.dto.inputs import CreateKnowledgeProperties
|
||||
from modules.knowledge.service.knowledge_service import KnowledgeService
|
||||
from modules.brain.service.brain_service import BrainService
|
||||
from modules.brain.rags.quivr_rag import QuivrRAG
|
||||
from ragas import evaluate
|
||||
from ragas.embeddings.base import LangchainEmbeddingsWrapper
|
||||
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
|
||||
from langchain_core.runnables.base import RunnableSerializable
|
||||
from repository.files.upload_file import upload_file_storage
|
||||
|
||||
|
||||
def main(testset_path, input_folder, output_folder, model, context_size, metrics):
|
||||
def main(
|
||||
testset_path, input_folder, output_folder, model, context_size, metrics, brain_id
|
||||
):
|
||||
# Create a fake user and brain
|
||||
# Check if a brain ID was provided
|
||||
if args.brain_id:
|
||||
brain = brain_service.get_brain(args.brain_id)
|
||||
if not brain:
|
||||
print('Invalid brain ID provided.')
|
||||
print("Invalid brain ID provided.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
# Create a new brain
|
||||
@ -42,7 +45,7 @@ def main(testset_path, input_folder, output_folder, model, context_size, metrics
|
||||
brain = brain_service.create_brain(user_id=uuid_value, brain=None)
|
||||
brain_id = brain.brain_id
|
||||
|
||||
for document_path in glob.glob(input_folder + '/*'):
|
||||
for document_path in glob.glob(input_folder + "/*"):
|
||||
# Process each document here
|
||||
process_document(knowledge_service, brain_id, document_path)
|
||||
|
||||
@ -56,7 +59,7 @@ def main(testset_path, input_folder, output_folder, model, context_size, metrics
|
||||
chat_id=str(uuid.uuid4()),
|
||||
streaming=False,
|
||||
max_input=context_size,
|
||||
max_tokens=1000
|
||||
max_tokens=1000,
|
||||
)
|
||||
brain_chain = knowledge_qa.get_chain()
|
||||
|
||||
@ -64,10 +67,15 @@ def main(testset_path, input_folder, output_folder, model, context_size, metrics
|
||||
response_dataset = generate_replies(test_data, brain_chain)
|
||||
|
||||
ragas_metrics = [getattr(ragas.metrics, metric) for metric in metrics]
|
||||
score = evaluate(response_dataset,metrics=ragas_metrics,
|
||||
llm=ChatOpenAI(model="gpt-4", temperature=0.1),
|
||||
embeddings=LangchainEmbeddingsWrapper(OpenAIEmbeddings(model="text-embedding-3-large", dimensions=3072))).to_pandas()
|
||||
|
||||
score = evaluate(
|
||||
response_dataset,
|
||||
metrics=ragas_metrics,
|
||||
llm=ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0.1),
|
||||
embeddings=LangchainEmbeddingsWrapper(
|
||||
OpenAIEmbeddings(model="text-embedding-3-large", dimensions=1536)
|
||||
),
|
||||
).to_pandas()
|
||||
|
||||
score.to_json(output_folder + "/score.json", orient="records")
|
||||
for metric in metrics:
|
||||
print(f"{metric} mean score: {score[metric].mean()}")
|
||||
@ -76,7 +84,10 @@ def main(testset_path, input_folder, output_folder, model, context_size, metrics
|
||||
if not args.brain_id:
|
||||
brain_service.delete_brain(brain_id)
|
||||
|
||||
def process_document(knowledge_service: KnowledgeService, brain_id: uuid.UUID, document_path: str) -> None:
|
||||
|
||||
def process_document(
|
||||
knowledge_service: KnowledgeService, brain_id: uuid.UUID, document_path: str
|
||||
) -> None:
|
||||
"""
|
||||
Process a document by uploading it to the file storage, adding knowledge to the knowledge service,
|
||||
and then processing the file and sending a notification.
|
||||
@ -94,24 +105,27 @@ def process_document(knowledge_service: KnowledgeService, brain_id: uuid.UUID, d
|
||||
file_in_storage = upload_file_storage(document_path, filename_with_brain_id)
|
||||
|
||||
knowledge_to_add = CreateKnowledgeProperties(
|
||||
brain_id=brain_id,
|
||||
file_name=filename,
|
||||
extension=os.path.splitext(
|
||||
filename # pyright: ignore reportPrivateUsage=none
|
||||
)[-1].lower(),
|
||||
)
|
||||
brain_id=brain_id,
|
||||
file_name=filename,
|
||||
extension=os.path.splitext(filename)[ # pyright: ignore reportPrivateUsage=none
|
||||
-1
|
||||
].lower(),
|
||||
)
|
||||
|
||||
added_knowledge = knowledge_service.add_knowledge(knowledge_to_add)
|
||||
print(f"Knowledge {added_knowledge} added successfully")
|
||||
|
||||
process_file_and_notify(
|
||||
file_name=filename_with_brain_id,
|
||||
file_original_name=filename,
|
||||
brain_id=brain_id,
|
||||
notification_id=None,
|
||||
)
|
||||
file_name=filename_with_brain_id,
|
||||
file_original_name=filename,
|
||||
brain_id=brain_id,
|
||||
notification_id=None,
|
||||
)
|
||||
|
||||
def generate_replies(test_data: pd.DataFrame, brain_chain: RunnableSerializable) -> Dataset:
|
||||
|
||||
def generate_replies(
|
||||
test_data: pd.DataFrame, brain_chain: RunnableSerializable
|
||||
) -> Dataset:
|
||||
"""
|
||||
Generate replies for a given test data using a brain chain.
|
||||
|
||||
@ -128,27 +142,76 @@ def generate_replies(test_data: pd.DataFrame, brain_chain: RunnableSerializable)
|
||||
test_groundtruths = test_data.ground_truth.tolist()
|
||||
|
||||
for question in test_questions:
|
||||
response = brain_chain.invoke({"question" : question})
|
||||
response = brain_chain.invoke({"question": question})
|
||||
answers.append(response["answer"].content)
|
||||
contexts.append([context.page_content for context in response["docs"]])
|
||||
|
||||
return Dataset.from_dict({
|
||||
"question" : test_questions,
|
||||
"answer" : answers,
|
||||
"contexts" : contexts,
|
||||
"ground_truth" : test_groundtruths
|
||||
})
|
||||
|
||||
return Dataset.from_dict(
|
||||
{
|
||||
"question": test_questions,
|
||||
"answer": answers,
|
||||
"contexts": contexts,
|
||||
"ground_truth": test_groundtruths,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Run Ragas evaluation on a test dataset')
|
||||
parser.add_argument('--input_folder', type=str, required=True, help='Path to the testset documents folder')
|
||||
parser.add_argument('--output_folder', type=str, default='./', help='Path to the output folder')
|
||||
parser.add_argument('--testset_path', type=str, required=True, help='Path to the testset JSON file')
|
||||
parser.add_argument('--model', type=str, default='gpt-3.5-turbo-0125', help='Model to use')
|
||||
parser.add_argument('--context_size', type=int, default=4000, help='Context size for the model')
|
||||
parser.add_argument('--metrics', type=str, nargs='+', choices=['answer_correctness', 'context_relevancy', 'context_precision', 'faithfulness', 'answer_similarity'], default=['answer_correctness'], help='Metrics to evaluate')
|
||||
parser.add_argument('--brain_id', type=str, default=None, help='ID of an existing brain to use for the evaluation')
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run Ragas evaluation on a test dataset"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input_folder",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to the testset documents folder",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output_folder", type=str, default="./", help="Path to the output folder"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--testset_path", type=str, required=True, help="Path to the testset JSON file"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model", type=str, default="gpt-3.5-turbo-0125", help="Model to use"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--context_size", type=int, default=4000, help="Context size for the model"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--metrics",
|
||||
type=str,
|
||||
nargs="+",
|
||||
choices=[
|
||||
"answer_correctness",
|
||||
"context_relevancy",
|
||||
"context_precision",
|
||||
"faithfulness",
|
||||
"answer_similarity",
|
||||
],
|
||||
default=["answer_correctness"],
|
||||
help="Metrics to evaluate",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--brain_id",
|
||||
type=str,
|
||||
default=None,
|
||||
help="ID of an existing brain to use for the evaluation",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args.testset_path, args.input_folder, args.output_folder, args.model, args.context_size, args.metrics, args.brain_id)
|
||||
main(
|
||||
args.testset_path,
|
||||
args.input_folder,
|
||||
args.output_folder,
|
||||
args.model,
|
||||
args.context_size,
|
||||
args.metrics,
|
||||
args.brain_id,
|
||||
)
|
||||
|
||||
|
||||
# Run by doing from the backend folder:
|
||||
# python3 tests/ragas_evaluation/run_evaluation.py --input_folder ./tests/input --testset_path ./tests/ragas_tests/experiment.json --context_size 1000
|
||||
# Make sure to copy the .env file to the backend folder and modify http://host.docker.internal to http://localhost
|
||||
# in the .env file
|
||||
|
||||
1
backend/tests/ragas_tests/experiment.json
Normal file
1
backend/tests/ragas_tests/experiment.json
Normal file
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user