From bdb115ad0a2ecc243f6a6aa0d2a274dca297df5b Mon Sep 17 00:00:00 2001 From: Stan Girard Date: Sun, 28 Apr 2024 06:10:21 -0700 Subject: [PATCH] feat(profiler): Add pyinstrument package and update Makefile and backend code (#2512) This pull request adds the pyinstrument package and updates the Makefile and backend code. The pyinstrument package is used for profiling and the Makefile and backend code have been modified to support profiling. --- Makefile | 3 + Pipfile | 1 + Pipfile.lock | 530 +++++++++++++++-------- backend/main.py | 22 +- backend/packages/files/parsers/common.py | 6 + backend/requirements.txt | 60 +-- 6 files changed, 421 insertions(+), 201 deletions(-) diff --git a/Makefile b/Makefile index 09023a5b3..892c67733 100644 --- a/Makefile +++ b/Makefile @@ -26,3 +26,6 @@ test-type: else \ python -m pyright; \ fi + +front: + cd frontend && yarn build && yarn start \ No newline at end of file diff --git a/Pipfile b/Pipfile index 6bea8c238..d01d2b51c 100644 --- a/Pipfile +++ b/Pipfile @@ -62,6 +62,7 @@ fpdf2 = "*" unidecode = "*" flashrank = "*" langchain-cohere = "*" +pyinstrument = "*" [dev-packages] black = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 42330d71b..a2e3a43d2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5f16e6c41e1a2d88d01793f7b95b607dc71dd47d6871388567affc382d79e08b" + "sha256": "25cb55a8851739854e62cd99c9b76023592d57c2d6b08a96da881ddaaaae0a4b" }, "pipfile-spec": 6, "requires": { @@ -251,49 +251,57 @@ }, "black": { "hashes": [ - "sha256:1bb9ca06e556a09f7f7177bc7cb604e5ed2d2df1e9119e4f7d2f1f7071c32e5d", - "sha256:21f9407063ec71c5580b8ad975653c66508d6a9f57bd008bb8691d273705adcd", - "sha256:4396ca365a4310beef84d446ca5016f671b10f07abdba3e4e4304218d2c71d33", - "sha256:44d99dfdf37a2a00a6f7a8dcbd19edf361d056ee51093b2445de7ca09adac965", - "sha256:5cd5b4f76056cecce3e69b0d4c228326d2595f506797f40b9233424e2524c070", - "sha256:64578cf99b6b46a6301bc28bdb89f9d6f9b592b1c5837818a177c98525dbe397", - "sha256:64e60a7edd71fd542a10a9643bf369bfd2644de95ec71e86790b063aa02ff745", - "sha256:652e55bb722ca026299eb74e53880ee2315b181dfdd44dca98e43448620ddec1", - "sha256:6644f97a7ef6f401a150cca551a1ff97e03c25d8519ee0bbc9b0058772882665", - "sha256:6ad001a9ddd9b8dfd1b434d566be39b1cd502802c8d38bbb1ba612afda2ef436", - "sha256:71d998b73c957444fb7c52096c3843875f4b6b47a54972598741fe9a7f737fcb", - "sha256:74eb9b5420e26b42c00a3ff470dc0cd144b80a766128b1771d07643165e08d0e", - "sha256:75a2d0b4f5eb81f7eebc31f788f9830a6ce10a68c91fbe0fade34fff7a2836e6", - "sha256:7852b05d02b5b9a8c893ab95863ef8986e4dda29af80bbbda94d7aee1abf8702", - "sha256:7f2966b9b2b3b7104fca9d75b2ee856fe3fdd7ed9e47c753a4bb1a675f2caab8", - "sha256:8e5537f456a22cf5cfcb2707803431d2feeb82ab3748ade280d6ccd0b40ed2e8", - "sha256:d4e71cdebdc8efeb6deaf5f2deb28325f8614d48426bed118ecc2dcaefb9ebf3", - "sha256:dae79397f367ac8d7adb6c779813328f6d690943f64b32983e896bcccd18cbad", - "sha256:e3a3a092b8b756c643fe45f4624dbd5a389f770a4ac294cf4d0fce6af86addaf", - "sha256:eb949f56a63c5e134dfdca12091e98ffb5fd446293ebae123d10fc1abad00b9e", - "sha256:f07b69fda20578367eaebbd670ff8fc653ab181e1ff95d84497f9fa20e7d0641", - "sha256:f95cece33329dc4aa3b0e1a771c41075812e46cf3d6e3f1dfe3d91ff09826ed2" + "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474", + "sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1", + "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0", + "sha256:48a85f2cb5e6799a9ef05347b476cce6c182d6c71ee36925a6c194d074336ef8", + "sha256:7768a0dbf16a39aa5e9a3ded568bb545c8c2727396d063bbaf847df05b08cd96", + "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1", + "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04", + "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021", + "sha256:98e123f1d5cfd42f886624d84464f7756f60ff6eab89ae845210631714f6db94", + "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d", + "sha256:b1530ae42e9d6d5b670a34db49a94115a64596bc77710b1d05e9801e62ca0a7c", + "sha256:b9176b9832e84308818a99a561e90aa479e73c523b3f77afd07913380ae2eab7", + "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c", + "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc", + "sha256:bf10f7310db693bb62692609b397e8d67257c55f949abde4c67f9cc574492cc7", + "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d", + "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c", + "sha256:da33a1a5e49c4122ccdfd56cd021ff1ebc4a1ec4e2d01594fef9b6f267a9e741", + "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce", + "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb", + "sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063", + "sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e" ], "markers": "python_version >= '3.8'", - "version": "==24.4.0" + "version": "==24.4.2" }, "boto3": { "hashes": [ - "sha256:2824e3dd18743ca50e5b10439d20e74647b1416e8a94509cb30beac92d27a18d", - "sha256:b2e5cb5b95efcc881e25a3bc872d7a24e75ff4e76f368138e4baf7b9d6ee3422" + "sha256:b59355bf4a1408563969526f314611dbeacc151cf90ecb22af295dcc4fe18def", + "sha256:e39516e4ca21612932599819662759c04485d53ca457996a913163da11f052a4" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.90" + "version": "==1.34.93" }, "botocore": { "hashes": [ - "sha256:113cd4c0cb63e13163ccbc2bb13d551be314ba7f8ba5bfab1c51a19ca01aa133", - "sha256:d48f152498e2c60b43ce25b579d26642346a327b6fb2c632d57219e0a4f63392" + "sha256:6fbd5a53a2adc9b3d4ebd90ae0ede83a91a41d96231f8a5984051f75495f246d", + "sha256:79d39b0b87e962991c6dd55e78ce15155099f6fb741be88b1b8a456a702cc150" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.90" + "version": "==1.34.93" + }, + "cachetools": { + "hashes": [ + "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945", + "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105" + ], + "markers": "python_version >= '3.7'", + "version": "==5.3.3" }, "celery": { "extras": [ @@ -1137,6 +1145,40 @@ "markers": "python_version >= '3.7'", "version": "==3.1.43" }, + "google-api-core": { + "extras": [ + "grpc" + ], + "hashes": [ + "sha256:5a63aa102e0049abe85b5b88cb9409234c1f70afcda21ce1e40b285b9629c1d6", + "sha256:62d97417bfc674d6cef251e5c4d639a9655e00c45528c4364fbfebb478ce72a9" + ], + "markers": "python_version >= '3.7'", + "version": "==2.18.0" + }, + "google-auth": { + "hashes": [ + "sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360", + "sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415" + ], + "markers": "python_version >= '3.7'", + "version": "==2.29.0" + }, + "google-cloud-vision": { + "hashes": [ + "sha256:044330ad618c810333ff2296cd27ffd145f249638d1b35b270de6b460b00e8d2", + "sha256:a313088fcd9c016af0427c2447eea1ad01bc192ca4cc33babaace1be587bbf75" + ], + "version": "==3.7.2" + }, + "googleapis-common-protos": { + "hashes": [ + "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e", + "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632" + ], + "markers": "python_version >= '3.7'", + "version": "==1.63.0" + }, "gotrue": { "hashes": [ "sha256:64cd40933d1f0a5d5cc4f4bd93bc51d730b94812447b6600f774790a4901e455", @@ -1208,6 +1250,72 @@ ], "version": "==3.0.3" }, + "grpcio": { + "hashes": [ + "sha256:07ce1f775d37ca18c7a141300e5b71539690efa1f51fe17f812ca85b5e73262f", + "sha256:112eaa7865dd9e6d7c0556c8b04ae3c3a2dc35d62ad3373ab7f6a562d8199200", + "sha256:162ccf61499c893831b8437120600290a99c0bc1ce7b51f2c8d21ec87ff6af8b", + "sha256:16da954692fd61aa4941fbeda405a756cd96b97b5d95ca58a92547bba2c1624f", + "sha256:17708db5b11b966373e21519c4c73e5a750555f02fde82276ea2a267077c68ad", + "sha256:1bcfe5070e4406f489e39325b76caeadab28c32bf9252d3ae960c79935a4cc36", + "sha256:1c1bb80299bdef33309dff03932264636450c8fdb142ea39f47e06a7153d3063", + "sha256:2507006c8a478f19e99b6fe36a2464696b89d40d88f34e4b709abe57e1337467", + "sha256:262cda97efdabb20853d3b5a4c546a535347c14b64c017f628ca0cc7fa780cc6", + "sha256:26f415f40f4a93579fd648f48dca1c13dfacdfd0290f4a30f9b9aeb745026811", + "sha256:2a0204532aa2f1afd467024b02b4069246320405bc18abec7babab03e2644e75", + "sha256:2e72ddfee62430ea80133d2cbe788e0d06b12f865765cb24a40009668bd8ea05", + "sha256:3abe6838196da518863b5d549938ce3159d809218936851b395b09cad9b5d64a", + "sha256:3ad00f3f0718894749d5a8bb0fa125a7980a2f49523731a9b1fabf2b3522aa43", + "sha256:3c3ed41f4d7a3aabf0f01ecc70d6b5d00ce1800d4af652a549de3f7cf35c4abd", + "sha256:404d3b4b6b142b99ba1cff0b2177d26b623101ea2ce51c25ef6e53d9d0d87bcc", + "sha256:41955b641c34db7d84db8d306937b72bc4968eef1c401bea73081a8d6c3d8033", + "sha256:53d3a59a10af4c2558a8e563aed9f256259d2992ae0d3037817b2155f0341de1", + "sha256:55ddaf53474e8caeb29eb03e3202f9d827ad3110475a21245f3c7712022882a9", + "sha256:589ea8e75de5fd6df387de53af6c9189c5231e212b9aa306b6b0d4f07520fbb9", + "sha256:5dab7ac2c1e7cb6179c6bfad6b63174851102cbe0682294e6b1d6f0981ad7138", + "sha256:65034473fc09628a02fb85f26e73885cf1ed39ebd9cf270247b38689ff5942c5", + "sha256:66344ea741124c38588a664237ac2fa16dfd226964cca23ddc96bd4accccbde5", + "sha256:6e784f60e575a0de554ef9251cbc2ceb8790914fe324f11e28450047f264ee6f", + "sha256:80407bc007754f108dc2061e37480238b0dc1952c855e86a4fc283501ee6bb5d", + "sha256:82af3613a219512a28ee5c95578eb38d44dd03bca02fd918aa05603c41018051", + "sha256:88b4f9ee77191dcdd8810241e89340a12cbe050be3e0d5f2f091c15571cd3930", + "sha256:99701979bcaaa7de8d5f60476487c5df8f27483624f1f7e300ff4669ee44d1f2", + "sha256:a1511a303f8074f67af4119275b4f954189e8313541da7b88b1b3a71425cdb10", + "sha256:a5eb4844e5e60bf2c446ef38c5b40d7752c6effdee882f716eb57ae87255d20a", + "sha256:a75af2fc7cb1fe25785be7bed1ab18cef959a376cdae7c6870184307614caa3f", + "sha256:a90ac47a8ce934e2c8d71e317d2f9e7e6aaceb2d199de940ce2c2eb611b8c0f4", + "sha256:aa787b83a3cd5e482e5c79be030e2b4a122ecc6c5c6c4c42a023a2b581fdf17b", + "sha256:aaae70364a2d1fb238afd6cc9fcb10442b66e397fd559d3f0968d28cc3ac929c", + "sha256:af15e9efa4d776dfcecd1d083f3ccfb04f876d613e90ef8432432efbeeac689d", + "sha256:af7dc3f7a44f10863b1b0ecab4078f0a00f561aae1edbd01fd03ad4dcf61c9e9", + "sha256:b7ec9e2f8ffc8436f6b642a10019fc513722858f295f7efc28de135d336ac189", + "sha256:b94d41b7412ef149743fbc3178e59d95228a7064c5ab4760ae82b562bdffb199", + "sha256:c1624aa686d4b36790ed1c2e2306cc3498778dffaf7b8dd47066cf819028c3ad", + "sha256:c5ffeb269f10cedb4f33142b89a061acda9f672fd1357331dbfd043422c94e9e", + "sha256:c6ad9c39704256ed91a1cffc1379d63f7d0278d6a0bad06b0330f5d30291e3a3", + "sha256:c772f225483905f675cb36a025969eef9712f4698364ecd3a63093760deea1bc", + "sha256:c77618071d96b7a8be2c10701a98537823b9c65ba256c0b9067e0594cdbd954d", + "sha256:c79b518c56dddeec79e5500a53d8a4db90da995dfe1738c3ac57fe46348be049", + "sha256:cfd23ad29bfa13fd4188433b0e250f84ec2c8ba66b14a9877e8bce05b524cf54", + "sha256:d0695ae31a89f1a8fc8256050329a91a9995b549a88619263a594ca31b76d756", + "sha256:d2c1771d0ee3cf72d69bb5e82c6a82f27fbd504c8c782575eddb7839729fbaad", + "sha256:da6a7b6b938c15fa0f0568e482efaae9c3af31963eec2da4ff13a6d8ec2888e4", + "sha256:db068bbc9b1fa16479a82e1ecf172a93874540cb84be69f0b9cb9b7ac3c82670", + "sha256:db707e3685ff16fc1eccad68527d072ac8bdd2e390f6daa97bc394ea7de4acea", + "sha256:e2cc8a308780edbe2c4913d6a49dbdb5befacdf72d489a368566be44cadaef1a", + "sha256:f27246d7da7d7e3bd8612f63785a7b0c39a244cf14b8dd9dd2f2fab939f2d7f1", + "sha256:f4aa94361bb5141a45ca9187464ae81a92a2a135ce2800b2203134f7a1a1d479", + "sha256:fa63245271920786f4cb44dcada4983a3516be8f470924528cf658731864c14b" + ], + "version": "==1.62.2" + }, + "grpcio-status": { + "hashes": [ + "sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f", + "sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a" + ], + "version": "==1.62.2" + }, "h11": { "hashes": [ "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", @@ -1570,12 +1678,12 @@ }, "langchain-cohere": { "hashes": [ - "sha256:5a8e01d5e612dc0fdb55bcfb1ccc0f8754c22af97f5e82a961b3ab2e4565b9e9", - "sha256:c501b290a8de57d98c24cf7be2a537246342e958d132b87629b385fc1f3f1f32" + "sha256:59058032158c9d94ff251bc9264cd69249878e6d4b1b5ed80f66c4f56e23ac9f", + "sha256:c50b33fb6dc6552f696f03e4e2b9ab7bd5ffea4d8c14c37771dc765e9122dde8" ], "index": "pypi", "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.1.3" + "version": "==0.1.4" }, "langchain-community": { "hashes": [ @@ -1588,20 +1696,20 @@ }, "langchain-core": { "hashes": [ - "sha256:526532c1af279a9e2debe7a4e143ba6e980cf90b5ab2e0991c2230ee04c693e2", - "sha256:91eff20de0bcf5f025e1d8c4582cb597a9c17527965eb03b314486e7c834e7df" + "sha256:17c416349f5c7a9808e70e3725749a3a2df5088f1ecca045c883871aa95f9c9e", + "sha256:1c0befcd2665dd4aa153318aa9bf729071644b4c179e491769b8e583b4bf7441" ], "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.1.45" + "version": "==0.1.46" }, "langchain-openai": { "hashes": [ - "sha256:7f6e377d6bf88d6c2b1969fe5eecc1326271757512739e2f17c855cd7af53345", - "sha256:fa1f27815649291447e5370cb08e2f5a84e5c7c6121d0c055a6e296bd16d1e47" + "sha256:1a3220464c270d73ea3987010617789adc2099d4d4740b15c7734ab07e1f054b", + "sha256:a349ada8724921e380aab03ee312568f5ca99adbc806f6878d79ff9cd1d6d353" ], "index": "pypi", "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.1.3" + "version": "==0.1.4" }, "langchain-text-splitters": { "hashes": [ @@ -1620,20 +1728,20 @@ }, "langfuse": { "hashes": [ - "sha256:84cd3e5b884fc47ba1d162f5fd66b226df09badfbc64f9c4ebb222ff3282abe9", - "sha256:8de6fa69207b98c02e5816861e0b3f73baabb057cbd4309efac8725254dad957" + "sha256:6556dcd942ed596c6cbcd688bfed2aeca37065b68b9009f5e4ed829a32a9e513", + "sha256:7b77f188fd72261473fb329151a15ff676c49261b5084eb25a3ded49d1577234" ], "index": "pypi", "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==2.27.1" + "version": "==2.27.2" }, "langsmith": { "hashes": [ - "sha256:9fd22df8c689c044058536ea5af66f5302067e7551b60d7a335fede8d479572b", - "sha256:a81e9809fcaa277bfb314d729e58116554f186d1478fcfdf553b1c2ccce54b85" + "sha256:1e7363a3f472ecf02a1d91f6dbacde25519554b98c490be71716fcffaab0ca6b", + "sha256:b99b40a8c00e66174540865caa61412622fa1dc4f02602965364919c90528f97" ], "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.1.50" + "version": "==0.1.51" }, "layoutparser": { "extras": [ @@ -1649,21 +1757,21 @@ }, "litellm": { "hashes": [ - "sha256:907230b7ff57c853e32d04274c2bb01f75e77d49220bd3d4d8fa02cfe6d3492a", - "sha256:be0f9452fa357996e194c88eebc94f742be2fa623afd137a91b1e60ce5c3821f" + "sha256:8691fddef3c14733dfef2f34c11daf47a317790a258c5315eabe7d1eafbfeb24", + "sha256:bf8b437201bbcde50f8508f8712dd7ac7f323a1cdc2628aba0d35a4c312a801d" ], "index": "pypi", "markers": "python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8'", - "version": "==1.35.21" + "version": "==1.35.31" }, "llama-index": { "hashes": [ - "sha256:2d79ce49ed6f7e55f36a19cc60b4b8c07d622f98aa5945e1c0bdf82f56bd0afc", - "sha256:68476fb34364cdfcaccd4e09114d7489b13db431f4987c921d9a848893549215" + "sha256:d39b8f55491e51fdcf610e277580089a6e00f73669c9fd7dd78c5264e9c9c584", + "sha256:d77ab29883b05cc57adfa50fa44ababf2b7bc62a988fb6ed0b6460179e67a10b" ], "index": "pypi", "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.10.31" + "version": "==0.10.33" }, "llama-index-agent-openai": { "hashes": [ @@ -1683,27 +1791,27 @@ }, "llama-index-core": { "hashes": [ - "sha256:66d39d6f253e20311a21e0b98ea386089f099be12f2d23dbe11379a6d908ddf1", - "sha256:b894680fa320a94de56d9a933ac7edb646cabf15fe67ae1cf8fa53ac52ab4542" + "sha256:21b98b2c45e0c6b673aa505c7add1e8b730f472ad58d4572b909a34f4a22c36c", + "sha256:943114fb02dfe62fec5d882d749ad8adf113081aadcb0d4cb2c083b2c9052ed0" ], "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.10.31" + "version": "==0.10.33" }, "llama-index-embeddings-openai": { "hashes": [ - "sha256:186be52c29a8c82a39ff6e033e3a61b9c5a17cf6177a5507abe4f5c79fea3a53", - "sha256:aae4ce3ec2fe2cb604d944646982c71663a8b99ed5f8c370823ee82eef4ddd3b" + "sha256:0fd292b2f9a0ad4534a790d6374726bc885853188087eb018167dcf239643924", + "sha256:fbd16d6197b91f4dbdc6d0707e573cc224ac2b0a48d5b370c6232dd8a2282473" ], "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.1.8" + "version": "==0.1.9" }, "llama-index-indices-managed-llama-cloud": { "hashes": [ - "sha256:47cdde9f06bbddd508f0efcf41de425e85171ac2c8fda8a5fb2a89673e1c8c71", - "sha256:79f636cb6f4fabb12fec153564110f7f4dfda3cacb087793a5fec988484d7d2c" + "sha256:74b3b0e9ebf9d348d3054f9fc0c657031acceb9351c31116ad8d5a7ae4729f5c", + "sha256:cba33e1a3677b2a2ae7f239119acbf6dc3818f105edc92315729842b56fbc949" ], "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.1.5" + "version": "==0.1.6" }, "llama-index-legacy": { "hashes": [ @@ -1763,20 +1871,20 @@ }, "llama-parse": { "hashes": [ - "sha256:2c08962b66791c61fc360ae2042f953729c7b8decc3590d01fea5a98ca1f6676", - "sha256:d723af84d6a1fc99eb431915d21865d20b76d8a246dbaa124d1f96c956a644f7" + "sha256:5ce0390141f216dcd88c1123fea7f2a4f561d177f791a97217a3db3509dec4ff", + "sha256:fa04c09730b102155f6505de9cf91998c86d334581f0f12597c5eb47ca5db859" ], "index": "pypi", "markers": "python_version < '4.0' and python_full_version >= '3.8.1'", - "version": "==0.4.1" + "version": "==0.4.2" }, "llamaindex-py-client": { "hashes": [ - "sha256:091ee49a92592e3894777ade12516c2137093f9d6441a549f406461917ce9b7e", - "sha256:5417e41666504a77ecf5bdd9b403ffff1d714880ee30d49e234fb7686177eeeb" + "sha256:73f74792bb8c092bae6dc626627a09ac13a099fa8d10f8fcc83e17a2b332cca7", + "sha256:fd9416fd78b97209bf323bc3c7fab314499778563e7274f10853ad560563d10e" ], "markers": "python_version >= '3.8' and python_version < '4'", - "version": "==0.1.18" + "version": "==0.1.19" }, "lxml": { "extras": [ @@ -2381,12 +2489,12 @@ }, "openai": { "hashes": [ - "sha256:6730b8468a0235e5f289dfdfacaa374001645099c4ad1740b58eab378bcf7723", - "sha256:6eef764a8870095d256d59e6be243acf560a21227e9e3588b508972818929ef7" + "sha256:612de2d54cf580920a1156273f84aada6b3dca26d048f62eb5364a4314d7f449", + "sha256:f406c76ba279d16b9aca5a89cee0d968488e39f671f4dc6f0d690ac3c6f6fca1" ], "index": "pypi", "markers": "python_full_version >= '3.7.1'", - "version": "==1.23.3" + "version": "==1.23.6" }, "opencv-python": { "hashes": [ @@ -2516,12 +2624,12 @@ }, "pandasai": { "hashes": [ - "sha256:6c87c622d6bf2063964fd558a72011e76bb389e26622203b88c0420060847918", - "sha256:df19c8cb0ab1c750fafb264aa1edd39e964c64d44c5dfddf876691545d61163b" + "sha256:1ab6bd9fdda64bf5e100f8ffbf7da6cd7b072d2d169fb96fb524530c03c185bd", + "sha256:a110cf52440f2124db91d3286066bf7701043a1fd3d64aadcb38e4bdaec947b8" ], "index": "pypi", "markers": "python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8' and python_version >= '3.9'", - "version": "==2.0.35" + "version": "==2.0.36" }, "pathspec": { "hashes": [ @@ -2784,22 +2892,30 @@ "markers": "python_full_version >= '3.7.0'", "version": "==3.0.43" }, + "proto-plus": { + "hashes": [ + "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2", + "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c" + ], + "markers": "python_version >= '3.6'", + "version": "==1.23.0" + }, "protobuf": { "hashes": [ - "sha256:38aa5f535721d5bb99861166c445c4105c4e285c765fbb2ac10f116e32dcd46d", - "sha256:3c388ea6ddfe735f8cf69e3f7dc7611e73107b60bdfcf5d0f024c3ccd3794e23", - "sha256:7ee014c2c87582e101d6b54260af03b6596728505c79f17c8586e7523aaa8f8c", - "sha256:8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51", - "sha256:9b557c317ebe6836835ec4ef74ec3e994ad0894ea424314ad3552bc6e8835b4e", - "sha256:b9ba3ca83c2e31219ffbeb9d76b63aad35a3eb1544170c55336993d7a18ae72c", - "sha256:d693d2504ca96750d92d9de8a103102dd648fda04540495535f0fec7577ed8fc", - "sha256:da612f2720c0183417194eeaa2523215c4fcc1a1949772dc65f05047e08d5932", - "sha256:e6039957449cb918f331d32ffafa8eb9255769c96aa0560d9a5bf0b4e00a2a33", - "sha256:f7417703f841167e5a27d48be13389d52ad705ec09eade63dfc3180a959215d7", - "sha256:fbfe61e7ee8c1860855696e3ac6cfd1b01af5498facc6834fcc345c9684fb2ca" + "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4", + "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8", + "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c", + "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d", + "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4", + "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa", + "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c", + "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019", + "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9", + "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c", + "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2" ], "markers": "python_version >= '3.8'", - "version": "==5.26.1" + "version": "==4.25.3" }, "psutil": { "hashes": [ @@ -2988,6 +3104,14 @@ "markers": "python_version >= '3.8'", "version": "==0.6.0" }, + "pyasn1-modules": { + "hashes": [ + "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6", + "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b" + ], + "markers": "python_version >= '3.8'", + "version": "==0.4.0" + }, "pycocotools": { "hashes": [ "sha256:0ac4f30bac1503c780072053e6922971392fa3628b2e6967192bfca1f14736e2", @@ -3181,6 +3305,73 @@ "markers": "python_version >= '3.8'", "version": "==3.2.0" }, + "pyinstrument": { + "hashes": [ + "sha256:0002ee517ed8502bbda6eb2bb1ba8f95a55492fcdf03811ba13d4806e50dd7f6", + "sha256:01fc45dedceec3df81668d702bca6d400d956c8b8494abc206638c167c78dfd9", + "sha256:06a8578b2943eb1dbbf281e1e59e44246acfefd79e1b06d4950f01b693de12af", + "sha256:08fdc7f88c989316fa47805234c37a40fafe7b614afd8ae863f0afa9d1707b37", + "sha256:0de2c1714a37a820033b19cf134ead43299a02662f1379140974a9ab733c5f3a", + "sha256:113d2fc534c9ca7b6b5661d6ada05515bf318f6eb34e8d05860fe49eb7cfe17e", + "sha256:1e474c56da636253dfdca7cd1998b240d6b39f7ed34777362db69224fcf053b1", + "sha256:20e15b4e1d29ba0b7fc81aac50351e0dc0d7e911e93771ebc3f408e864a2c93b", + "sha256:23c3e3ca8553b9aac09bd978c73d21b9032c707ac6d803bae6a20ecc048df4a8", + "sha256:28af084aa84bbfd3620ebe71d5f9a0deca4451267f363738ca824f733de55056", + "sha256:2e625fc6ffcd4fd420493edd8276179c3f784df207bef4c2192725c1b310534c", + "sha256:2fd8e547cf3df5f0ec6e4dffbe2e857f6b28eda51b71c3c0b5a2fc0646527835", + "sha256:3098cd72b71a322a72dafeb4ba5c566465e193d2030adad4c09566bd2f89bf4f", + "sha256:32ec8db6896b94af790a530e1e0edad4d0f941a0ab8dd9073e5993e7ea46af7d", + "sha256:34e59e91c88ec9ad5630c0964eca823949005e97736bfa838beb4789e94912a2", + "sha256:3a165e0d2deb212d4cf439383982a831682009e1b08733c568cac88c89784e62", + "sha256:46992e855d630575ec635eeca0068a8ddf423d4fd32ea0875a94e9f8688f0b95", + "sha256:4fba3244e94c117bf4d9b30b8852bbdcd510e7329fdd5c7c8b3799e00a9215a8", + "sha256:5b6e161ef268d43ee6bbfae7fd2cdd0a52c099ddd21001c126ca1805dc906539", + "sha256:5ebeba952c0056dcc9b9355328c78c4b5c2a33b4b4276a9157a3ab589f3d1bac", + "sha256:5f329f5534ca069420246f5ce57270d975229bcb92a3a3fd6b2ca086527d9764", + "sha256:62f6014d2b928b181a52483e7c7b82f2c27e22c577417d1681153e5518f03317", + "sha256:67268bb0d579330cff40fd1c90b8510363ca1a0e7204225840614068658dab77", + "sha256:6ba8e368d0421f15ba6366dfd60ec131c1b46505d021477e0f865d26cf35a605", + "sha256:6c0f0e1d8f8c70faa90ff57f78ac0dda774b52ea0bfb2d9f0f41ce6f3e7c869e", + "sha256:6c761372945e60fc1396b7a49f30592e8474e70a558f1a87346d27c8c4ce50f7", + "sha256:6ed4e8c6c84e0e6429ba7008a66e435ede2d8cb027794c20923c55669d9c5633", + "sha256:73db0c2c99119c65b075feee76e903b4ed82e59440fe8b5724acf5c7cb24721f", + "sha256:7a1b1cd768ea7ea9ab6f5490f7e74431321bcc463e9441dbc2f769617252d9e2", + "sha256:7ba858b3d6f6e5597c641edcc0e7e464f85aba86d71bc3b3592cb89897bf43f6", + "sha256:7bd3da31c46f1c1cb7ae89031725f6a1d1015c2041d9c753fe23980f5f9fd86c", + "sha256:7c671057fad22ee3ded897a6a361204ea2538e44c1233cad0e8e30f6d27f33db", + "sha256:803ac64e526473d64283f504df3b0d5c2c203ea9603cab428641538ffdc753a7", + "sha256:8a386b9d09d167451fb2111eaf86aabf6e094fed42c15f62ec51d6980bce7d96", + "sha256:8a9791bf8916c1cf439c202fded32de93354b0f57328f303d71950b0027c7811", + "sha256:8b3c44cb037ad0d6e9d9a48c14d856254ada641fbd0ae9de40da045fc2226a2a", + "sha256:8d104b7a7899d5fa4c5bf1ceb0c1a070615a72c5dc17bc321b612467ad5c5d88", + "sha256:90350533396071cb2543affe01e40bf534c35cb0d4b8fa9fdb0f052f9ca2cfe3", + "sha256:a59fc4f7db738a094823afe6422509fa5816a7bf74e768ce5a7a2ddd91af40ac", + "sha256:af1a953bce9fd530040895d01ff3de485e25e1576dccb014f76ba9131376fcad", + "sha256:b082df0bbf71251a7f4880a12ed28421dba84ea7110bb376e0533067a4eaff40", + "sha256:b2b66ff0b16c8ecf1ec22de001cfff46872b2c163c62429055105564eef50b2e", + "sha256:b55983a884f083f93f0fc6d12ff8df0acd1e2fb0580d2f4c7bfe6def33a84b58", + "sha256:baf375953b02fe94d00e716f060e60211ede73f49512b96687335f7071adb153", + "sha256:be9901f17ac2f527c352f2fdca3d717c1d7f2ce8a70bad5a490fc8cc5d2a6007", + "sha256:cd0320c39e99e3c0a3129d1ed010ac41e5a7eb96fb79900d270080a97962e995", + "sha256:d02f31fa13a9e8dc702a113878419deba859563a32474c9f68e04619d43d6f01", + "sha256:d4b559322f30509ad8f082561792352d0805b3edfa508e492a36041fdc009259", + "sha256:d4dcdcc7ba224a0c5edfbd00b0f530f5aed2b26da5aaa2f9af5519d4aa8c7e41", + "sha256:d6162615e783c59e36f2d7caf903a7e3ecb6b32d4a4ae8907f2760b2ef395bf6", + "sha256:da58f265326f3cf3975366ccb8b39014f1e69ff8327958a089858d71c633d654", + "sha256:dcb5c8d763c5df55131670ba2a01a8aebd0d490a789904a55eb6a8b8d497f110", + "sha256:dd5c53a0159126b5ce7cbc4994433c9c671e057c85297ff32645166a06ad2c50", + "sha256:dd6007d3c2e318e09e582435dd8d111cccf30d342af66886b783208813caf3d7", + "sha256:e2e554b1bb0df78f5ce8a92df75b664912ca93aa94208386102af454ec31b647", + "sha256:e3813c8ecfab9d7d855c5f0f71f11793cf1507f40401aa33575c7fd613577c23", + "sha256:e63f4916001aa9c625976a50779282e0a5b5e9b17c52a50ef4c651e468ed5b88", + "sha256:edca46f04a573ac2fb11a84b937844e6a109f38f80f4b422222fb5be8ecad8cb", + "sha256:fdc0a53b27e5d8e47147489c7dab596ddd1756b1e053217ef5bc6718567099ff", + "sha256:feebcf860f955401df30d029ec8de7a0c5515d24ea809736430fd1219686fe14" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==4.6.2" + }, "pypandoc": { "hashes": [ "sha256:31652073c7960c2b03570bd1e94f602ca9bc3e70099df5ead4cea98ff5151c1e", @@ -3253,12 +3444,12 @@ }, "pytest": { "hashes": [ - "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7", - "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044" + "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233", + "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==8.1.1" + "version": "==8.2.0" }, "pytest-celery": { "hashes": [ @@ -3320,11 +3511,11 @@ }, "python-iso639": { "hashes": [ - "sha256:7b149623ff74230f4ee3061fb01d18e57a8d07c5fee2aa72907f39b7f6d16cbc", - "sha256:c323233348c34d57c601e3e6d824088e492896bcb97a61a87f7d93401a305377" + "sha256:27526a84cebc4c4d53fea9d1ebbc7209c8d279bebaa343e6765a1fc8780565ab", + "sha256:97e63b5603e085c6a56a12a95740010e75d9134e0aab767e0978b53fd8824f13" ], "markers": "python_version >= '3.8'", - "version": "==2024.2.7" + "version": "==2024.4.27" }, "python-jose": { "hashes": [ @@ -3831,10 +4022,11 @@ "fastapi" ], "hashes": [ - "sha256:1ce29e30240cc289a027011103a8c83885b15ef2f316a60bcc7c5300afa144f1", - "sha256:509aa9678c0512344ca886281766c2e538682f8acfa50fd8d405f8c417ad0625" + "sha256:b54c54a2160f509cf2757260d0cf3885b608c6192c2555a3857e3a4d0f84bdb3", + "sha256:c278e0f523f6f0ee69dc43ad26dcdb1202dffe5ac326ae31472e012d941bee21" ], - "version": "==1.45.0" + "markers": "python_version >= '3.6'", + "version": "==2.0.1" }, "setuptools": { "hashes": [ @@ -4184,63 +4376,53 @@ }, "torch": { "hashes": [ - "sha256:11e8fe261233aeabd67696d6b993eeb0896faa175c6b41b9a6c9f0334bdad1c5", - "sha256:15dffa4cc3261fa73d02f0ed25f5fa49ecc9e12bf1ae0a4c1e7a88bbfaad9030", - "sha256:31f4310210e7dda49f1fb52b0ec9e59382cfcb938693f6d5378f25b43d7c1d29", - "sha256:32827fa1fbe5da8851686256b4cd94cc7b11be962862c2293811c94eea9457bf", - "sha256:3a4dd910663fd7a124c056c878a52c2b0be4a5a424188058fe97109d4436ee42", - "sha256:3dbcd563a9b792161640c0cffe17e3270d85e8f4243b1f1ed19cca43d28d235b", - "sha256:451331406b760f4b1ab298ddd536486ab3cfb1312614cfe0532133535be60bea", - "sha256:49aa4126ede714c5aeef7ae92969b4b0bbe67f19665106463c39f22e0a1860d1", - "sha256:539d5ef6c4ce15bd3bd47a7b4a6e7c10d49d4d21c0baaa87c7d2ef8698632dfb", - "sha256:67dcd726edff108e2cd6c51ff0e416fd260c869904de95750e80051358680d24", - "sha256:877b3e6593b5e00b35bbe111b7057464e76a7dd186a287280d941b564b0563c2", - "sha256:89ddac2a8c1fb6569b90890955de0c34e1724f87431cacff4c1979b5f769203c", - "sha256:95b9b44f3bcebd8b6cd8d37ec802048c872d9c567ba52c894bba90863a439059", - "sha256:a6e5770d68158d07456bfcb5318b173886f579fdfbf747543901ce718ea94782", - "sha256:ad4c03b786e074f46606f4151c0a1e3740268bcf29fbd2fdf6666d66341c1dcb", - "sha256:b2e2200b245bd9f263a0d41b6a2dab69c4aca635a01b30cca78064b0ef5b109e", - "sha256:b421448d194496e1114d87a8b8d6506bce949544e513742b097e2ab8f7efef32", - "sha256:bc889d311a855dd2dfd164daf8cc903a6b7273a747189cebafdd89106e4ad585", - "sha256:bf9558da7d2bf7463390b3b2a61a6a3dbb0b45b161ee1dd5ec640bf579d479fc", - "sha256:c795feb7e8ce2e0ef63f75f8e1ab52e7fd5e1a4d7d0c31367ade1e3de35c9e95", - "sha256:cd2bf7697c9e95fb5d97cc1d525486d8cf11a084c6af1345c2c2c22a6b0029d0", - "sha256:cf12cdb66c9c940227ad647bc9cf5dba7e8640772ae10dfe7569a0c1e2a28aca", - "sha256:dff696de90d6f6d1e8200e9892861fd4677306d0ef604cb18f2134186f719f82", - "sha256:eb4d6e9d3663e26cd27dc3ad266b34445a16b54908e74725adb241aa56987533", - "sha256:f9ef0a648310435511e76905f9b89612e45ef2c8b023bee294f5e6f7e73a3e7c" + "sha256:09c81c5859a5b819956c6925a405ef1cdda393c9d8a01ce3851453f699d3358c", + "sha256:1bf023aa20902586f614f7682fedfa463e773e26c58820b74158a72470259459", + "sha256:20572f426965dd8a04e92a473d7e445fa579e09943cc0354f3e6fef6130ce061", + "sha256:493d54ee2f9df100b5ce1d18c96dbb8d14908721f76351e908c9d2622773a788", + "sha256:4fb27b35dbb32303c2927da86e27b54a92209ddfb7234afb1949ea2b3effffea", + "sha256:5515503a193781fd1b3f5c474e89c9dfa2faaa782b2795cc4a7ab7e67de923f6", + "sha256:6ae9f64b09516baa4ef890af0672dc981c20b1f0d829ce115d4420a247e88fba", + "sha256:729804e97b7cf19ae9ab4181f91f5e612af07956f35c8b2c8e9d9f3596a8e877", + "sha256:758ef938de87a2653bba74b91f703458c15569f1562bf4b6c63c62d9c5a0c1f5", + "sha256:760f8bedff506ce9e6e103498f9b1e9e15809e008368594c3a66bf74a8a51380", + "sha256:a306c87a3eead1ed47457822c01dfbd459fe2920f2d38cbdf90de18f23f72542", + "sha256:b0de2bdc0486ea7b14fc47ff805172df44e421a7318b7c4d92ef589a75d27410", + "sha256:bce43af735c3da16cc14c7de2be7ad038e2fbf75654c2e274e575c6c05772ace", + "sha256:cd0dc498b961ab19cb3f8dbf0c6c50e244f2f37dbfa05754ab44ea057c944ef9", + "sha256:d24e328226d8e2af7cf80fcb1d2f1d108e0de32777fab4aaa2b37b9765d8be73", + "sha256:d8ea5a465dbfd8501f33c937d1f693176c9aef9d1c1b0ca1d44ed7b0a18c52ac", + "sha256:dca986214267b34065a79000cee54232e62b41dff1ec2cab9abc3fc8b3dee0ad", + "sha256:e05f836559251e4096f3786ee99f4a8cbe67bc7fbedba8ad5e799681e47c5e80", + "sha256:e65ba85ae292909cde0dde6369826d51165a3fc8823dc1854cd9432d7f79b932", + "sha256:f9b98bf1a3c8af2d4c41f0bf1433920900896c446d1ddc128290ff146d1eb4bd" ], - "version": "==2.2.2" + "version": "==2.3.0" }, "torchvision": { "hashes": [ - "sha256:067095e87a020a7a251ac1d38483aa591c5ccb81e815527c54db88a982fc9267", - "sha256:14fd1d4a033c325bdba2d03a69c3450cab6d3a625f85cc375781d9237ca5d04d", - "sha256:1ece40557e122d79975860a005aa7e2a9e2e6c350a03e78a00ec1450083312fd", - "sha256:1f2910fe3c21ad6875b2720d46fad835b2e4b336e9553d31ca364d24c90b1d4f", - "sha256:2f69570f50b1d195e51bc03feffb7b7728207bc36efcfb1f0813712b2379d881", - "sha256:32dbeba3987e20f2dc1bce8d1504139fff582898346dfe8ad98d649f97ca78fa", - "sha256:35ba5c1600c3203549d2316422a659bd20c0cfda1b6085eec94fb9f35f55ca43", - "sha256:3bbc24b7713e8f22766992562547d8b4b10001208d372fe599255af84bfd1a69", - "sha256:3f784381419f3ed3f2ec2aa42fb4aeec5bf4135e298d1631e41c926e6f1a0dff", - "sha256:4868bbfa55758c8107e69a0e7dd5e77b89056035cd38b767ad5b98cdb71c0f0d", - "sha256:6835897df852fad1015e6a106c167c83848114cbcc7d86112384a973404e4431", - "sha256:77d680adf6ce367166a186d2c7fda3a73807ab9a03b2c31a03fa8812c8c5335b", - "sha256:7dc85b397f6c6d9ef12716ce0d6e11ac2b803f5cccff6fe3966db248e7774478", - "sha256:833fd2e4216ced924c8aca0525733fe727f9a1af66dfad7c5be7257e97c39678", - "sha256:9b83e55ee7d0a1704f52b9c0ac87388e7a6d1d98a6bde7b0b35f9ab54d7bda54", - "sha256:9c3acbebbe379af112b62b535820174277b1f3eed30df264a4e458d58ee4e5b2", - "sha256:b83aac8d78f48981146d582168d75b6c947cfb0a7693f76e219f1926f6e595a3", - "sha256:ba2e62f233eab3d42b648c122a3a29c47cc108ca314dfd5cbb59cd3a143fd623", - "sha256:d506854c5acd69b20a8b6641f01fe841685a21c5406b56813184f1c9fc94279e", - "sha256:e031004a1bc432c980a7bd642f6c189a3efc316e423fc30b5569837166a4e28d", - "sha256:e9e4bed404af33dfc92eecc2b513d21ddc4c242a7fd8708b3b09d3a26aa6f444", - "sha256:ecc1c503fa8a54fbab777e06a7c228032b8ab78efebf35b28bc8f22f544f51f1", - "sha256:efd6d0dd0668e15d01a2cffadc74068433b32cbcf5692e0c4aa15fc5cb250ce7", - "sha256:f1c9ab3152cfb27f83aca072cac93a3a4c4e4ab0261cf0f2d516b9868a4e96f3", - "sha256:f400145fc108833e7c2fc28486a04989ca742146d7a2a2cc48878ebbb40cdbbd" + "sha256:2115a1906c015f5da9ceedc40a983313b0fd6e2c8a17108a92991706f51f6987", + "sha256:36efd87001c6bee2383e043e46a025affb03179747c8f4777b9918527ffce756", + "sha256:3d7955398d4ceaad77c487c2c44f6f7813112402c9bab8cd906d346005891048", + "sha256:493c45f9937dad37aa1b64b14da17c7a589c72b91adc4837d431009cfe29bd53", + "sha256:4c334b3e719ba0a9ba6e15d4aff1178f5e6d029174f346163fed525f0ccfffd3", + "sha256:5337f6acfa1fe959d5cb340d01a00614d6b31ce7a4824ccb95435a85c5273b95", + "sha256:6323f7e5423ff2594d5891863b919deb9d0de95f01c36bf26fbd879036b6ed08", + "sha256:6896a52168befe1105fb3c9335287390ed227e71d1e4ec4d68b62e8a3099fc09", + "sha256:6ad70ddfa879bda5ed886b2518fe562640e0059787cbd65cb2bffa7674541410", + "sha256:75e22ecf44a13b8f95b8ad421c0261282d859c61816badaca1959e073ccdd691", + "sha256:7c770f0f748e0b17f57c0297508d7254f686cdf03fc2e2949f422b20574f4c0f", + "sha256:925d0a82cccf6f986c18b29b4392a942db65cbdb73c13a129c8493822eb9e36f", + "sha256:95b42d0dc599b47a01530c7439a5751e67e45b85e3a67113989cf7c7c70f2039", + "sha256:a964afbc7ddf50a46b941477f6c35729b416deedd139756befd488245e2e226d", + "sha256:b657d052d146f24cb3b2a78219bfc82ae70a9706671c50f632528907d10cccec", + "sha256:bd8e6f3b5beb49965f15c461302488edfa3d8c2d01d3bb79b150d6fb62711e3a", + "sha256:ccc292e093771d5baacf5535ac4416306b6b5f15676341cd4d010d8542eace25", + "sha256:dd61628a3d189c6852a12dc5ed4cd2eece66d2d67f35a866cb16f1dcb06c8c62", + "sha256:e5a24d620cea14a4bb89f24aa2b506230c0a16a3ada57fc53ad80cfd256a2128", + "sha256:eb9d83c0e1dbb54ecb0fb04c87f786333e3a6fb8b9c400aca7c31081f9aa5707" ], - "version": "==0.17.2" + "version": "==0.18.0" }, "tornado": { "hashes": [ @@ -4320,12 +4502,12 @@ "all-docs" ], "hashes": [ - "sha256:190b51fbd282602e68b44708798dedd85479a718e4e50a300f12abaa5643df32", - "sha256:852f8db24d82568fbd26c0a90ab85803d2dcb507f8b64a56932455a46b543422" + "sha256:5dbd091a6503b3624f7419874de68667cd16bf55354390467ed2c5a4fb937554", + "sha256:bbc9ca6d6c134631934ad7f751eeb3f6f0b9b8746626e05558c7639548f9d46e" ], "index": "pypi", "markers": "python_version < '3.12' and python_full_version >= '3.9.0'", - "version": "==0.13.3" + "version": "==0.13.4" }, "unstructured-client": { "hashes": [ @@ -4337,10 +4519,10 @@ }, "unstructured-inference": { "hashes": [ - "sha256:a56d8b4434cd4e0f14f0293362296de5b4cbf16ba6524c00035e4cc58da6dc90", - "sha256:f1d5a534fb2c125757022d6cb4b2b40f0e0f99f09894ea23b82685f99b88d310" + "sha256:535e8e6de6daeb1cf6486cffc081feb7929915430edbb5cdcafbb698048933f7", + "sha256:75ff1c5496d17ab20525a5b5fee8fef5b14d61a4584f8705fd736f0a348d28ca" ], - "version": "==0.7.27" + "version": "==0.7.28" }, "unstructured.pytesseract": { "hashes": [ @@ -4817,31 +4999,31 @@ "develop": { "black": { "hashes": [ - "sha256:1bb9ca06e556a09f7f7177bc7cb604e5ed2d2df1e9119e4f7d2f1f7071c32e5d", - "sha256:21f9407063ec71c5580b8ad975653c66508d6a9f57bd008bb8691d273705adcd", - "sha256:4396ca365a4310beef84d446ca5016f671b10f07abdba3e4e4304218d2c71d33", - "sha256:44d99dfdf37a2a00a6f7a8dcbd19edf361d056ee51093b2445de7ca09adac965", - "sha256:5cd5b4f76056cecce3e69b0d4c228326d2595f506797f40b9233424e2524c070", - "sha256:64578cf99b6b46a6301bc28bdb89f9d6f9b592b1c5837818a177c98525dbe397", - "sha256:64e60a7edd71fd542a10a9643bf369bfd2644de95ec71e86790b063aa02ff745", - "sha256:652e55bb722ca026299eb74e53880ee2315b181dfdd44dca98e43448620ddec1", - "sha256:6644f97a7ef6f401a150cca551a1ff97e03c25d8519ee0bbc9b0058772882665", - "sha256:6ad001a9ddd9b8dfd1b434d566be39b1cd502802c8d38bbb1ba612afda2ef436", - "sha256:71d998b73c957444fb7c52096c3843875f4b6b47a54972598741fe9a7f737fcb", - "sha256:74eb9b5420e26b42c00a3ff470dc0cd144b80a766128b1771d07643165e08d0e", - "sha256:75a2d0b4f5eb81f7eebc31f788f9830a6ce10a68c91fbe0fade34fff7a2836e6", - "sha256:7852b05d02b5b9a8c893ab95863ef8986e4dda29af80bbbda94d7aee1abf8702", - "sha256:7f2966b9b2b3b7104fca9d75b2ee856fe3fdd7ed9e47c753a4bb1a675f2caab8", - "sha256:8e5537f456a22cf5cfcb2707803431d2feeb82ab3748ade280d6ccd0b40ed2e8", - "sha256:d4e71cdebdc8efeb6deaf5f2deb28325f8614d48426bed118ecc2dcaefb9ebf3", - "sha256:dae79397f367ac8d7adb6c779813328f6d690943f64b32983e896bcccd18cbad", - "sha256:e3a3a092b8b756c643fe45f4624dbd5a389f770a4ac294cf4d0fce6af86addaf", - "sha256:eb949f56a63c5e134dfdca12091e98ffb5fd446293ebae123d10fc1abad00b9e", - "sha256:f07b69fda20578367eaebbd670ff8fc653ab181e1ff95d84497f9fa20e7d0641", - "sha256:f95cece33329dc4aa3b0e1a771c41075812e46cf3d6e3f1dfe3d91ff09826ed2" + "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474", + "sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1", + "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0", + "sha256:48a85f2cb5e6799a9ef05347b476cce6c182d6c71ee36925a6c194d074336ef8", + "sha256:7768a0dbf16a39aa5e9a3ded568bb545c8c2727396d063bbaf847df05b08cd96", + "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1", + "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04", + "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021", + "sha256:98e123f1d5cfd42f886624d84464f7756f60ff6eab89ae845210631714f6db94", + "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d", + "sha256:b1530ae42e9d6d5b670a34db49a94115a64596bc77710b1d05e9801e62ca0a7c", + "sha256:b9176b9832e84308818a99a561e90aa479e73c523b3f77afd07913380ae2eab7", + "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c", + "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc", + "sha256:bf10f7310db693bb62692609b397e8d67257c55f949abde4c67f9cc574492cc7", + "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d", + "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c", + "sha256:da33a1a5e49c4122ccdfd56cd021ff1ebc4a1ec4e2d01594fef9b6f267a9e741", + "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce", + "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb", + "sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063", + "sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e" ], "markers": "python_version >= '3.8'", - "version": "==24.4.0" + "version": "==24.4.2" }, "click": { "hashes": [ diff --git a/backend/main.py b/backend/main.py index a58d87b90..ed35373f7 100644 --- a/backend/main.py +++ b/backend/main.py @@ -10,8 +10,8 @@ import logging import litellm import sentry_sdk -from fastapi import FastAPI, HTTPException -from fastapi.responses import JSONResponse +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse from logger import get_logger from middlewares.cors import add_cors_middleware from modules.analytics.controller.analytics_routes import analytics_router @@ -29,6 +29,7 @@ from modules.upload.controller import upload_router from modules.user.controller import user_router from packages.utils import handle_request_validation_error from packages.utils.telemetry import maybe_send_telemetry +from pyinstrument import Profiler from routes.crawl_routes import crawl_router from routes.subscription_routes import subscription_router from sentry_sdk.integrations.fastapi import FastApiIntegration @@ -91,6 +92,23 @@ app.include_router(notification_router) app.include_router(knowledge_router) app.include_router(contact_router) +PROFILING = os.getenv("PROFILING", "false").lower() == "true" + + +if PROFILING: + + @app.middleware("http") + async def profile_request(request: Request, call_next): + profiling = request.query_params.get("profile", False) + if profiling: + profiler = Profiler() + profiler.start() + await call_next(request) + profiler.stop() + return HTMLResponse(profiler.output_html()) + else: + return await call_next(request) + @app.exception_handler(HTTPException) async def http_exception_handler(_, exc): diff --git a/backend/packages/files/parsers/common.py b/backend/packages/files/parsers/common.py index 484916240..74aec76a1 100644 --- a/backend/packages/files/parsers/common.py +++ b/backend/packages/files/parsers/common.py @@ -1,3 +1,4 @@ +import re import time import tiktoken @@ -43,6 +44,11 @@ async def process_file( new_metadata = metadata.copy() # Add filename at beginning of page content doc.page_content = f"Filename: {new_metadata['original_file_name']} Content: {doc.page_content}" + + doc.page_content = doc.page_content.replace("\u0000", "") + # Replace unsupported Unicode characters + doc.page_content = re.sub(r"[^\x00-\x7F]+", " ", doc.page_content) + len_chunk = len(enc.encode(doc.page_content)) # Ensure the text is in UTF-8 diff --git a/backend/requirements.txt b/backend/requirements.txt index 6205e98db..36b0971b8 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,9 +16,10 @@ attrs==23.2.0; python_version >= '3.7' backoff==2.2.1; python_version >= '3.7' and python_version < '4.0' beautifulsoup4==4.12.3; python_full_version >= '3.6.0' billiard==4.2.0; python_version >= '3.7' -black==24.4.0; python_version >= '3.8' -boto3==1.34.90; python_version >= '3.8' -botocore==1.34.90; python_version >= '3.8' +black==24.4.2; python_version >= '3.8' +boto3==1.34.93; python_version >= '3.8' +botocore==1.34.93; python_version >= '3.8' +cachetools==5.3.3; python_version >= '3.7' celery[redis,sqs]==5.4.0; python_version >= '3.8' certifi==2024.2.2; python_version >= '3.6' cffi==1.16.0; platform_python_implementation != 'PyPy' @@ -71,8 +72,14 @@ frozenlist==1.4.1; python_version >= '3.8' fsspec[http]==2024.3.1; python_version >= '3.8' gitdb==4.0.11; python_version >= '3.7' gitpython==3.1.43; python_version >= '3.7' +google-api-core[grpc]==2.18.0; python_version >= '3.7' +google-auth==2.29.0; python_version >= '3.7' +google-cloud-vision==3.7.2 +googleapis-common-protos==1.63.0; python_version >= '3.7' gotrue==2.4.2; python_version >= '3.8' and python_version < '4.0' greenlet==3.0.3 +grpcio==1.62.2 +grpcio-status==1.62.2 h11==0.14.0; python_version >= '3.7' html5lib==1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' httpcore==1.0.5; python_version >= '3.8' @@ -96,22 +103,22 @@ jsonpointer==2.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3 kiwisolver==1.4.5; python_version >= '3.7' kombu[sqs]==5.3.7; python_version >= '3.8' langchain==0.1.16; python_version < '4.0' and python_full_version >= '3.8.1' -langchain-cohere==0.1.3; python_version < '4.0' and python_full_version >= '3.8.1' +langchain-cohere==0.1.4; python_version < '4.0' and python_full_version >= '3.8.1' langchain-community==0.0.34; python_version < '4.0' and python_full_version >= '3.8.1' -langchain-core==0.1.45; python_version < '4.0' and python_full_version >= '3.8.1' -langchain-openai==0.1.3; python_version < '4.0' and python_full_version >= '3.8.1' +langchain-core==0.1.46; python_version < '4.0' and python_full_version >= '3.8.1' +langchain-openai==0.1.4; python_version < '4.0' and python_full_version >= '3.8.1' langchain-text-splitters==0.0.1; python_version < '4.0' and python_full_version >= '3.8.1' langdetect==1.0.9 -langfuse==2.27.1; python_version < '4.0' and python_full_version >= '3.8.1' -langsmith==0.1.50; python_version < '4.0' and python_full_version >= '3.8.1' +langfuse==2.27.2; python_version < '4.0' and python_full_version >= '3.8.1' +langsmith==0.1.51; python_version < '4.0' and python_full_version >= '3.8.1' layoutparser[layoutmodels,tesseract]==0.3.4; python_version >= '3.6' -litellm==1.35.21; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8' -llama-index==0.10.31; python_version < '4.0' and python_full_version >= '3.8.1' +litellm==1.35.31; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8' +llama-index==0.10.33; python_version < '4.0' and python_full_version >= '3.8.1' llama-index-agent-openai==0.2.3; python_version < '4.0' and python_full_version >= '3.8.1' llama-index-cli==0.1.12; python_version < '4.0' and python_full_version >= '3.8.1' -llama-index-core==0.10.31; python_version < '4.0' and python_full_version >= '3.8.1' -llama-index-embeddings-openai==0.1.8; python_version < '4.0' and python_full_version >= '3.8.1' -llama-index-indices-managed-llama-cloud==0.1.5; python_version < '4.0' and python_full_version >= '3.8.1' +llama-index-core==0.10.33; python_version < '4.0' and python_full_version >= '3.8.1' +llama-index-embeddings-openai==0.1.9; python_version < '4.0' and python_full_version >= '3.8.1' +llama-index-indices-managed-llama-cloud==0.1.6; python_version < '4.0' and python_full_version >= '3.8.1' llama-index-legacy==0.9.48; python_version < '4.0' and python_full_version >= '3.8.1' llama-index-llms-openai==0.1.16; python_version < '4.0' and python_full_version >= '3.8.1' llama-index-multi-modal-llms-openai==0.1.5; python_version < '4.0' and python_full_version >= '3.8.1' @@ -119,8 +126,8 @@ llama-index-program-openai==0.1.6; python_version < '4.0' and python_full_versio llama-index-question-gen-openai==0.1.3; python_version < '4.0' and python_full_version >= '3.8.1' llama-index-readers-file==0.1.19; python_version < '4.0' and python_full_version >= '3.8.1' llama-index-readers-llama-parse==0.1.4; python_version < '4.0' and python_full_version >= '3.8.1' -llama-parse==0.4.1; python_version < '4.0' and python_full_version >= '3.8.1' -llamaindex-py-client==0.1.18; python_version >= '3.8' and python_version < '4' +llama-parse==0.4.2; python_version < '4.0' and python_full_version >= '3.8.1' +llamaindex-py-client==0.1.19; python_version >= '3.8' and python_version < '4' lxml[html_clean]==5.2.1; python_version >= '3.6' lxml-html-clean==0.1.1 markdown==3.6 @@ -144,14 +151,14 @@ olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, omegaconf==2.3.0; python_version >= '3.6' onnx==1.16.0 onnxruntime==1.17.3 -openai==1.23.3; python_full_version >= '3.7.1' +openai==1.23.6; python_full_version >= '3.7.1' opencv-python==4.9.0.80; python_version >= '3.6' openpyxl==3.1.2 ordered-set==4.1.0; python_version >= '3.7' orjson==3.10.1; python_version >= '3.8' packaging==23.2; python_version >= '3.7' pandas==1.5.3; python_version >= '3.8' -pandasai==2.0.35; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8' and python_version >= '3.9' +pandasai==2.0.36; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8' and python_version >= '3.9' pathspec==0.12.1; python_version >= '3.8' pdf2image==1.17.0 pdfminer.six==20231228 @@ -166,7 +173,8 @@ postgrest==0.16.3; python_version >= '3.8' and python_version < '4.0' posthog==3.5.0 prometheus-client==0.20.0; python_version >= '3.8' prompt-toolkit==3.0.43; python_full_version >= '3.7.0' -protobuf==5.26.1; python_version >= '3.8' +proto-plus==1.23.0; python_version >= '3.6' +protobuf==4.25.3; python_version >= '3.8' psutil==5.9.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' psycopg2==2.9.9; python_version >= '3.7' psycopg2-binary==2.9.9; python_version >= '3.7' @@ -174,6 +182,7 @@ py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3. pyarrow==16.0.0; python_version >= '3.8' pyarrow-hotfix==0.6; python_version >= '3.5' pyasn1==0.6.0; python_version >= '3.8' +pyasn1-modules==0.4.0; python_version >= '3.8' pycocotools==2.0.7; python_version >= '3.5' pycodestyle==2.11.1; python_version >= '3.8' pycparser==2.22; python_version >= '3.8' @@ -182,6 +191,7 @@ pydantic==2.7.1; python_version >= '3.8' pydantic-core==2.18.2; python_version >= '3.8' pydantic-settings==2.2.1; python_version >= '3.8' pyflakes==3.2.0; python_version >= '3.8' +pyinstrument==4.6.2; python_version >= '3.7' pypandoc==1.13; python_version >= '3.6' pyparsing==3.1.2; python_full_version >= '3.6.8' pypdf==4.2.0; python_version >= '3.6' @@ -189,7 +199,7 @@ pypdfium2==4.29.0; python_version >= '3.6' pyright==1.1.360; python_version >= '3.7' pysbd==0.3.4; python_version >= '3' pytesseract==0.3.10; python_version >= '3.7' -pytest==8.1.1; python_version >= '3.8' +pytest==8.2.0; python_version >= '3.8' pytest-celery==1.0.0; python_version >= '3.8' and python_version < '4.0' pytest-docker-tools==3.1.3; python_full_version >= '3.7.0' and python_full_version < '4.0.0' pytest-dotenv==0.5.2 @@ -197,7 +207,7 @@ pytest-mock==3.14.0; python_version >= '3.8' python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-docx==1.1.0 python-dotenv==1.0.1; python_version >= '3.8' -python-iso639==2024.2.7; python_version >= '3.8' +python-iso639==2024.4.27; python_version >= '3.8' python-jose==3.3.0 python-magic==0.4.27; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' python-multipart==0.0.9; python_version >= '3.8' @@ -217,7 +227,7 @@ rsa==4.9; python_version >= '3.6' and python_version < '4' s3transfer==0.10.1; python_version >= '3.8' safetensors==0.4.3; python_version >= '3.7' scipy==1.13.0; python_version >= '3.9' -sentry-sdk[fastapi]==1.45.0 +sentry-sdk[fastapi]==2.0.1; python_version >= '3.6' setuptools==69.5.1; python_version >= '3.8' sgmllib3k==1.0.0 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' @@ -239,8 +249,8 @@ timm==0.9.16; python_version >= '3.8' tinysegmenter==0.3 tldextract==5.1.2; python_version >= '3.8' tokenizers==0.19.1; python_version >= '3.7' -torch==2.2.2 -torchvision==0.17.2 +torch==2.3.0 +torchvision==0.18.0 tornado==6.4; python_version >= '3.8' tqdm==4.66.2; python_version >= '3.7' transformers==4.40.1; python_full_version >= '3.8.0' @@ -249,9 +259,9 @@ typing-extensions==4.11.0; python_version >= '3.8' typing-inspect==0.9.0 tzdata==2024.1; python_version >= '2' unidecode==1.3.8; python_version >= '3.5' -unstructured[all-docs]==0.13.3; python_version < '3.12' and python_full_version >= '3.9.0' +unstructured[all-docs]==0.13.4; python_version < '3.12' and python_full_version >= '3.9.0' unstructured-client==0.22.0; python_version >= '3.8' -unstructured-inference==0.7.27 +unstructured-inference==0.7.28 unstructured.pytesseract==0.3.12 urllib3==2.2.1; python_version >= '3.10' uvicorn==0.29.0; python_version >= '3.8'