zed-industries/zed
1
1
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2024-09-20 02:47:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix for error when truncating a length less than the string length

Browse Source
This commit is contained in:
KCaverly 2023-10-18 16:31:29 -04:00
parent 32853c2044
commit a0e01e075d
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
crates/ai/src/models.rs
View File

@ -38,7 +38,11 @@ impl LanguageModel for OpenAILanguageModel {
fn truncate(&self, content: &str, length: usize) -> anyhow::Result<String> {
if let Some(bpe) = &self.bpe {
let tokens = bpe.encode_with_special_tokens(content);
bpe.decode(tokens[..length].to_vec())
if tokens.len() > length {
bpe.decode(tokens[..length].to_vec())
} else {
bpe.decode(tokens)
}
} else {
Err(anyhow!("bpe for open ai model was not retrieved"))
}