hasktorch/tokenizers
1
1
Fork 0
mirror of https://github.com/hasktorch/tokenizers.git synced 2024-09-17 16:47:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

debugging corrupt token ids (wip)

Browse Source
This commit is contained in:
austinvhuang 2021-04-10 22:23:56 -04:00 committed by Torsten Scholak
parent 8edebba50f
commit 874798777f
No known key found for this signature in database
GPG Key ID: EF135E6C40866D80
2 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/haskell/src/lib.rs
View File

@ -127,9 +127,8 @@ pub extern "C" fn get_ids(ptr: *mut Encoding) -> *mut CIDs {
for id in result {
println!("{} ", id);
}
forget(result);
// forget(result);
let mut array = CIDs { length: result.len() as c_uint, data: result.as_ptr()};
// let mut array = CIDs { length: result.len() as c_uint, data: result.as_ptr()};
return Box::into_raw(Box::new(array));
}
}

2
bindings/haskell/tokenizers-haskell/src/Main.hs
View File

@ -25,5 +25,5 @@ main = do
test "hey hey there hey hey hey there hey" tokenizer
test "hi hi hi hello hello" tokenizer
test "hi" tokenizer
test "hi" tokenizer
test "hi hi hi hi hi hi" tokenizer