mirror of
https://github.com/hasktorch/tokenizers.git
synced 2024-09-19 01:27:55 +03:00
debugging corrupt token ids (wip)
This commit is contained in:
parent
8edebba50f
commit
874798777f
@ -127,9 +127,8 @@ pub extern "C" fn get_ids(ptr: *mut Encoding) -> *mut CIDs {
|
||||
for id in result {
|
||||
println!("{} ", id);
|
||||
}
|
||||
forget(result);
|
||||
// forget(result);
|
||||
let mut array = CIDs { length: result.len() as c_uint, data: result.as_ptr()};
|
||||
// let mut array = CIDs { length: result.len() as c_uint, data: result.as_ptr()};
|
||||
return Box::into_raw(Box::new(array));
|
||||
}
|
||||
}
|
||||
|
@ -25,5 +25,5 @@ main = do
|
||||
test "hey hey there hey hey hey there hey" tokenizer
|
||||
test "hi hi hi hello hello" tokenizer
|
||||
test "hi" tokenizer
|
||||
test "hi" tokenizer
|
||||
test "hi hi hi hi hi hi" tokenizer
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user