mirror of
https://github.com/guillaume-be/rust-bert.git
synced 2024-09-11 12:55:34 +03:00
Created QaExample and parsing from question and context pair
This commit is contained in:
parent
ed82d934e6
commit
1a7da783a9
@ -2,10 +2,7 @@ extern crate failure;
|
||||
extern crate dirs;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use tch::{Device, nn};
|
||||
use rust_tokenizers::{BertTokenizer};
|
||||
use rust_bert::pipelines::question_answering::QaExample;
|
||||
//use rust_bert::{DistilBertForQuestionAnswering, DistilBertConfig};
|
||||
|
||||
|
||||
fn main() -> failure::Fallible<()> {
|
||||
@ -14,35 +11,10 @@ fn main() -> failure::Fallible<()> {
|
||||
home.push("rustbert");
|
||||
home.push("distilbert-qa");
|
||||
let _config_path = &home.as_path().join("config.json");
|
||||
let vocab_path = &home.as_path().join("vocab.txt");
|
||||
let _vocab_path = &home.as_path().join("vocab.txt");
|
||||
let _weights_path = &home.as_path().join("model.ot");
|
||||
|
||||
// Set-up masked LM model
|
||||
let device = Device::Cpu;
|
||||
let _vs = nn::VarStore::new(device);
|
||||
let _tokenizer: BertTokenizer = BertTokenizer::from_file(vocab_path.to_str().unwrap(), true);
|
||||
// let config = DistilBertConfig::from_file(config_path);
|
||||
// let _distilbert_model = DistilBertForQuestionAnswering::new(&vs.root(), &config);
|
||||
// vs.load(weights_path);
|
||||
|
||||
// Define input
|
||||
// let input = [
|
||||
// "Looks like one thing is missing", "It\'s like comparing oranges to apples"
|
||||
// ];
|
||||
// let tokenized_input = tokenizer.encode_list(input.to_vec(), 128, &TruncationStrategy::LongestFirst, 0);
|
||||
// let max_len = tokenized_input.iter().map(|input| input.token_ids.len()).max().unwrap();
|
||||
// let tokenized_input = tokenized_input.
|
||||
// iter().
|
||||
// map(|input| input.token_ids.clone()).
|
||||
// map(|mut input| {
|
||||
// input.extend(vec![0; max_len - input.len()]);
|
||||
// input
|
||||
// }).
|
||||
// map(|input|
|
||||
// Tensor::of_slice(&(input))).
|
||||
// collect::<Vec<_>>();
|
||||
// let _input_tensor = Tensor::stack(tokenized_input.as_slice(), 0).to(device);
|
||||
|
||||
let question = "Where does Amy live ?";
|
||||
let answer = "Amy lives in Amsterdam.";
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user