guillaume-be/rust-bert
1
1
Fork 0
mirror of https://github.com/guillaume-be/rust-bert.git synced 2024-09-11 12:55:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Created QaExample and parsing from question and context pair

Browse Source
This commit is contained in:
Guillaume B 2020-02-24 16:24:43 +01:00
parent ed82d934e6
commit 1a7da783a9
1 changed files with 1 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
examples/question_answering.rs
View File

@ -2,10 +2,7 @@ extern crate failure;
extern crate dirs;
use std::path::PathBuf;
use tch::{Device, nn};
use rust_tokenizers::{BertTokenizer};
use rust_bert::pipelines::question_answering::QaExample;
//use rust_bert::{DistilBertForQuestionAnswering, DistilBertConfig};
fn main() -> failure::Fallible<()> {
@ -14,35 +11,10 @@ fn main() -> failure::Fallible<()> {
home.push("rustbert");
home.push("distilbert-qa");
let _config_path = &home.as_path().join("config.json");
let vocab_path = &home.as_path().join("vocab.txt");
let _vocab_path = &home.as_path().join("vocab.txt");
let _weights_path = &home.as_path().join("model.ot");
// Set-up masked LM model
let device = Device::Cpu;
let _vs = nn::VarStore::new(device);
let _tokenizer: BertTokenizer = BertTokenizer::from_file(vocab_path.to_str().unwrap(), true);
// let config = DistilBertConfig::from_file(config_path);
// let _distilbert_model = DistilBertForQuestionAnswering::new(&vs.root(), &config);
// vs.load(weights_path);
// Define input
// let input = [
// "Looks like one thing is missing", "It\'s like comparing oranges to apples"
// ];
// let tokenized_input = tokenizer.encode_list(input.to_vec(), 128, &TruncationStrategy::LongestFirst, 0);
// let max_len = tokenized_input.iter().map(|input| input.token_ids.len()).max().unwrap();
// let tokenized_input = tokenized_input.
// iter().
// map(|input| input.token_ids.clone()).
// map(|mut input| {
// input.extend(vec![0; max_len - input.len()]);
// input
// }).
// map(|input|
// Tensor::of_slice(&(input))).
// collect::<Vec<_>>();
// let _input_tensor = Tensor::stack(tokenized_input.as_slice(), 0).to(device);
let question = "Where does Amy live ?";
let answer = "Amy lives in Amsterdam.";