moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

sacre bleu

Browse Source
This commit is contained in:
Hieu Hoang 2018-12-04 15:27:09 +00:00
parent 345dabcde6
commit dbbc47292f
2 changed files with 49 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scripts/ems/experiment.meta
View File

@ -1514,6 +1514,23 @@ multi-bleu-c-detok
template: $multi-bleu-c-detok IN1 < IN > OUT
final-model: yes
sacre-bleu
in: detokenized-output raw-reference
out: sacre-bleu-score
default-name: evaluation/sacre-bleu
ignore-unless: sacre-bleu
rerun-on-change: sacre-bleu
template: $sacre-bleu IN1 < IN > OUT
final-model: yes
sacre-bleu-c
in: detokenized-output raw-reference
out: sacre-bleu-c-score
default-name: evaluation/sacre-bleu-c
ignore-unless: sacre-bleu-c
rerun-on-change: sacre-bleu-c
template: $sacre-bleu-c IN1 < IN > OUT
final-model: yes
ter
in: wrapped-output reference-sgm
out: ter-score
@ -1705,6 +1722,6 @@ train
[REPORTING] single
report
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:multi-bleu-detok-score EVALUATION:multi-bleu-c-detok-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:multi-bleu-detok-score EVALUATION:multi-bleu-c-detok-score EVALUATION:sacre-bleu-score EVALUATION:sacre-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis
out: report
default-name: evaluation/report

37
scripts/ems/support/report-experiment-scores.perl
View File

@ -22,10 +22,12 @@ $TYPE{"bolt-bleu"} = "BLEU";
$TYPE{"bolt-bleu-c"} = "BLEU-c";
$TYPE{"bolt-ter"} = "TER";
$TYPE{"bolt-ter-c"} = "TER-c";
$TYPE{"multi-bleu-detok"} = "BLEU";
$TYPE{"multi-bleu-c-detok"}= "BLEU-c";
$TYPE{"sacre-bleu"} = "BLEU";
$TYPE{"sacre-bleu-c"}= "BLEU-c";
my %SCORE;
my %AVERAGE;
foreach (@ARGV) {
@ -57,26 +59,49 @@ sub process {
my ($set,$type,$file) = @_;
$SCORE{$set} .= "; " if defined($SCORE{$set});
if (! -e $file) {
print STDERR "ERROR (score $type for set $set): file '$file' does not exist!\n";
print STDERR "ERROR (score $type for set $set): file '$file' does not exist!\n";
}
elsif ($type eq 'nist-bleu' || $type eq 'nist-bleu-c') {
$SCORE{$set} .= &extract_nist_bleu($file,$type)." ";
$SCORE{$set} .= &extract_nist_bleu($file,$type)." ";
}
elsif ($type eq 'ibm-bleu' || $type eq 'ibm-bleu-c') {
$SCORE{$set} .= &extract_ibm_bleu($file,$type)." ";
$SCORE{$set} .= &extract_ibm_bleu($file,$type)." ";
}
elsif ($type eq 'multi-bleu' || $type eq 'multi-bleu-c'
|| $type eq 'multi-bleu-detok' || $type eq 'multi-bleu-c-detok') {
$SCORE{$set} .= &extract_multi_bleu($file,$type)." ";
$SCORE{$set} .= &extract_multi_bleu($file,$type)." ";
}
elsif ($type eq 'sacre-bleu' || $type eq 'sacre-bleu-c') {
$SCORE{$set} .= &extract_sacre_bleu($file,$type)." ";
}
elsif ($type eq 'meteor') {
$SCORE{$set} .= &extract_meteor($file,$type)." ";
$SCORE{$set} .= &extract_meteor($file,$type)." ";
}
elsif ($type =~ /^bolt-(.+)$/) {
$SCORE{$set} .= &extract_bolt($file,$1)." ";
}
}
sub extract_sacre_bleu {
my ($file,$type) = @_;
my ($bleu,$ratio);
#print STDERR "type=$type, file=$file \n";
foreach (my $line = `cat $file`) {
chomp($line);
#print STDERR "line=$line \n";
my @toks = split(" ", $line);
$bleu = $toks[2];
$ratio = $toks[9];
}
my $output = sprintf("%.02f ",$bleu);
$output .= sprintf("(%.03f) ",$ratio) if $ratio;
$AVERAGE{"sacre-bleu"} += $bleu;
return $output.$TYPE{$type};
}
sub extract_nist_bleu {
my ($file,$type) = @_;
my ($bleu,$ratio);