diff --git a/scripts/ems/example/config.basic b/scripts/ems/example/config.basic index 3eee66565..c0a530bd8 100644 --- a/scripts/ems/example/config.basic +++ b/scripts/ems/example/config.basic @@ -504,9 +504,13 @@ nist-bleu-c = "$moses-script-dir/generic/mteval-v13a.pl -c" # ter = ### METEOR: gives credit to stem / worknet synonym matches -# not yet integrated -# -# meteor = +## recently integrated - use with care +## only for supported languages, needs to be installed separately +## +## uncomment following 3 lines, modify first one adding the location of meteor installation on your disk +# meteor-script = "/project/software/meteor-1.3/meteor-1.3.jar" +# meteor = "java -Xmx2G -jar $meteor-script" +# meteor-params = " -l $output-extension -norm" ### Analysis: carry out various forms of analysis on the output # diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index 93afd24eb..b3e076ee5 100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta @@ -745,11 +745,12 @@ wer rerun-on-change: wer template: $wer IN IN1 > OUT meteor - in: wrapped-output reference-sgm + in: cleaned-output reference out: meteor-score - default-name: evaluation/detokenized.sgm.METEOR + default-name: evaluation/meteor ignore-unless: meteor rerun-on-change: meteor + template: $meteor IN IN1 $meteor-params > OUT analysis in: recased-output reference input out: analysis diff --git a/scripts/ems/support/report-experiment-scores.perl b/scripts/ems/support/report-experiment-scores.perl index 59f43f6eb..2efd86517 100755 --- a/scripts/ems/support/report-experiment-scores.perl +++ b/scripts/ems/support/report-experiment-scores.perl @@ -13,6 +13,7 @@ $TYPE{"nist-bleu-c"} = "BLEU-c"; $TYPE{"multi-bleu-c"}= "BLEU-c"; $TYPE{"ibm-bleu"} = "IBM"; $TYPE{"ibm-bleu-c"} = "IBM-c"; +$TYPE{"meteor"} = "METEOR"; my %SCORE; my %AVERAGE; @@ -56,6 +57,9 @@ sub process { elsif ($type eq 'multi-bleu' || $type eq 'multi-bleu-c') { $SCORE{$set} .= &extract_multi_bleu($file,$type)." "; } + elsif ($type eq 'meteor') { + $SCORE{$set} .= &extract_meteor($file,$type)." "; + } } sub extract_nist_bleu { @@ -110,3 +114,18 @@ sub extract_multi_bleu { return $output.$TYPE{$type}; } + +sub extract_meteor { + my ($file,$type) = @_; + my ($meteor, $precision); + foreach (`cat $file`) { + $meteor = $1*100 if /Final score:\s*(\S+)/; + $precision = $1 if /Precision:\s*(\S+)/; + } + my $output = sprintf("%.02f ",$meteor); + $output .= sprintf("(%.03f) ",$precision) if $precision; + $AVERAGE{"meteor"} += $meteor; + + return $output.$TYPE{$type}; + +} diff --git a/scripts/ems/web/overview.php b/scripts/ems/web/overview.php index 7280bc35b..179dda464 100644 --- a/scripts/ems/web/overview.php +++ b/scripts/ems/web/overview.php @@ -282,29 +282,28 @@ function output_score($id,$info) { $each_score = explode(" ; ",$score); for($i=0;$i0) { print "
"; } - $opened_a_tag = 0; - if ($set != "avg") { - if (file_exists("$dir/evaluation/$set.cleaned.$id")) { - print ""; - $opened_a_tag = 1; - } - else if (file_exists("$dir/evaluation/$set.output.$id")) { - print ""; - $opened_a_tag = 1; - } - } - if ($set == "avg" && count($each_score)>1) { print $match[2].": "; } - print $match[1]; - if ($opened_a_tag) { print ""; } + if (preg_match('/([\d\(\)\.\s]+) (\S*)/',$each_score[$i],$match)) { + //if ($i>0) { print " "; } + $opened_a_tag = 0; + if ($set != "avg") { + if (file_exists("$dir/evaluation/$set.cleaned.$id")) { + print ""; + $opened_a_tag = 1; + } + else if (file_exists("$dir/evaluation/$set.output.$id")) { + print ""; + $opened_a_tag = 1; + } + } + if ($set == "avg" && count($each_score)>1) { print $match[2].": "; } + print "
".$match[1]."
"; + if ($opened_a_tag) { print "
"; } } else { - print "-"; + print "-"; } } - + print ""; if ($has_analysis && array_key_exists($set,$has_analysis)) { print "";