mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2025-01-03 17:33:52 +03:00
tm-mt integration. Runs all the way through to creating rule table without crashing
This commit is contained in:
Hieu Hoang
parent
fbd651c9d4
commit
07498356d2
296
contrib/tm-mt-integration/make-pt-from-tm.perl
Executable file
296
contrib/tm-mt-integration/make-pt-from-tm.perl
Executable file
@ -0,0 +1,296 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use FindBin qw($RealBin);
|
||||
use File::Basename;
|
||||
|
||||
my $DEBUG = 1;
|
||||
my $OUTPUT_RULES = 1;
|
||||
|
||||
#my $data_root = "/Users/hieuhoang/workspace/experiment/data/tm-mt-integration/";
|
||||
my $in_file = $ARGV[0]; #"$data_root/in/ac-test.input.tc.4";
|
||||
my $source_file = $ARGV[1]; #"$data_root/in/acquis.truecased.4.en.uniq";
|
||||
my $target_file = $ARGV[2]; #"$data_root/in/acquis.truecased.4.fr.uniq";
|
||||
my $alignment_file = $ARGV[3]; #"$data_root/in/acquis.truecased.4.align.uniq";
|
||||
my $lex_file = $ARGV[4]; #$data_root/in/lex.4;
|
||||
my $pt_file = $ARGV[5]; #"$data_root/out/pt";
|
||||
|
||||
my $cmd;
|
||||
|
||||
print "pt_file=$pt_file \n";
|
||||
my $TMPDIR=dirname($pt_file) ."/tmp.$$";
|
||||
$cmd = "mkdir -p $TMPDIR";
|
||||
`$cmd`;
|
||||
|
||||
my $match_file = "$TMPDIR/match";
|
||||
|
||||
# suffix array creation and extraction
|
||||
$cmd = "$RealBin/fuzzy-match --multiple $in_file $source_file > $match_file";
|
||||
`$cmd`;
|
||||
|
||||
# make into xml and pt
|
||||
my $out_file = "$TMPDIR/ac-test.input.xml.4.uniq.multi.tuning";
|
||||
|
||||
my @INPUT = `cat $in_file`; chop(@INPUT);
|
||||
my @ALL_SOURCE = `cat $source_file`; chop(@ALL_SOURCE);
|
||||
my @ALL_TARGET = `cat $target_file`; chop(@ALL_TARGET);
|
||||
my @ALL_ALIGNMENT = `cat $alignment_file`; chop(@ALL_ALIGNMENT);
|
||||
|
||||
open(MATCH,$match_file);
|
||||
open(FRAME,">$out_file");
|
||||
open(RULE,">$out_file.extract") if $OUTPUT_RULES;
|
||||
open(RULE_INV,">$out_file.extract.inv") if $OUTPUT_RULES;
|
||||
open(INFO,">$out_file.info");
|
||||
while( my $match = <MATCH> ) {
|
||||
chop($match);
|
||||
my ($score,$sentence,$path) = split(/ \|\|\| /,$match);
|
||||
|
||||
$score =~ /^(\d+) (.+)/ || die;
|
||||
my ($i,$match_score) = ($1,$2);
|
||||
|
||||
# construct frame
|
||||
if ($sentence < 1e9 && $sentence >= 0) {
|
||||
my $SOURCE = $ALL_SOURCE[$sentence];
|
||||
my @ALIGNMENT = split(/ \|\|\| /,$ALL_ALIGNMENT[$sentence]);
|
||||
my @TARGET = split(/ \|\|\| /,$ALL_TARGET[$sentence]);
|
||||
|
||||
for(my $j=0;$j<scalar(@TARGET);$j++) {
|
||||
$TARGET[$j] =~ /^(\d+) (.+)$/ || die;
|
||||
my ($target_count,$target) = ($1,$2);
|
||||
my ($frame,$rule_s,$rule_t,$rule_alignment,$rule_alignment_inv) =
|
||||
&create_xml($SOURCE,
|
||||
$INPUT[$i],
|
||||
$target,
|
||||
$ALIGNMENT[$j],
|
||||
$path);
|
||||
print FRAME $frame."\n";
|
||||
print RULE "$rule_s [X] ||| $rule_t [X] ||| $rule_alignment ||| $target_count\n" if $OUTPUT_RULES;
|
||||
print RULE_INV "$rule_t [X] ||| $rule_s [X] ||| $rule_alignment_inv ||| $target_count\n" if $OUTPUT_RULES;
|
||||
print INFO "$i ||| $match_score ||| $target_count\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
close(FRAME);
|
||||
close(MATCH);
|
||||
close(RULE) if $OUTPUT_RULES;
|
||||
close(RULE_INV) if $OUTPUT_RULES;
|
||||
|
||||
`LC_ALL=C sort $out_file.extract | gzip -c > $out_file.extract.sorted.gz`;
|
||||
`LC_ALL=C sort $out_file.extract.inv | gzip -c > $out_file.extract.inv.sorted.gz`;
|
||||
|
||||
`$RealBin/../../scripts/training/train-model.perl -dont-zip -first-step 6 -last-step 6 -f en -e fr -hierarchical -extract-file $out_file.extract -lexical-file $lex_file -phrase-translation-table $pt_file` if $OUTPUT_RULES;
|
||||
|
||||
sub create_xml {
|
||||
my ($source,$input,$target,$alignment,$path) = @_;
|
||||
|
||||
my @INPUT = split(/ /,$input);
|
||||
my @SOURCE = split(/ /,$source);
|
||||
my @TARGET = split(/ /,$target);
|
||||
my %ALIGN = &create_alignment($alignment);
|
||||
|
||||
my %FRAME_INPUT;
|
||||
my (@NT,@INPUT_BITMAP,@TARGET_BITMAP,%ALIGNMENT_I_TO_S);
|
||||
foreach (@TARGET) { push @TARGET_BITMAP,1 }
|
||||
|
||||
### STEP 1: FIND MISMATCHES
|
||||
|
||||
my ($s,$i) = (0,0);
|
||||
my $currently_matching = 0;
|
||||
my ($start_s,$start_i) = (0,0);
|
||||
|
||||
$path .= "X"; # indicate end
|
||||
print "$input\n$source\n$target\n$path\n";
|
||||
for(my $p=0;$p<length($path);$p++) {
|
||||
my $action = substr($path,$p,1);
|
||||
|
||||
# beginning of a mismatch
|
||||
if ($currently_matching && $action ne "M" && $action ne "X") {
|
||||
$start_i = $i;
|
||||
$start_s = $s;
|
||||
$currently_matching = 0;
|
||||
}
|
||||
|
||||
# end of a mismatch
|
||||
elsif (!$currently_matching &&
|
||||
($action eq "M" || $action eq "X")) {
|
||||
|
||||
# remove use of affected target words
|
||||
for(my $ss = $start_s; $ss<$s; $ss++) {
|
||||
foreach my $tt (keys %{${$ALIGN{'s'}}[$ss]}) {
|
||||
$TARGET_BITMAP[$tt] = 0;
|
||||
}
|
||||
|
||||
# also remove enclosed unaligned words?
|
||||
}
|
||||
|
||||
# are there input words that need to be inserted ?
|
||||
print "($start_i<$i)?\n";
|
||||
if ($start_i<$i) {
|
||||
|
||||
# take note of input words to be inserted
|
||||
my $insertion = "";
|
||||
for(my $ii = $start_i; $ii<$i; $ii++) {
|
||||
$insertion .= $INPUT[$ii]." ";
|
||||
}
|
||||
|
||||
# find position for inserted input words
|
||||
|
||||
# find first removed target word
|
||||
my $start_t = 1000;
|
||||
for(my $ss = $start_s; $ss<$s; $ss++) {
|
||||
foreach my $tt (keys %{${$ALIGN{'s'}}[$ss]}) {
|
||||
$start_t = $tt if $tt < $start_t;
|
||||
}
|
||||
}
|
||||
|
||||
# end of sentence? add to end
|
||||
if ($start_t == 1000 && $i > $#INPUT) {
|
||||
$start_t = $#TARGET;
|
||||
}
|
||||
|
||||
# backtrack to previous words if unaligned
|
||||
if ($start_t == 1000) {
|
||||
$start_t = -1;
|
||||
for(my $ss = $s-1; $start_t==-1 && $ss>=0; $ss--) {
|
||||
foreach my $tt (keys %{${$ALIGN{'s'}}[$ss]}) {
|
||||
$start_t = $tt if $tt > $start_t;
|
||||
}
|
||||
}
|
||||
}
|
||||
$FRAME_INPUT{$start_t} .= $insertion;
|
||||
my %NT = ("start_t" => $start_t,
|
||||
"start_i" => $start_i );
|
||||
push @NT,\%NT;
|
||||
}
|
||||
$currently_matching = 1;
|
||||
}
|
||||
|
||||
print "$action $s $i ($start_s $start_i) $currently_matching";
|
||||
if ($action ne "I") {
|
||||
print " ->";
|
||||
foreach my $tt (keys %{${$ALIGN{'s'}}[$s]}) {
|
||||
print " ".$tt;
|
||||
}
|
||||
}
|
||||
print "\n";
|
||||
$s++ unless $action eq "I";
|
||||
$i++ unless $action eq "D";
|
||||
$ALIGNMENT_I_TO_S{$i} = $s unless $action eq "D";
|
||||
push @INPUT_BITMAP, 1 if $action eq "M";
|
||||
push @INPUT_BITMAP, 0 if $action eq "I" || $action eq "S";
|
||||
}
|
||||
|
||||
|
||||
print $target."\n";
|
||||
foreach (@TARGET_BITMAP) { print $_; } print "\n";
|
||||
foreach (sort keys %FRAME_INPUT) {
|
||||
print "$_: $FRAME_INPUT{$_}\n";
|
||||
}
|
||||
|
||||
### STEP 2: BUILD RULE AND FRAME
|
||||
|
||||
# hierarchical rule
|
||||
my $rule_s = "";
|
||||
my $rule_pos_s = 0;
|
||||
my %RULE_ALIGNMENT_S;
|
||||
for(my $i=0;$i<scalar(@INPUT_BITMAP);$i++) {
|
||||
if ($INPUT_BITMAP[$i]) {
|
||||
$rule_s .= $INPUT[$i]." ";
|
||||
$RULE_ALIGNMENT_S{$ALIGNMENT_I_TO_S{$i}} = $rule_pos_s++;
|
||||
}
|
||||
foreach my $NT (@NT) {
|
||||
if ($i == $$NT{"start_i"}) {
|
||||
$rule_s .= "[X][X] ";
|
||||
$$NT{"rule_pos_s"} = $rule_pos_s++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
my $rule_t = "";
|
||||
my $rule_pos_t = 0;
|
||||
my %RULE_ALIGNMENT_T;
|
||||
for(my $t=-1;$t<scalar(@TARGET_BITMAP);$t++) {
|
||||
if ($t>=0 && $TARGET_BITMAP[$t]) {
|
||||
$rule_t .= $TARGET[$t]." ";
|
||||
$RULE_ALIGNMENT_T{$t} = $rule_pos_t++;
|
||||
}
|
||||
foreach my $NT (@NT) {
|
||||
if ($t == $$NT{"start_t"}) {
|
||||
$rule_t .= "[X][X] ";
|
||||
$$NT{"rule_pos_t"} = $rule_pos_t++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
my $rule_alignment = "";
|
||||
foreach my $s (sort { $a <=> $b} keys %RULE_ALIGNMENT_S) {
|
||||
foreach my $t (keys %{$ALIGN{"s"}[$s]}) {
|
||||
next unless defined($RULE_ALIGNMENT_T{$t});
|
||||
$rule_alignment .= $RULE_ALIGNMENT_S{$s}."-".$RULE_ALIGNMENT_T{$t}." ";
|
||||
}
|
||||
}
|
||||
foreach my $NT (@NT) {
|
||||
$rule_alignment .= $$NT{"rule_pos_s"}."-".$$NT{"rule_pos_t"}." ";
|
||||
}
|
||||
|
||||
chop($rule_s);
|
||||
chop($rule_t);
|
||||
chop($rule_alignment);
|
||||
|
||||
my $rule_alignment_inv = "";
|
||||
foreach (split(/ /,$rule_alignment)) {
|
||||
/^(\d+)\-(\d+)$/;
|
||||
$rule_alignment_inv .= "$2-$1 ";
|
||||
}
|
||||
chop($rule_alignment_inv);
|
||||
|
||||
# frame
|
||||
my $frame = "";
|
||||
$frame = $FRAME_INPUT{-1} if defined $FRAME_INPUT{-1};
|
||||
|
||||
my $currently_included = 0;
|
||||
my $start_t = -1;
|
||||
push @TARGET_BITMAP,0; # indicate end
|
||||
|
||||
for(my $t=0;$t<=scalar(@TARGET);$t++) {
|
||||
# beginning of tm target inclusion
|
||||
if (!$currently_included && $TARGET_BITMAP[$t]) {
|
||||
$start_t = $t;
|
||||
$currently_included = 1;
|
||||
}
|
||||
|
||||
# end of tm target inclusion (not included word or inserted input)
|
||||
elsif ($currently_included &&
|
||||
(!$TARGET_BITMAP[$t] || defined($FRAME_INPUT{$t}))) {
|
||||
# add xml (unless change is at the beginning of the sentence
|
||||
if ($start_t >= 0) {
|
||||
my $target = "";
|
||||
print "for(tt=$start_t;tt<$t+$TARGET_BITMAP[$t]);\n";
|
||||
for(my $tt=$start_t;$tt<$t+$TARGET_BITMAP[$t];$tt++) {
|
||||
$target .= $TARGET[$tt] . " ";
|
||||
}
|
||||
chop($target);
|
||||
$frame .= "<xml translation=\"$target\"> x </xml> ";
|
||||
}
|
||||
$currently_included = 0;
|
||||
}
|
||||
|
||||
$frame .= $FRAME_INPUT{$t} if defined $FRAME_INPUT{$t};
|
||||
print "$TARGET_BITMAP[$t] $t ($start_t) $currently_included\n";
|
||||
}
|
||||
|
||||
print $frame."\n-------------------------------------\n";
|
||||
return ($frame,$rule_s,$rule_t,$rule_alignment,$rule_alignment_inv);
|
||||
}
|
||||
|
||||
sub create_alignment {
|
||||
my ($line) = @_;
|
||||
my (@ALIGNED_TO_S,@ALIGNED_TO_T);
|
||||
foreach my $point (split(/ /,$line)) {
|
||||
my ($s,$t) = split(/\-/,$point);
|
||||
$ALIGNED_TO_S[$s]{$t}++;
|
||||
$ALIGNED_TO_T[$t]{$s}++;
|
||||
}
|
||||
my %ALIGNMENT = ( 's' => \@ALIGNED_TO_S, 't' => \@ALIGNED_TO_T );
|
||||
return %ALIGNMENT;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user