mosesdecoder/scripts/training/convert-moses-ini-to-v2.perl

#!/usr/bin/perl -w

use strict;

my $header = "";
my @INI = <STDIN>;

my %TTABLE_IMPLEMENTATION = ( 0 => "PhraseDictionaryMemory",
			      1 => "PhraseDictionaryBinary" );
my %LM_IMPLEMENTATION = ( 0 => "SRILM",
			  8 => "KENLM lazyken=0" );


my (%FEATURE,%WEIGHT);
my $i=0;

for(; $i<scalar(@INI); $i++) {
    my $line = $INI[$i];
    if ($line =~ /^\[(.+)\]/) {
	my $section = $1;
	if ($section eq "ttable-file" ||
	    $section eq "distortion-file" ||
	    $section eq "generation-file" ||
	    $section eq "lmodel-file" || 
	    $section eq "ttable-limit" ||
	    $section eq "target-word-insertion-feature" ||
	    $section eq "source-word-deletion-feature" ||
	    $section eq "word-translation-feature" ||
	    $section eq "phrase-length-feature") {
	    $FEATURE{$section} = &get_data();
	}
	elsif ($section =~ /weight-(.+)/ && $section ne "weight-file") {
	    $WEIGHT{$1} = &get_data();
	}
	elsif ($section eq "report-sparse-features") {}
	else {
	    print STDERR "include section [$section] verbatim.\n";
	    print $header.$line;
	    my $SECTION = &get_data();
	    foreach (@{$SECTION}) {
		print $_."\n";
	    }
	}
	$header = "";
    }
    else {
	$header .= $line;
    }
}
print $header;

my ($feature,$weight) = ("","");
$feature .= "UnknownWordPenalty\n";
$weight .= "UnknownWordPenalty0= 1\n";

$feature .= "WordPenalty\n";
$weight .= "WordPenalty0= ".$WEIGHT{"w"}[0]."\n";

$feature .= "Distortion\n";
$weight .= "Distortion0= ".$WEIGHT{"d"}[0]."\n";

foreach my $section (keys %FEATURE) {
    if ($section eq "phrase-length-feature") {
	$feature .= "PhraseLengthFeature name=pl\n";
    }
    elsif ($section eq "target-word-insertion-feature") {
	my ($factor,$file) = split(/ /,$FEATURE{$section}[0]);
	$feature .= "TargetWordInsertionFeature name=twi factor=$factor";
	$feature .= " path=$file" if defined($file);
	$feature .= "\n";
    }
    elsif ($section eq "source-word-insertion-feature") {
	my ($factor,$file) = split(/ /,$FEATURE{$section}[0]);
	$feature .= "SourceWordDeletionFeature name=swd factor=$file";
 	$feature .= " path=$file" if defined($file);
	$feature .= "\n";
   }
    elsif ($section eq "word-translation-feature") {
	my ($factors,$simple,$dummy1,$dummy2,$dummy3,$dummy4,$file_f,$file_e) = split(/ /,$FEATURE{$section}[0]);
	my ($input_factor,$output_factor) = split(/\-/, $factors);
	$feature .= "WordTranslationFeature name=wt input-factor=$input_factor output-factor=$output_factor simple=$simple source-context=0 target-context=0";
	$feature .= " source-path=$file_f target-path=$file_e" if defined($file_f);
	$feature .= "\n";
    }
    elsif ($section eq "ttable-file") {
	my $i = 0;
	my @TTABLE_LIMIT = @{$FEATURE{"ttable-limit"}};
	my @W = @{$WEIGHT{"t"}};
	foreach my $line (@{$FEATURE{$section}}) {
	    my ($imp, $input_factor, $output_factor, $weight_count, $file) = split(/ /,$line);
	    my $implementation = $TTABLE_IMPLEMENTATION{$imp};
	    if (!defined($implementation)) {
		print STDERR "ERROR: Unknown translation table implementation: $implementation\n";
		$implementation = "UNKNOWN";
	    }
	    $feature .= "$implementation name=TranslationModel$i num-features=$weight_count path=$file input-factor=$input_factor output-factor=$output_factor";
	    $feature .= " ttable-limit=".$TTABLE_LIMIT[$i] if $#TTABLE_LIMIT >= $i;
	    $feature .= "\n";
	    $weight .= "TranslationModel$i=".&get_weights(\@W,$weight_count)."\n";
	    $i++;
	}
    }
    elsif ($section eq "generation-file") {
	my $i = 0;
	my @W = @{$WEIGHT{"generation"}};
	foreach my $line (@{$FEATURE{$section}}) {
	    my ($input_factor,$output_factor,$weight_count,$file) = split(/ /,$line);
	    $feature .= "Generation name=GenerationModel$i num-features=$weight_count path=$file input-factor=$input_factor output-factor=$output_factor\n";
	    $weight .= "GenerationModel$i=".&get_weights(\@W,$weight_count)."\n";
	    $i++;
	}
    }
    elsif ($section eq "distortion-file") {
	my $i = 0;
	my @W = @{$WEIGHT{"d"}};
	my $ignore = shift @W;
	foreach my $line (@{$FEATURE{$section}}) {
	    my ($factors,$type,$weight_count,$file) = split(/ /,$line);
	    my ($input_factor,$output_factor) = split(/\-/, $factors);
	    $feature .= "LexicalReordering name=LexicalReordering$i num-features=$weight_count type=$type input-factor=$input_factor output-factor=$output_factor path=$file\n";	    
	    $weight .= "LexicalReordering$i=".&get_weights(\@W,$weight_count)."\n";
	    $i++;
	}
    }
	
    elsif ($section eq "lmodel-file") {
	my $i = 0;
	my @W = @{$WEIGHT{"l"}};
	foreach my $line (@{$FEATURE{$section}}) {
	    my ($imp,$factor,$order,$file) = split(/ /,$line);
	    my $implementation = $LM_IMPLEMENTATION{$imp};
	    if (!defined($implementation)) {
		print STDERR "ERROR: Unknown language model implementation: $implementation\n";
		$implementation = "UNKNOWN";
	    }
	    $feature .= "$implementation name=LM$i factor=$factor path=$file order=$order\n";
	    $weight .= "LM$i=".&get_weights(\@W,1)."\n";
	    $i++;
	}
    }
}

print "\n[feature]\n$feature\n";
print "\n[weight]\n$weight\n";

sub get_data {
    my ($pattern) = @_;
    my @DATA;
    while (++$i < scalar(@INI) &&
	   $INI[$i] !~ /^\s*$/ &&
	   $INI[$i] !~ /^\[/ &&
	   $INI[$i] !~ /^\#/) {
	push @DATA,$INI[$i];
    }
    $i--;
    chop(@DATA);
    return \@DATA;
}

sub get_weights {
    my ($W,$count) = @_;
    my $list = "";
    for(my $w=0;$w<$count;$w++) {
	my $value = shift @{$W};
	chop($value);
	$list .= " $value";
    }
    return $list;
}