mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-19 23:27:46 +03:00
resolved conflicts
This commit is contained in:
Hieu Hoang
commit
4b37050853
@ -39,7 +39,7 @@ my($_EXTERNAL_BINDIR, $_ROOT_DIR, $_CORPUS_DIR, $_GIZA_E2F, $_GIZA_F2E, $_MODEL_
|
||||
$_CONTINUE,$_MAX_LEXICAL_REORDERING,$_DO_STEPS,
|
||||
@_ADDITIONAL_INI,$_ADDITIONAL_INI_FILE,
|
||||
$_SPARSE_TRANSLATION_TABLE, @_BASELINE_ALIGNMENT_MODEL, $_BASELINE_EXTRACT, $_BASELINE_CORPUS, $_BASELINE_ALIGNMENT,
|
||||
$_DICTIONARY, $_SPARSE_PHRASE_FEATURES, $_EPPEX, $_INSTANCE_WEIGHTS_FILE, $IGNORE);
|
||||
$_DICTIONARY, $_SPARSE_PHRASE_FEATURES, $_EPPEX, $_INSTANCE_WEIGHTS_FILE, $_LMODEL_OOV_FEATURE, $IGNORE);
|
||||
my $_CORES = 1;
|
||||
|
||||
my $debug = 0; # debug this script, do not delete any files in debug mode
|
||||
@ -133,7 +133,8 @@ $_HELP = 1
|
||||
'baseline-corpus=s' => \$_BASELINE_CORPUS,
|
||||
'baseline-alignment=s' => \$_BASELINE_ALIGNMENT,
|
||||
'cores=i' => \$_CORES,
|
||||
'instance-weights-file=s' => \$_INSTANCE_WEIGHTS_FILE
|
||||
'instance-weights-file=s' => \$_INSTANCE_WEIGHTS_FILE,
|
||||
'lmodel-oov-feature' => \$_LMODEL_OOV_FEATURE,
|
||||
);
|
||||
|
||||
if ($_HELP) {
|
||||
@ -1809,12 +1810,8 @@ sub create_ini {
|
||||
### MOSES CONFIG FILE ###
|
||||
#########################
|
||||
\n";
|
||||
my $weightHandle;
|
||||
my $weightStr;
|
||||
open $weightHandle, '>', \$weightStr or die "Can't open variable: $!";
|
||||
print $weightHandle "\n[weight]\n";
|
||||
|
||||
if (defined $___TRANSLATION_FACTORS) {
|
||||
|
||||
if (defined $___TRANSLATION_FACTORS) {
|
||||
print INI "# input factors\n";
|
||||
print INI "[input-factors]\n";
|
||||
my $INPUT_FACTOR_MAX = 0;
|
||||
@ -1960,84 +1957,68 @@ sub create_ini {
|
||||
}
|
||||
print INI "\n";
|
||||
|
||||
my $weight_d_count = 1;
|
||||
if ($___REORDERING ne "distance") {
|
||||
my $file = "# distortion (reordering) files\n\[distortion-file]\n";
|
||||
my $factor_i = 0;
|
||||
|
||||
my @SPECIFIED_TABLE = @_REORDERING_TABLE;
|
||||
foreach my $factor (split(/\+/,$___REORDERING_FACTORS)) {
|
||||
foreach my $model (@REORDERING_MODELS) {
|
||||
my $table_file = "$___MODEL_DIR/reordering-table";
|
||||
$table_file .= ".$factor" unless $___NOT_FACTORED;
|
||||
$table_file = shift @SPECIFIED_TABLE if scalar(@SPECIFIED_TABLE);
|
||||
$table_file .= ".";
|
||||
$table_file .= $model->{"filename"};
|
||||
$table_file .= ".gz";
|
||||
$file .= "$factor ".$model->{"config"}." ".$model->{"numfeatures"}." $table_file\n";
|
||||
}
|
||||
$factor_i++;
|
||||
}
|
||||
print INI $file."\n";
|
||||
foreach my $model (@REORDERING_MODELS) {
|
||||
$weight_d_count += $model->{"numfeatures"};
|
||||
my $table_file = "$___MODEL_DIR/reordering-table";
|
||||
$table_file .= ".$factor" unless $___NOT_FACTORED;
|
||||
$table_file = shift @SPECIFIED_TABLE if scalar(@SPECIFIED_TABLE);
|
||||
$table_file .= ".";
|
||||
$table_file .= $model->{"filename"};
|
||||
$table_file .= ".gz";
|
||||
$file .= "$factor ".$model->{"config"}." ".$model->{"numfeatures"}." $table_file\n";
|
||||
}
|
||||
$factor_i++;
|
||||
}
|
||||
print INI $file."\n";
|
||||
}
|
||||
else {
|
||||
$weight_d_count = 1;
|
||||
}
|
||||
|
||||
if (!$_HIERARCHICAL) {
|
||||
my $weight = 0.6/(scalar @REORDERING_MODELS+1);
|
||||
print $weightHandle "# distortion (reordering)\n";
|
||||
print $weightHandle "Distortion0= " .$weight ."\n";
|
||||
|
||||
my $indModel = 0;
|
||||
foreach my $model (@REORDERING_MODELS) {
|
||||
print $weightHandle "LexicalReordering" .$indModel ."=";
|
||||
|
||||
my $weight_count += $model->{"numfeatures"};
|
||||
for (my $i = 0; $i < $weight_count; ++$i) {
|
||||
print $weightHandle " " .$weight;
|
||||
}
|
||||
print $weightHandle "\n";
|
||||
++$indModel;
|
||||
}
|
||||
}
|
||||
print $weightHandle "\n# language model weights\n";
|
||||
my $lmweighttotal = 0.5;
|
||||
my $indLM = 0;
|
||||
foreach(1..scalar @___LM) {
|
||||
print $weightHandle "LM" .$indLM;
|
||||
printf $weightHandle "= %.4f\n", $lmweighttotal / scalar @___LM;
|
||||
++$indLM;
|
||||
}
|
||||
|
||||
printf $weightHandle "\n# translation model weights\n";
|
||||
my $indPT = 0;
|
||||
foreach my $f (split(/\+/,$___TRANSLATION_FACTORS)) {
|
||||
print $weightHandle "PhraseModel" .$indPT ."=";
|
||||
for(1..$basic_weight_count) {
|
||||
printf $weightHandle " %.2f", 1/$basic_weight_count;
|
||||
print INI "# distortion (reordering) weight\n[weight-d]\n";
|
||||
for(my $i=0;$i<$weight_d_count;$i++) {
|
||||
print INI "".(0.6/(scalar @REORDERING_MODELS+1))."\n";
|
||||
}
|
||||
print $weightHandle "\n";
|
||||
++$indPT;
|
||||
}
|
||||
|
||||
if ($_HIERARCHICAL) {
|
||||
# glue grammar
|
||||
print $weightHandle "\n# glue grammar\n";
|
||||
print $weightHandle "PhraseModel" .$indPT ."= 1.0\n";
|
||||
}
|
||||
print INI "\n# language model weights\n[weight-l]\n";
|
||||
my $lmweighttotal = 0.5;
|
||||
my $lmoovweighttotal = 0.1;
|
||||
foreach(1..scalar @___LM) {
|
||||
printf INI "%.4f\n", $lmweighttotal / scalar @___LM;
|
||||
if ($_LMODEL_OOV_FEATURE) {
|
||||
printf INI "%.4f\n", $lmoovweighttotal / scalar @___LM;
|
||||
}
|
||||
}
|
||||
|
||||
print INI "\n\n# translation model weights\n[weight-t]\n";
|
||||
foreach my $f (split(/\+/,$___TRANSLATION_FACTORS)) {
|
||||
for(1..$basic_weight_count) {
|
||||
printf INI "%.2f\n", 1/$basic_weight_count;
|
||||
}
|
||||
}
|
||||
print INI "1.0\n" if $_HIERARCHICAL; # glue grammar
|
||||
|
||||
if (defined $___GENERATION_FACTORS) {
|
||||
print $weightHandle "\n# generation model weights\n";
|
||||
print INI "\n# generation model weights\n";
|
||||
print INI "[weight-generation]\n";
|
||||
my @TYPE = @_GENERATION_TYPE;
|
||||
my $indGen = 0;
|
||||
foreach my $f (split(/\+/,$___GENERATION_FACTORS)) {
|
||||
print $weightHandle "Generation" .$indGen ."= 0.3 ";
|
||||
print $weightHandle "0" unless scalar(@TYPE) && (shift @TYPE) eq 'single';
|
||||
print $weightHandle "\n";
|
||||
++$indGen;
|
||||
print INI "0.3\n";
|
||||
print INI "0\n" unless scalar(@TYPE) && (shift @TYPE) eq 'single';
|
||||
}
|
||||
} else {
|
||||
print $weightHandle "\n# no generation models, no weight-generation section\n";
|
||||
print INI "\n# no generation models, no weight-generation section\n";
|
||||
}
|
||||
|
||||
print $weightHandle "\n# word penalty\nWordPenalty0= -1\n\n";
|
||||
print INI "\n# word penalty\n[weight-w]\n-1\n\n";
|
||||
|
||||
if ($_HIERARCHICAL) {
|
||||
print INI "[unknown-lhs]\n$_UNKNOWN_WORD_LABEL_FILE\n\n" if $_TARGET_SYNTAX && defined($_UNKNOWN_WORD_LABEL_FILE);
|
||||
@ -2058,6 +2039,10 @@ sub create_ini {
|
||||
print INI "\n# delimiter between factors in input\n[factor-delimiter]\n$___FACTOR_DELIMITER\n\n"
|
||||
}
|
||||
|
||||
if ($_LMODEL_OOV_FEATURE) {
|
||||
print INI "\n# language model OOV feature enabled\n[lmodel-oov-feature]\n1\n\n";
|
||||
}
|
||||
|
||||
# get addititional content for config file from switch or file
|
||||
if ($_ADDITIONAL_INI) {
|
||||
print INI "\n# additional settings\n\n";
|
||||
@ -2068,9 +2053,6 @@ sub create_ini {
|
||||
print INI `cat $_ADDITIONAL_INI_FILE`;
|
||||
}
|
||||
|
||||
close $weightHandle;
|
||||
print INI "\n\n$weightStr\n\n";
|
||||
|
||||
close(INI);
|
||||
}
|
||||
|
||||
@ -2124,3 +2106,4 @@ sub open_or_zcat {
|
||||
return $hdl;
|
||||
}
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user