moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-16 15:00:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

factor for oov is 0, not <unk> - interferes with source input. Add extra argument to lowercase input words or not

Browse Source
This commit is contained in:
Hieu Hoang 2014-07-13 02:54:58 +01:00
parent a402523ef5
commit a3bd695cd4
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/training/wrappers/make-factor-brown-cluster-mkcls.perl
View File

@ -2,7 +2,7 @@
use strict;
my ($cluster_file,$in,$out,$tmp) = @ARGV;
my ($lowercase, $cluster_file,$in,$out,$tmp) = @ARGV;
my $CLUSTER = &read_cluster_from_mkcls($cluster_file);
@ -17,7 +17,10 @@ while(<IN>) {
s/ $//;
my $first = 1;
foreach my $word (split) {
my $cluster = defined($$CLUSTER{$word}) ? $$CLUSTER{$word} : "<unk>";
if ($lowercase) {
$word = lc($word);
}
my $cluster = defined($$CLUSTER{$word}) ? $$CLUSTER{$word} : "0";
print OUT " " unless $first;
print OUT $cluster;
$first = 0;