mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
split_xml should be consistent for training and using
This commit is contained in:
parent
413ba6b583
commit
40748e528d
@ -103,8 +103,20 @@ sub split_xml {
|
|||||||
while($line =~ /\S/) {
|
while($line =~ /\S/) {
|
||||||
# XML tag
|
# XML tag
|
||||||
if ($line =~ /^\s*(<\S[^>]*>)(.*)$/) {
|
if ($line =~ /^\s*(<\S[^>]*>)(.*)$/) {
|
||||||
$MARKUP[$i] .= $1." ";
|
my $potential_xml = $1;
|
||||||
$line = $2;
|
my $line_next = $2;
|
||||||
|
# exception for factor that is an XML tag
|
||||||
|
if ($line =~ /^\S/ && scalar(@WORD)>0 && $WORD[$i-1] =~ /\|$/) {
|
||||||
|
$WORD[$i-1] .= $potential_xml;
|
||||||
|
if ($line_next =~ /^(\|+)(.*)$/) {
|
||||||
|
$WORD[$i-1] .= $1;
|
||||||
|
$line_next = $2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
$MARKUP[$i] .= $potential_xml." ";
|
||||||
|
}
|
||||||
|
$line = $line_next;
|
||||||
}
|
}
|
||||||
# non-XML text
|
# non-XML text
|
||||||
elsif ($line =~ /^\s*([^\s<>]+)(.*)$/) {
|
elsif ($line =~ /^\s*([^\s<>]+)(.*)$/) {
|
||||||
|
Loading…
Reference in New Issue
Block a user