moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-16 23:10:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

flag to turn off sentence splitter from emitting <P>

Browse Source
This commit is contained in:
William Waites 2020-02-26 14:08:26 +00:00
parent 22923ddcf0
commit 696a5d9833
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/ems/support/split-sentences.perl
View File

@ -23,6 +23,7 @@ my $is_cjk = 0;
my $QUIET = 0;
my $HELP = 0;
my $LIST_ITEM = 0;
my $NOP = 0;
while (@ARGV) {
$_ = shift;
@ -31,6 +32,7 @@ while (@ARGV) {
/^-q$/ && ($QUIET = 1, next);
/^-h$/ && ($HELP = 1, next);
/^-i$/ && ($LIST_ITEM = 1, next);
/^-n$/ && ($NOP = 1, next);
/^-b$/ && ($|++, next); # no output buffering
}
@ -40,6 +42,7 @@ if ($HELP) {
print "-b: no output buffering (for use in bidirectional pipes)\n";
print "-p: use a custom prefix file, overriding the installed one\n";
print "-i: avoid splitting on list items (e.g. 1. This is the first)\n";
print "-n: do not emit <P> after paragraphs\n";
exit;
}
if (!$QUIET) {
@ -89,7 +92,7 @@ while (<STDIN>) {
if (/^<.+>$/ || /^\s*$/) {
# Time to process this block; we've hit a blank or <p>
&do_it_for($text, $_);
print "<P>\n" if (/^\s*$/ && $text); ## If we have text followed by <P>
print "<P>\n" if $NOP == 0 && (/^\s*$/ && $text); ## If we have text followed by <P>
$text = "";
}
else {