moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-02 17:09:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

add script for acquis cleaning

Browse Source
This commit is contained in:
Hieu Hoang 2016-04-19 10:02:46 +01:00
parent 31c8946c8c
commit 7d96adb2a7
1 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scripts/tokenizer/delete-long-words.perl Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/perl -w
use strict;
while(<STDIN>) {
chop;
my $first = 1;
foreach (split) {
if (length($_)<200) {
print " " unless $first;
print $_;
$first = 0;
}
}
print "\n";
}