mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-28 14:32:38 +03:00
a25193cc5d
This is lint reported by the new lint-checking functionality in beautify.py. (We can change to a different lint checker if we have a better one, but it would probably still flag these same problems.) Lint checking can help a lot, but only if we get the lint under control.
80 lines
1.8 KiB
Perl
Executable File
80 lines
1.8 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
# Collects and prints all n-grams that appear in the given corpus both
|
|
# tokenized as well as untokenized.
|
|
# Ondrej Bojar
|
|
|
|
use strict;
|
|
use warnings;
|
|
|
|
use Getopt::Long;
|
|
|
|
binmode(STDIN, ":utf8");
|
|
binmode(STDOUT, ":utf8");
|
|
binmode(STDERR, ":utf8");
|
|
|
|
my $usage = 0;
|
|
my $lowercase = 0;
|
|
my $n = 2;
|
|
GetOptions(
|
|
"n=i" => \$n, # the n-grams to search for (default: bigrams)
|
|
"lc|lowercase" => \$lowercase, # ignore case
|
|
"h|help|usage" => \$usage, # show info
|
|
) or exit 1;
|
|
|
|
my $nl = 0;
|
|
my $ngrams;
|
|
my $words;
|
|
while (<>) {
|
|
$nl++;
|
|
print STDERR "." if $nl % 100000 == 0;
|
|
print STDERR "($nl)" if $nl % 500000 == 0;
|
|
chomp;
|
|
$_ = lc($_) if $lowercase;
|
|
my @words = split /\s+/;
|
|
foreach my $w (@words) {
|
|
$words->{$w}++;
|
|
}
|
|
$ngrams = ngrams($n, \@words, $ngrams); # add ngram counts from this
|
|
}
|
|
print STDERR "Done.\n";
|
|
|
|
# Find suspicious
|
|
my $report;
|
|
foreach my $ngr (keys %$ngrams) {
|
|
my $w = $ngr;
|
|
$w =~ s/ //g;
|
|
my $untokcnt = $words->{$w};
|
|
next if ! $untokcnt; # never seen untokenized
|
|
my $tokcnt = $ngrams->{$ngr};
|
|
$report->{$ngr}->{"tok"} = $tokcnt;
|
|
$report->{$ngr}->{"untok"} = $untokcnt;
|
|
$report->{$ngr}->{"diff"} = abs($untokcnt-$tokcnt);
|
|
$report->{$ngr}->{"sum"} = $untokcnt+$tokcnt;
|
|
}
|
|
|
|
# Report
|
|
foreach my $ngr (sort {
|
|
$report->{$a}->{"diff"} <=> $report->{$b}->{"diff"}
|
|
|| $report->{$b}->{"sum"} <=> $report->{$a}->{"sum"}
|
|
}
|
|
keys %$report) {
|
|
print "$ngr\t$report->{$ngr}->{untok}\t$report->{$ngr}->{tok}\t$report->{$ngr}->{diff}\n";
|
|
}
|
|
|
|
sub ngrams {
|
|
my $n = shift;
|
|
my @words = @{shift()};
|
|
my $out = shift;
|
|
if ($n == 1) {
|
|
foreach my $w (@words) {
|
|
$out->{$w}++;
|
|
}
|
|
} else {
|
|
while ($#words >= $n-1) {
|
|
$out->{join(" ", @words[0..$n-1])}++;
|
|
shift @words;
|
|
}
|
|
}
|
|
return $out;
|
|
}
|