moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-26 21:42:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge github.com:moses-smt/mosesdecoder into weight-new

Browse Source
This commit is contained in:
Hieu Hoang 2013-03-04 18:19:28 +00:00
commit 35bffae402
6 changed files with 39 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
moses-cmd/IOWrapper.cpp
View File

@ -189,24 +189,6 @@ InputType*IOWrapper::GetInput(InputType* inputType)
}
}
ofstream* IOWrapper::GetOutputSearchGraphSLFStream(size_t sentenceNumber) {
const StaticData &staticData = StaticData::Instance();
stringstream fileName;
fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << sentenceNumber << ".slf";
std::ofstream *file = new std::ofstream;
file->open(fileName.str().c_str());
return file;
}
ofstream* IOWrapper::GetOutputSearchGraphHypergraphStream(size_t sentenceNumber) {
const StaticData &staticData = StaticData::Instance();
stringstream fileName;
fileName << staticData.GetParam("output-search-graph-hypergraph")[0] << "/" << sentenceNumber;
std::ofstream *file = new std::ofstream;
file->open(fileName.str().c_str());
return file;
}
ofstream* IOWrapper::GetOutputSearchGraphHypergraphWeightsStream() {
const StaticData &staticData = StaticData::Instance();
stringstream fileName;

2
moses-cmd/IOWrapper.h
View File

@ -124,8 +124,6 @@ public:
return *m_outputSearchGraphStream;
}
std::ofstream *GetOutputSearchGraphSLFStream(size_t sentenceNumber);
std::ofstream *GetOutputSearchGraphHypergraphStream(size_t sentenceNumber);
std::ofstream *GetOutputSearchGraphHypergraphWeightsStream();
std::ostream &GetDetailedTranslationReportingStream() {

52
moses-cmd/Main.cpp
View File

@ -84,8 +84,8 @@ public:
OutputCollector* detailedTranslationCollector,
OutputCollector* alignmentInfoCollector,
OutputCollector* unknownsCollector,
std::ofstream* searchGraphSLFStream,
std::ofstream* searchGraphHypergraphStream) :
bool outputSearchGraphSLF,
bool outputSearchGraphHypergraph) :
m_source(source), m_lineNumber(lineNumber),
m_outputCollector(outputCollector), m_nbestCollector(nbestCollector),
m_latticeSamplesCollector(latticeSamplesCollector),
@ -93,8 +93,8 @@ public:
m_detailedTranslationCollector(detailedTranslationCollector),
m_alignmentInfoCollector(alignmentInfoCollector),
m_unknownsCollector(unknownsCollector),
m_searchGraphSLFStream(searchGraphSLFStream),
m_searchGraphHypergraphStream(searchGraphHypergraphStream) {}
m_outputSearchGraphSLF(outputSearchGraphSLF),
m_outputSearchGraphHypergraph(outputSearchGraphHypergraph) {}
/** Translate one sentence
* gets called by main function implemented at end of this source file */
@ -148,29 +148,39 @@ public:
}
// Output search graph in HTK standard lattice format (SLF)
if (m_searchGraphSLFStream) {
if (m_searchGraphSLFStream->is_open() && m_searchGraphSLFStream->good()) {
if (m_outputSearchGraphSLF) {
stringstream fileName;
fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf";
std::ofstream *file = new std::ofstream;
file->open(fileName.str().c_str());
if (file->is_open() && file->good()) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraphAsSLF(m_lineNumber, out);
*m_searchGraphSLFStream << out.str();
m_searchGraphSLFStream -> flush();
*file << out.str();
file -> flush();
} else {
TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl);
}
}
// Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder
if (m_searchGraphHypergraphStream) {
if (m_searchGraphHypergraphStream->is_open() && m_searchGraphHypergraphStream->good()) {
if (m_outputSearchGraphHypergraph) {
stringstream fileName;
fileName << staticData.GetParam("output-search-graph-hypergraph")[0] << "/" << m_lineNumber;
std::ofstream *file = new std::ofstream;
file->open(fileName.str().c_str());
if (file->is_open() && file->good()) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraphAsHypergraph(m_lineNumber, out);
*m_searchGraphHypergraphStream << out.str();
m_searchGraphHypergraphStream -> flush();
*file << out.str();
file -> flush();
} else {
TRACE_ERR("Cannot output hypergraph for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl);
}
file -> close();
delete file;
}
// apply decision rule and output best translation(s)
@ -327,15 +337,7 @@ public:
}
~TranslationTask() {
if (m_searchGraphSLFStream) {
m_searchGraphSLFStream->close();
}
delete m_searchGraphSLFStream;
delete m_searchGraphHypergraphStream;
delete m_source;
}
private:
@ -349,8 +351,8 @@ private:
OutputCollector* m_detailedTranslationCollector;
OutputCollector* m_alignmentInfoCollector;
OutputCollector* m_unknownsCollector;
std::ofstream *m_searchGraphSLFStream;
std::ofstream *m_searchGraphHypergraphStream;
bool m_outputSearchGraphSLF;
bool m_outputSearchGraphHypergraph;
std::ofstream *m_alignmentStream;
@ -632,10 +634,8 @@ int main(int argc, char** argv)
detailedTranslationCollector.get(),
alignmentInfoCollector.get(),
unknownsCollector.get(),
staticData.GetOutputSearchGraphSLF() ?
ioWrapper->GetOutputSearchGraphSLFStream(lineCount) : NULL,
staticData.GetOutputSearchGraphHypergraph() ?
ioWrapper->GetOutputSearchGraphHypergraphStream(lineCount) : NULL);
staticData.GetOutputSearchGraphSLF(),
staticData.GetOutputSearchGraphHypergraph());
// execute task
#ifdef WITH_THREADS
pool.Submit(task);

7
scripts/recaser/detruecase.perl
View File

@ -6,11 +6,12 @@ use Getopt::Long "GetOptions";
binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");
my ($SRC,$INFILE);
my ($SRC,$INFILE,$UNBUFFERED);
die("detruecase.perl < in > out")
unless &GetOptions('headline=s' => \$SRC,
'in=s' => \$INFILE);
'in=s' => \$INFILE,
'b|unbuffered' => \$UNBUFFERED);
if (defined($UNBUFFERED) && $UNBUFFERED) { $|=1; }
my %SENTENCE_END = ("."=>1,":"=>1,"?"=>1,"!"=>1);
my %DELAYED_SENTENCE_START = ("("=>1,"["=>1,"\""=>1,"'"=>1,"&quot;"=>1,"&apos;"=>1,"&#91;"=>1,"&#93;"=>1);

6
scripts/recaser/recase.perl
View File

@ -4,7 +4,7 @@
use strict;
use Getopt::Long "GetOptions";
my ($SRC,$INFILE,$RECASE_MODEL);
my ($SRC,$INFILE,$RECASE_MODEL,$UNBUFFERED);
my $MOSES = "moses";
my $LANGUAGE = "en"; # English by default;
die("recase.perl --in file --model ini-file > out")
@ -12,9 +12,11 @@ die("recase.perl --in file --model ini-file > out")
'headline=s' => \$SRC,
'lang=s' => \$LANGUAGE,
'moses=s' => \$MOSES,
'model=s' => \$RECASE_MODEL)
'model=s' => \$RECASE_MODEL,
'b|unbuffered' => \$UNBUFFERED)
&& defined($INFILE)
&& defined($RECASE_MODEL);
if (defined($UNBUFFERED) && $UNBUFFERED) { $|=1; }
my %treated_languages = map { ($_,1) } qw/en cs/;
die "I don't know any rules for $LANGUAGE. Use 'en' as the default."

8
scripts/recaser/truecase.perl
View File

@ -8,9 +8,11 @@ binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");
# apply switches
my $MODEL;
die("truecase.perl --model truecaser < in > out")
unless &GetOptions('model=s' => \$MODEL);
my ($MODEL, $UNBUFFERED);
die("truecase.perl --model MODEL [-b] < in > out")
unless &GetOptions('model=s' => \$MODEL,'b|unbuffered' => \$UNBUFFERED)
&& defined($MODEL);
if (defined($UNBUFFERED) && $UNBUFFERED) { $|=1; }
my (%BEST,%KNOWN);
open(MODEL,$MODEL) || die("ERROR: could not open '$MODEL'");