mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
Compiles, not tested. Conflicts: Jamroot OnDiskPt/PhraseNode.h OnDiskPt/TargetPhrase.cpp OnDiskPt/TargetPhrase.h OnDiskPt/TargetPhraseCollection.cpp mert/BleuScorer.cpp mert/Data.cpp mert/FeatureData.cpp moses-chart-cmd/src/Main.cpp moses/src/AlignmentInfo.h moses/src/ChartManager.cpp moses/src/LM/Ken.cpp moses/src/LM/Ken.h moses/src/LMList.h moses/src/LexicalReordering.h moses/src/PhraseDictionaryTree.h moses/src/ScoreIndexManager.h moses/src/StaticData.h moses/src/TargetPhrase.h moses/src/Word.cpp scripts/ems/experiment.meta scripts/ems/experiment.perl scripts/training/train-model.perl
248 lines
7.1 KiB
C++
248 lines
7.1 KiB
C++
/**
|
|
* Extract features and score statistics from nvest file, optionally merging with
|
|
* those from the previous iteration.
|
|
* Developed during the 2nd MT marathon.
|
|
**/
|
|
|
|
#include <iostream>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <getopt.h>
|
|
#include <boost/scoped_ptr.hpp>
|
|
|
|
#include "Data.h"
|
|
#include "Scorer.h"
|
|
#include "ScorerFactory.h"
|
|
#include "Timer.h"
|
|
#include "Util.h"
|
|
|
|
using namespace std;
|
|
using namespace MosesTuning;
|
|
|
|
namespace {
|
|
|
|
void usage()
|
|
{
|
|
cerr << "usage: extractor [options])" << endl;
|
|
cerr << "[--sctype|-s] the scorer type (default BLEU)" << endl;
|
|
cerr << "[--scconfig|-c] configuration string passed to scorer" << endl;
|
|
cerr << "\tThis is of the form NAME1:VAL1,NAME2:VAL2 etc " << endl;
|
|
cerr << "[--reference|-r] comma separated list of reference files" << endl;
|
|
cerr << "[--binary|-b] use binary output format (default to text )" << endl;
|
|
cerr << "[--nbest|-n] the nbest file" << endl;
|
|
cerr << "[--scfile|-S] the scorer data output file" << endl;
|
|
cerr << "[--ffile|-F] the feature data output file" << endl;
|
|
cerr << "[--prev-ffile|-E] comma separated list of previous feature data" << endl;
|
|
cerr << "[--prev-scfile|-R] comma separated list of previous scorer data" << endl;
|
|
cerr << "[--factors|-f] list of factors passed to the scorer (e.g. 0|2)" << endl;
|
|
cerr << "[--filter|-l] filter command used to preprocess the sentences" << endl;
|
|
cerr << "[--allow-duplicates|-d] omit the duplicate removal step" << endl;
|
|
cerr << "[-v] verbose level" << endl;
|
|
cerr << "[--help|-h] print this message and exit" << endl;
|
|
exit(1);
|
|
}
|
|
|
|
static struct option long_options[] = {
|
|
{"sctype", required_argument, 0, 's'},
|
|
{"scconfig", required_argument,0, 'c'},
|
|
{"factors", required_argument,0, 'f'},
|
|
{"filter", required_argument,0, 'l'},
|
|
{"reference", required_argument, 0, 'r'},
|
|
{"binary", no_argument, 0, 'b'},
|
|
{"nbest", required_argument, 0, 'n'},
|
|
{"scfile", required_argument, 0, 'S'},
|
|
{"ffile", required_argument, 0, 'F'},
|
|
{"prev-scfile", required_argument, 0, 'R'},
|
|
{"prev-ffile", required_argument, 0, 'E'},
|
|
{"verbose", required_argument, 0, 'v'},
|
|
{"help", no_argument, 0, 'h'},
|
|
{"allow-duplicates", no_argument, 0, 'd'},
|
|
{0, 0, 0, 0}
|
|
};
|
|
|
|
// Command line options used in extractor.
|
|
struct ProgramOption {
|
|
string scorerType;
|
|
string scorerConfig;
|
|
string scorerFactors;
|
|
string scorerFilter;
|
|
string referenceFile;
|
|
string nbestFile;
|
|
string scoreDataFile;
|
|
string featureDataFile;
|
|
string prevScoreDataFile;
|
|
string prevFeatureDataFile;
|
|
bool binmode;
|
|
bool allowDuplicates;
|
|
int verbosity;
|
|
|
|
ProgramOption()
|
|
: scorerType("BLEU"),
|
|
scorerConfig(""),
|
|
scorerFactors(""),
|
|
scorerFilter(""),
|
|
referenceFile(""),
|
|
nbestFile(""),
|
|
scoreDataFile("statscore.data"),
|
|
featureDataFile("features.data"),
|
|
prevScoreDataFile(""),
|
|
prevFeatureDataFile(""),
|
|
binmode(false),
|
|
allowDuplicates(false),
|
|
verbosity(0) { }
|
|
};
|
|
|
|
void ParseCommandOptions(int argc, char** argv, ProgramOption* opt) {
|
|
int c;
|
|
int option_index;
|
|
|
|
while ((c = getopt_long(argc, argv, "s:r:f:l:n:S:F:R:E:v:hbd", long_options, &option_index)) != -1) {
|
|
switch (c) {
|
|
case 's':
|
|
opt->scorerType = string(optarg);
|
|
break;
|
|
case 'c':
|
|
opt->scorerConfig = string(optarg);
|
|
break;
|
|
case 'f':
|
|
opt->scorerFactors = string(optarg);
|
|
break;
|
|
case 'l':
|
|
opt->scorerFilter = string(optarg);
|
|
break;
|
|
case 'r':
|
|
opt->referenceFile = string(optarg);
|
|
break;
|
|
case 'b':
|
|
opt->binmode = true;
|
|
break;
|
|
case 'n':
|
|
opt->nbestFile = string(optarg);
|
|
break;
|
|
case 'S':
|
|
opt->scoreDataFile = string(optarg);
|
|
break;
|
|
case 'F':
|
|
opt->featureDataFile = string(optarg);
|
|
break;
|
|
case 'E':
|
|
opt->prevFeatureDataFile = string(optarg);
|
|
break;
|
|
case 'R':
|
|
opt->prevScoreDataFile = string(optarg);
|
|
break;
|
|
case 'v':
|
|
opt->verbosity = atoi(optarg);
|
|
break;
|
|
case 'd':
|
|
opt->allowDuplicates = true;
|
|
break;
|
|
default:
|
|
usage();
|
|
}
|
|
}
|
|
}
|
|
|
|
} // anonymous namespace
|
|
|
|
int main(int argc, char** argv)
|
|
{
|
|
ResetUserTime();
|
|
|
|
ProgramOption option;
|
|
ParseCommandOptions(argc, argv, &option);
|
|
|
|
try {
|
|
// check whether score statistics file is specified
|
|
if (option.scoreDataFile.length() == 0) {
|
|
throw runtime_error("Error: output score statistics file is not specified");
|
|
}
|
|
|
|
// check wheter feature file is specified
|
|
if (option.featureDataFile.length() == 0) {
|
|
throw runtime_error("Error: output feature file is not specified");
|
|
}
|
|
|
|
// check whether reference file is specified when nbest is specified
|
|
if ((option.nbestFile.length() > 0 && option.referenceFile.length() == 0)) {
|
|
throw runtime_error("Error: reference file is not specified; you can not score the nbest");
|
|
}
|
|
|
|
vector<string> nbestFiles;
|
|
if (option.nbestFile.length() > 0) {
|
|
Tokenize(option.nbestFile.c_str(), ',', &nbestFiles);
|
|
}
|
|
|
|
vector<string> referenceFiles;
|
|
if (option.referenceFile.length() > 0) {
|
|
Tokenize(option.referenceFile.c_str(), ',', &referenceFiles);
|
|
}
|
|
|
|
vector<string> prevScoreDataFiles;
|
|
if (option.prevScoreDataFile.length() > 0) {
|
|
Tokenize(option.prevScoreDataFile.c_str(), ',', &prevScoreDataFiles);
|
|
}
|
|
|
|
vector<string> prevFeatureDataFiles;
|
|
if (option.prevFeatureDataFile.length() > 0) {
|
|
Tokenize(option.prevFeatureDataFile.c_str(), ',', &prevFeatureDataFiles);
|
|
}
|
|
|
|
if (prevScoreDataFiles.size() != prevFeatureDataFiles.size()) {
|
|
throw runtime_error("Error: there is a different number of previous score and feature files");
|
|
}
|
|
|
|
if (option.binmode) {
|
|
cerr << "Binary write mode is selected" << endl;
|
|
} else {
|
|
cerr << "Binary write mode is NOT selected" << endl;
|
|
}
|
|
|
|
TRACE_ERR("Scorer type: " << option.scorerType << endl);
|
|
|
|
boost::scoped_ptr<Scorer> scorer(
|
|
ScorerFactory::getScorer(option.scorerType, option.scorerConfig));
|
|
|
|
// set Factors and Filter used to preprocess the sentences
|
|
scorer->setFactors(option.scorerFactors);
|
|
scorer->setFilter(option.scorerFilter);
|
|
|
|
// load references
|
|
if (referenceFiles.size() > 0)
|
|
scorer->setReferenceFiles(referenceFiles);
|
|
|
|
// PrintUserTime("References loaded");
|
|
|
|
Data data(scorer.get());
|
|
|
|
// load old data
|
|
for (size_t i = 0; i < prevScoreDataFiles.size(); i++) {
|
|
data.load(prevFeatureDataFiles.at(i), prevScoreDataFiles.at(i));
|
|
}
|
|
|
|
// PrintUserTime("Previous data loaded");
|
|
|
|
// computing score statistics of each nbest file
|
|
for (size_t i = 0; i < nbestFiles.size(); i++) {
|
|
data.loadNBest(nbestFiles.at(i));
|
|
}
|
|
|
|
// PrintUserTime("Nbest entries loaded and scored");
|
|
|
|
//ADDED_BY_TS
|
|
if (!option.allowDuplicates) {
|
|
data.removeDuplicates();
|
|
}
|
|
//END_ADDED
|
|
|
|
data.save(option.featureDataFile, option.scoreDataFile, option.binmode);
|
|
PrintUserTime("Stopping...");
|
|
|
|
return EXIT_SUCCESS;
|
|
} catch (const exception& e) {
|
|
cerr << "Exception: " << e.what() << endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
}
|