moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-29 06:52:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Ulrich Germann 2015-05-03 15:09:45 +01:00
commit 2b671e67dd
436 changed files with 8537 additions and 8118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -68,6 +68,7 @@ contrib/other-builds/*.xcodeproj/xcuserdata/
*/*.xcodeproj/xcuserdata
mert/sentence-bleu
mert/sentence-bleu-nbest
._*
.DS_Store
*.pbxuser

5
BUILD-INSTRUCTIONS.txt
View File

@ -1,4 +1,3 @@
Please see the Moses website on how to compile and run Moses
http://www.statmt.org/moses/?n=Development.GetStarted
Instructions for building and installing Moses are online:
blah blah blah
http://www.statmt.org/moses/?n=Development.GetStarted

49
contrib/moses-speedtest/runtests.py
View File

@ -26,10 +26,17 @@ def parse_cmd():
arguments = parser.parse_args()
return arguments
def repoinit(testconfig):
"""Determines revision and sets up the repo."""
def repoinit(testconfig, profiler=True):
"""Determines revision and sets up the repo. If given the profiler optional
argument, wil init the profiler repo instead of the default one."""
revision = ''
#Update the repo
if profiler:
if testconfig.repo_prof is not None:
os.chdir(testconfig.repo_prof)
else:
raise ValueError('Profiling repo is not defined')
else:
os.chdir(testconfig.repo)
#Checkout specific branch, else maintain main branch
if testconfig.branch != 'master':
@ -54,8 +61,9 @@ def repoinit(testconfig):
class Configuration:
"""A simple class to hold all of the configuration constatns"""
def __init__(self, repo, drop_caches, tests, testlogs, basebranch, baserev):
def __init__(self, repo, drop_caches, tests, testlogs, basebranch, baserev, repo_prof=None):
self.repo = repo
self.repo_prof = repo_prof
self.drop_caches = drop_caches
self.tests = tests
self.testlogs = testlogs
@ -80,15 +88,16 @@ class Configuration:
class Test:
"""A simple class to contain all information about tests"""
def __init__(self, name, command, ldopts, permutations):
def __init__(self, name, command, ldopts, permutations, prof_command=None):
self.name = name
self.command = command
self.prof_command = prof_command
self.ldopts = ldopts.replace(' ', '').split(',') #Not tested yet
self.permutations = permutations
def parse_configfile(conffile, testdir, moses_repo):
def parse_configfile(conffile, testdir, moses_repo, moses_prof_repo=None):
"""Parses the config file"""
command, ldopts = '', ''
command, ldopts, prof_command = '', '', None
permutations = []
fileopen = open(conffile, 'r')
for line in fileopen:
@ -99,6 +108,8 @@ def parse_configfile(conffile, testdir, moses_repo):
if opt == 'Command:':
command = args.replace('\n', '')
if moses_prof is not None: # Get optional command for profiling
prof_command = moses_prof_repo + '/bin/' + command
command = moses_repo + '/bin/' + command
elif opt == 'LDPRE:':
ldopts = args.replace('\n', '')
@ -107,14 +118,14 @@ def parse_configfile(conffile, testdir, moses_repo):
else:
raise ValueError('Unrecognized option ' + opt)
#We use the testdir as the name.
testcase = Test(testdir, command, ldopts, permutations)
testcase = Test(testdir, command, ldopts, permutations, prof_command)
fileopen.close()
return testcase
def parse_testconfig(conffile):
"""Parses the config file for the whole testsuite."""
repo_path, drop_caches, tests_dir, testlog_dir = '', '', '', ''
basebranch, baserev = '', ''
basebranch, baserev, repo_prof_path = '', '', None
fileopen = open(conffile, 'r')
for line in fileopen:
line = line.split('#')[0] # Discard comments
@ -133,10 +144,12 @@ def parse_testconfig(conffile):
basebranch = args.replace('\n', '')
elif opt == 'BASEREV:':
baserev = args.replace('\n', '')
elif opt == 'MOSES_PROFILER_PATH:': # Optional
repo_prof_path = args.replace('\n', '')
else:
raise ValueError('Unrecognized option ' + opt)
config = Configuration(repo_path, drop_caches, tests_dir, testlog_dir,\
basebranch, baserev)
basebranch, baserev, repo_prof_path)
fileopen.close()
return config
@ -146,6 +159,8 @@ def get_config():
config = parse_testconfig(args.configfile)
config.additional_args(args.singletestdir, args.revision, args.branch)
revision = repoinit(config)
if config.repo_prof is not None:
repoinit(config, True)
config.set_revision(revision)
return config
@ -221,6 +236,10 @@ def execute_tests(testcase, cur_directory, config):
stderr=None, shell=True).communicate()
write_log('/tmp/time_moses_tests', testcase.name + '_ldpre_' +opt +'_cached', config)
#if 'profile' in testcase.permutations:
#TODO Separate the above into functions so we can execute them with profiling moses.
#Fix the logic in the main
# Go through all the test directories and executes tests
if __name__ == '__main__':
CONFIG = get_config()
@ -260,7 +279,7 @@ if __name__ == '__main__':
#Create a new configuration for base version tests:
BASECONFIG = Configuration(CONFIG.repo, CONFIG.drop_caches,\
CONFIG.tests, CONFIG.testlogs, CONFIG.basebranch,\
CONFIG.baserev)
CONFIG.baserev, CONFIG.repo_prof)
BASECONFIG.additional_args(None, CONFIG.baserev, CONFIG.basebranch)
#Set up the repository and get its revision:
REVISION = repoinit(BASECONFIG)
@ -268,6 +287,11 @@ if __name__ == '__main__':
#Build
os.chdir(BASECONFIG.repo)
subprocess.call(['./previous.sh'], shell=True)
#If profiler configuration exists also init it
if BASECONFIG.repo_prof is not None:
repoinit(BASECONFIG, True)
os.chdir(BASECONFIG.repo_prof)
subprocess.call(['./previous.sh'], shell=True)
#Perform tests
for directory in FIRSTTIME:
@ -277,10 +301,15 @@ if __name__ == '__main__':
#Reset back the repository to the normal configuration
repoinit(CONFIG)
if BASECONFIG.repo_prof is not None:
repoinit(CONFIG, True)
#Builds moses
os.chdir(CONFIG.repo)
subprocess.call(['./previous.sh'], shell=True)
if CONFIG.repo_prof is not None:
os.chdir(CONFIG.repo_prof)
subprocess.call(['./previous.sh'], shell=True)
if CONFIG.singletest:
TESTCASE = parse_configfile(CONFIG.tests + '/' +\

4
contrib/other-builds/all.workspace
View File

@ -1,13 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<CodeLite_Workspace Name="all" Database="all.tags">
<Project Name="manual-label" Path="manual-label/manual-label.project" Active="No"/>
<Project Name="manual-label" Path="manual-label/manual-label.project" Active="Yes"/>
<Project Name="extract" Path="extract/extract.project" Active="No"/>
<Project Name="util" Path="util/util.project" Active="No"/>
<Project Name="extract-mixed-syntax" Path="extract-mixed-syntax/extract-mixed-syntax.project" Active="No"/>
<Project Name="lm" Path="lm/lm.project" Active="No"/>
<Project Name="OnDiskPt" Path="OnDiskPt/OnDiskPt.project" Active="No"/>
<Project Name="search" Path="search/search.project" Active="No"/>
<Project Name="moses" Path="moses/moses.project" Active="Yes"/>
<Project Name="moses" Path="moses/moses.project" Active="No"/>
<Project Name="moses-cmd" Path="moses-cmd/moses-cmd.project" Active="No"/>
<Project Name="score" Path="score/score.project" Active="No"/>
<Project Name="consolidate" Path="consolidate/consolidate.project" Active="No"/>

2
mert/BleuDocScorer.cpp
View File

@ -174,7 +174,7 @@ statscore_t BleuDocScorer::calculateScore(const vector<int>& comps) const
UTIL_THROW_IF(comps.size() != kBleuNgramOrder * 2 + 1, util::Exception, "Error");
float logbleu = 0.0;
for (int i = 0; i < kBleuNgramOrder; ++i) {
for (size_t i = 0; i < kBleuNgramOrder; ++i) {
if (comps[2*i] == 0) {
return 0.0;
}

4
mert/BleuDocScorer.h
View File

@ -1,5 +1,4 @@
#ifndef MERT_BLEU_DOC_SCORER_H_
#define MERT_BLEU_DOC_SCORER_H_
#pragma once
#include <ostream>
#include <string>
@ -64,4 +63,3 @@ private:
}
#endif // MERT_BLEU_DOC_SCORER_H_

107
mert/BleuScorer.cpp
View File

@ -45,14 +45,14 @@ BleuScorer::BleuScorer(const string& config)
} else if (reflen == REFLEN_CLOSEST) {
m_ref_length_type = CLOSEST;
} else {
throw runtime_error("Unknown reference length strategy: " + reflen);
UTIL_THROW2("Unknown reference length strategy: " + reflen);
}
}
BleuScorer::~BleuScorer() {}
size_t BleuScorer::CountNgrams(const string& line, NgramCounts& counts,
unsigned int n, bool is_testing)
unsigned int n, bool is_testing) const
{
assert(n > 0);
vector<int> encoded_tokens;
@ -97,22 +97,13 @@ void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles)
for (size_t i = 0; i < referenceFiles.size(); ++i) {
TRACE_ERR("Loading reference from " << referenceFiles[i] << endl);
if (!OpenReference(referenceFiles[i].c_str(), i)) {
throw runtime_error("Unable to open " + referenceFiles[i]);
ifstream ifs(referenceFiles[i].c_str());
if (!OpenReferenceStream(&ifs, i)) {
UTIL_THROW2("Cannot open " + referenceFiles[i]);
}
}
}
bool BleuScorer::OpenReference(const char* filename, size_t file_id)
{
ifstream ifs(filename);
if (!ifs) {
cerr << "Cannot open " << filename << endl;
return false;
}
return OpenReferenceStream(&ifs, file_id);
}
bool BleuScorer::OpenReferenceStream(istream* is, size_t file_id)
{
if (is == NULL) return false;
@ -120,15 +111,27 @@ bool BleuScorer::OpenReferenceStream(istream* is, size_t file_id)
string line;
size_t sid = 0;
while (getline(*is, line)) {
// TODO: rather than loading the whole reference corpus into memory, can we stream it line by line?
// (loading the whole reference corpus can take gigabytes of RAM if done with millions of sentences)
line = preprocessSentence(line);
if (file_id == 0) {
Reference* ref = new Reference;
m_references.push_back(ref); // Take ownership of the Reference object.
}
if (m_references.size() <= sid) {
cerr << "Reference " << file_id << "has too many sentences." << endl;
return false;
UTIL_THROW_IF2(m_references.size() <= sid, "Reference " << file_id << "has too many sentences.");
ProcessReferenceLine(line, m_references[sid]);
if (sid > 0 && sid % 100 == 0) {
TRACE_ERR(".");
}
++sid;
}
return true;
}
void BleuScorer::ProcessReferenceLine(const std::string& line, Reference* ref) const
{
NgramCounts counts;
size_t length = CountNgrams(line, counts, kBleuNgramOrder);
@ -138,35 +141,42 @@ bool BleuScorer::OpenReferenceStream(istream* is, size_t file_id)
const NgramCounts::Value newcount = ci->second;
NgramCounts::Value oldcount = 0;
m_references[sid]->get_counts()->Lookup(ngram, &oldcount);
ref->get_counts()->Lookup(ngram, &oldcount);
if (newcount > oldcount) {
m_references[sid]->get_counts()->operator[](ngram) = newcount;
ref->get_counts()->operator[](ngram) = newcount;
}
}
//add in the length
m_references[sid]->push_back(length);
if (sid > 0 && sid % 100 == 0) {
TRACE_ERR(".");
}
++sid;
ref->push_back(length);
}
bool BleuScorer::GetNextReferenceFromStreams(std::vector<boost::shared_ptr<std::ifstream> >& referenceStreams, Reference& ref) const
{
for (vector<boost::shared_ptr<ifstream> >::iterator ifs=referenceStreams.begin(); ifs!=referenceStreams.end(); ++ifs) {
if (!(*ifs)) return false;
string line;
if (!getline(**ifs, line)) return false;
line = preprocessSentence(line);
ProcessReferenceLine(line, &ref);
}
return true;
}
void BleuScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
{
if (sid >= m_references.size()) {
stringstream msg;
msg << "Sentence id (" << sid << ") not found in reference set";
throw runtime_error(msg.str());
}
UTIL_THROW_IF2(sid >= m_references.size(), "Sentence id (" << sid << ") not found in reference set");
CalcBleuStats(*(m_references[sid]), text, entry);
}
void BleuScorer::CalcBleuStats(const Reference& ref, const std::string& text, ScoreStats& entry) const
{
NgramCounts testcounts;
// stats for this line
vector<ScoreStatsType> stats(kBleuNgramOrder * 2);
string sentence = preprocessSentence(text);
const size_t length = CountNgrams(sentence, testcounts, kBleuNgramOrder, true);
const int reference_len = CalcReferenceLength(sid, length);
const int reference_len = CalcReferenceLength(ref, length);
stats.push_back(reference_len);
//precision on each ngram type
@ -177,7 +187,7 @@ void BleuScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
NgramCounts::Value correct = 0;
NgramCounts::Value v = 0;
if (m_references[sid]->get_counts()->Lookup(testcounts_it->first, &v)) {
if (ref.get_counts()->Lookup(testcounts_it->first, &v)) {
correct = min(v, guess);
}
stats[len * 2 - 2] += correct;
@ -207,21 +217,20 @@ statscore_t BleuScorer::calculateScore(const vector<ScoreStatsType>& comps) cons
return exp(logbleu);
}
int BleuScorer::CalcReferenceLength(size_t sentence_id, size_t length)
int BleuScorer::CalcReferenceLength(const Reference& ref, std::size_t length) const
{
switch (m_ref_length_type) {
case AVERAGE:
return m_references[sentence_id]->CalcAverage();
return ref.CalcAverage();
break;
case CLOSEST:
return m_references[sentence_id]->CalcClosest(length);
return ref.CalcClosest(length);
break;
case SHORTEST:
return m_references[sentence_id]->CalcShortest();
return ref.CalcShortest();
break;
default:
cerr << "unknown reference types." << endl;
exit(1);
UTIL_THROW2("Unknown reference types");
}
}
@ -304,23 +313,15 @@ vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string&
}
vector<pair<size_t,size_t> > hypotheses;
if (featureDataIters[0] == FeatureDataIterator::end()) {
cerr << "Error: at the end of feature data iterator" << endl;
exit(1);
}
UTIL_THROW_IF2(featureDataIters[0] == FeatureDataIterator::end(),
"At the end of feature data iterator");
for (size_t i = 0; i < featureFiles.size(); ++i) {
if (featureDataIters[i] == FeatureDataIterator::end()) {
cerr << "Error: Feature file " << i << " ended prematurely" << endl;
exit(1);
}
if (scoreDataIters[i] == ScoreDataIterator::end()) {
cerr << "Error: Score file " << i << " ended prematurely" << endl;
exit(1);
}
if (featureDataIters[i]->size() != scoreDataIters[i]->size()) {
cerr << "Error: features and scores have different size" << endl;
exit(1);
}
UTIL_THROW_IF2(featureDataIters[i] == FeatureDataIterator::end(),
"Feature file " << i << " ended prematurely");
UTIL_THROW_IF2(scoreDataIters[i] == ScoreDataIterator::end(),
"Score file " << i << " ended prematurely");
UTIL_THROW_IF2(featureDataIters[i]->size() != scoreDataIters[i]->size(),
"Features and scores have different size");
for (size_t j = 0; j < featureDataIters[i]->size(); ++j) {
hypotheses.push_back(pair<size_t,size_t>(i,j));
}

32
mert/BleuScorer.h
View File

@ -1,23 +1,23 @@
#ifndef MERT_BLEU_SCORER_H_
#define MERT_BLEU_SCORER_H_
#pragma once
#include <ostream>
#include <fstream>
#include <string>
#include <vector>
#include "Types.h"
#include <boost/shared_ptr.hpp>
#include "Ngram.h"
#include "Reference.h"
#include "ScopedVector.h"
#include "ScoreData.h"
#include "StatisticsBasedScorer.h"
#include "ScopedVector.h"
#include "Types.h"
namespace MosesTuning
{
const size_t kBleuNgramOrder = 4;
class NgramCounts;
class Reference;
/**
* Bleu scoring
*/
@ -42,11 +42,14 @@ public:
return 2 * kBleuNgramOrder + 1;
}
int CalcReferenceLength(std::size_t sentence_id, std::size_t length);
void CalcBleuStats(const Reference& ref, const std::string& text, ScoreStats& entry) const;
int CalcReferenceLength(const Reference& ref, std::size_t length) const;
ReferenceLengthType GetReferenceLengthType() const {
return m_ref_length_type;
}
void SetReferenceLengthType(ReferenceLengthType type) {
m_ref_length_type = type;
}
@ -62,14 +65,16 @@ public:
/**
* Count the ngrams of each type, up to the given length in the input line.
*/
std::size_t CountNgrams(const std::string& line, NgramCounts& counts, unsigned int n, bool is_testing=false);
size_t CountNgrams(const std::string& line, NgramCounts& counts, unsigned int n, bool is_testing=false) const;
void DumpCounts(std::ostream* os, const NgramCounts& counts) const;
bool OpenReference(const char* filename, std::size_t file_id);
// NOTE: this function is used for unit testing.
virtual bool OpenReferenceStream(std::istream* is, std::size_t file_id);
bool OpenReferenceStream(std::istream* is, std::size_t file_id);
void ProcessReferenceLine(const std::string& line, Reference* ref) const;
bool GetNextReferenceFromStreams(std::vector<boost::shared_ptr<std::ifstream> >& referenceStreams, Reference& ref) const;
//private:
protected:
@ -99,4 +104,3 @@ float sentenceLevelBackgroundBleu(const std::vector<float>& sent, const std::vec
}
#endif // MERT_BLEU_SCORER_H_

6
mert/ForestRescoreTest.cpp
View File

@ -13,7 +13,8 @@
using namespace std;
using namespace MosesTuning;
BOOST_AUTO_TEST_CASE(viterbi_simple_lattice) {
BOOST_AUTO_TEST_CASE(viterbi_simple_lattice)
{
Vocab vocab;
WordVec words;
string wordStrings[] =
@ -244,7 +245,8 @@ BOOST_AUTO_TEST_CASE(viterbi_3branch_lattice)
BOOST_CHECK_EQUAL(6, hopeHypo.bleuStats[8]);
}
BOOST_AUTO_TEST_CASE(viterbi_full_hypergraph) {
BOOST_AUTO_TEST_CASE(viterbi_full_hypergraph)
{
Vocab vocab;
//References
ReferenceSet references;

2
mert/HopeFearDecoder.cpp
View File

@ -98,7 +98,7 @@ void NbestHopeFearDecoder::HopeFear(
size_t hope_index=0, fear_index=0, model_index=0;
ValType hope_score=0, fear_score=0, model_score=0;
for(size_t safe_loop=0; safe_loop<2; safe_loop++) {
ValType hope_bleu, hope_model;
ValType hope_bleu=0, hope_model=0;
for(size_t i=0; i< train_->cur_size(); i++) {
const MiraFeatureVector& vec=train_->featuresAt(i);
ValType score = wv.score(vec);

5
mert/HopeFearDecoder.h
View File

@ -16,8 +16,7 @@ You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef MERT_HOPEFEARDECODER_H
#define MERT_HOPEFEARDECODER_H
#pragma once
#include <vector>
@ -160,5 +159,3 @@ private:
};
#endif

4
mert/Jamfile
View File

@ -66,11 +66,13 @@ exe evaluator : evaluator.cpp mert_lib ;
exe sentence-bleu : sentence-bleu.cpp mert_lib ;
exe sentence-bleu-nbest : sentence-bleu-nbest.cpp mert_lib ;
exe pro : pro.cpp mert_lib ..//boost_program_options ;
exe kbmira : kbmira.cpp mert_lib ..//boost_program_options ..//boost_filesystem ;
alias programs : mert extractor evaluator pro kbmira sentence-bleu ;
alias programs : mert extractor evaluator pro kbmira sentence-bleu sentence-bleu-nbest ;
unit-test bleu_scorer_test : BleuScorerTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test feature_data_test : FeatureDataTest.cpp mert_lib ..//boost_unit_test_framework ;

3
mert/MiraFeatureVectorTest.cpp
View File

@ -11,7 +11,8 @@ how many of the features are really "dense". This is because in hg mira
all features (sparse and dense) are to get rolled in to SparseVector
*/
BOOST_AUTO_TEST_CASE(from_sparse) {
BOOST_AUTO_TEST_CASE(from_sparse)
{
SparseVector sp;
sp.set("dense0", 0.2);
sp.set("dense1", 0.3);

4
mert/Ngram.h
View File

@ -1,5 +1,4 @@
#ifndef MERT_NGRAM_H_
#define MERT_NGRAM_H_
#pragma once
#include <vector>
#include <string>
@ -121,4 +120,3 @@ private:
}
#endif // MERT_NGRAM_H_

5
mert/Reference.h
View File

@ -59,6 +59,11 @@ public:
int CalcClosest(std::size_t length) const;
int CalcShortest() const;
void clear() {
m_length.clear();
m_counts->clear();
}
private:
NgramCounts* m_counts;

4
mert/Scorer.cpp
View File

@ -64,7 +64,7 @@ void Scorer::InitConfig(const string& config)
}
}
void Scorer::TokenizeAndEncode(const string& line, vector<int>& encoded)
void Scorer::TokenizeAndEncode(const string& line, vector<int>& encoded) const
{
for (util::TokenIter<util::AnyCharacter, true> it(line, util::AnyCharacter(" "));
it; ++it) {
@ -81,7 +81,7 @@ void Scorer::TokenizeAndEncode(const string& line, vector<int>& encoded)
}
}
void Scorer::TokenizeAndEncodeTesting(const string& line, vector<int>& encoded)
void Scorer::TokenizeAndEncodeTesting(const string& line, vector<int>& encoded) const
{
for (util::TokenIter<util::AnyCharacter, true> it(line, util::AnyCharacter(" "));
it; ++it) {

8
mert/Scorer.h
View File

@ -1,5 +1,4 @@
#ifndef MERT_SCORER_H_
#define MERT_SCORER_H_
#pragma once
#include <iostream>
#include <sstream>
@ -187,12 +186,12 @@ protected:
* Tokenise line and encode.
* Note: We assume that all tokens are separated by whitespaces.
*/
void TokenizeAndEncode(const std::string& line, std::vector<int>& encoded);
void TokenizeAndEncode(const std::string& line, std::vector<int>& encoded) const;
/*
* Tokenize functions for testing only.
*/
void TokenizeAndEncodeTesting(const std::string& line, std::vector<int>& encoded);
void TokenizeAndEncodeTesting(const std::string& line, std::vector<int>& encoded) const;
/**
* Every inherited scorer should call this function for each sentence
@ -236,4 +235,3 @@ inline float score_average(const statscores_t& scores, size_t start, size_t end)
}
#endif // MERT_SCORER_H_

66
mert/sentence-bleu-nbest.cpp Normal file
View File

@ -0,0 +1,66 @@
#include <fstream>
#include <iostream>
#include <vector>
#include <string>
#include <boost/shared_ptr.hpp>
#include "BleuScorer.h"
#include "Reference.h"
#include "moses/Util.h"
#include "util/exception.hh"
using namespace MosesTuning;
int main(int argc, char **argv)
{
if (argc == 1) {
std::cerr << "Usage: ./sentence-bleu-nbest ref1 [ref2 ...] < plain-nbest > bleu-scores" << std::endl;
return 1;
}
std::vector<std::string> refFiles(argv + 1, argv + argc);
// TODO all of these are empty for now
std::string config;
std::string factors;
std::string filter;
BleuScorer scorer(config);
scorer.setFactors(factors);
scorer.setFilter(filter);
// initialize reference streams
std::vector<boost::shared_ptr<std::ifstream> > refStreams;
for (std::vector<std::string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) {
TRACE_ERR("Loading reference from " << *refFile << std::endl);
boost::shared_ptr<std::ifstream> ifs(new std::ifstream(refFile->c_str()));
UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile);
refStreams.push_back(ifs);
}
// load sentences, preparing statistics, score
std::string nbestLine;
int sid = -1;
Reference ref;
while ( getline(std::cin, nbestLine) ) {
std::vector<std::string> items;
Moses::TokenizeMultiCharSeparator(items, nbestLine, " ||| ");
int sidCurrent = Moses::Scan<int>(items[0]);
if (sidCurrent != sid) {
ref.clear();
if (!scorer.GetNextReferenceFromStreams(refStreams, ref)) {
UTIL_THROW2("Missing references");
}
sid = sidCurrent;
}
ScoreStats scoreStats;
scorer.CalcBleuStats(ref, items[1], scoreStats);
std::vector<float> stats(scoreStats.getArray(), scoreStats.getArray() + scoreStats.size());
std::cout << smoothedSentenceBleu(stats) << std::endl;
}
return 0;
}

43
mert/sentence-bleu.cpp
View File

@ -1,18 +1,26 @@
#include <fstream>
#include <iostream>
#include <vector>
#include <string>
#include <boost/shared_ptr.hpp>
#include "BleuScorer.h"
#include "Reference.h"
#include "moses/Util.h"
#include "util/exception.hh"
using namespace std;
using namespace MosesTuning;
int main(int argc, char **argv)
{
if (argc == 1) {
cerr << "Usage: ./sentence-bleu ref1 [ref2 ...] < candidate > bleu-scores" << endl;
return 1;
}
vector<string> refFiles(argv + 1, argv + argc);
// TODO all of these are empty for now
@ -23,22 +31,31 @@ int main(int argc, char **argv)
BleuScorer scorer(config);
scorer.setFactors(factors);
scorer.setFilter(filter);
scorer.setReferenceFiles(refFiles);
vector<ScoreStats> entries;
// Loading sentences and preparing statistics
ScoreStats scoreentry;
string line;
while (getline(cin, line)) {
scorer.prepareStats(entries.size(), line, scoreentry);
entries.push_back(scoreentry);
// initialize reference streams
vector<boost::shared_ptr<ifstream> > refStreams;
for (vector<string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) {
TRACE_ERR("Loading reference from " << *refFile << endl);
boost::shared_ptr<ifstream> ifs(new ifstream(refFile->c_str()));
UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile);
refStreams.push_back(ifs);
}
vector<ScoreStats>::const_iterator sentIt;
for (sentIt = entries.begin(); sentIt != entries.end(); sentIt++) {
vector<float> stats(sentIt->getArray(), sentIt->getArray() + sentIt->size());
cout << smoothedSentenceBleu(stats) << "\n";
// load sentences, preparing statistics, score
string hypothesisLine;
size_t sid = 0;
while (getline(std::cin, hypothesisLine)) {
Reference ref;
if (!scorer.GetNextReferenceFromStreams(refStreams, ref)) {
UTIL_THROW2("Missing references");
}
ScoreStats scoreStats;
scorer.CalcBleuStats(ref, hypothesisLine, scoreStats);
vector<float> stats(scoreStats.getArray(), scoreStats.getArray() + scoreStats.size());
std::cout << smoothedSentenceBleu(stats) << std::endl;
++sid;
}
return 0;
}

15
moses-cmd/LatticeMBRGrid.cpp
View File

@ -177,8 +177,7 @@ int main(int argc, char* argv[])
const vector<float>& scale_grid = grid.getGrid(lmbr_scale);
boost::shared_ptr<InputType> source;
while((source = ioWrapper->ReadInput()) != NULL)
{
while((source = ioWrapper->ReadInput()) != NULL) {
// set up task of translating one sentence
boost::shared_ptr<TranslationTask> ttask;
ttask = TranslationTask::create(source, ioWrapper);
@ -187,17 +186,13 @@ int main(int argc, char* argv[])
TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true);
//grid search
BOOST_FOREACH(float const& p, pgrid)
{
BOOST_FOREACH(float const& p, pgrid) {
SD.SetLatticeMBRPrecision(p);
BOOST_FOREACH(float const& r, rgrid)
{
BOOST_FOREACH(float const& r, rgrid) {
SD.SetLatticeMBRPRatio(r);
BOOST_FOREACH(size_t const prune_i, prune_grid)
{
BOOST_FOREACH(size_t const prune_i, prune_grid) {
SD.SetLatticeMBRPruningFactor(size_t(prune_i));
BOOST_FOREACH(float const& scale_i, scale_grid)
{
BOOST_FOREACH(float const& scale_i, scale_grid) {
SD.SetMBRScale(scale_i);
size_t lineCount = source->GetTranslationId();
cout << lineCount << " ||| " << p << " "

7
moses-cmd/MainVW.cpp
View File

@ -146,9 +146,10 @@ int main(int argc, char** argv)
// main loop over set of input sentences
boost::shared_ptr<InputType> source;
while ((source = ioWrapper->ReadInput()) != NULL)
{
IFVERBOSE(1) { ResetUserTime(); }
while ((source = ioWrapper->ReadInput()) != NULL) {
IFVERBOSE(1) {
ResetUserTime();
}
InputType* foo = source.get();
FeatureFunction::CallChangeSource(foo);

4
moses/BaseManager.cpp
View File

@ -17,7 +17,9 @@ BaseManager::BaseManager(ttasksptr const& ttask)
const InputType&
BaseManager::GetSource() const
{ return m_source; }
{
return m_source;
}

25
moses/ContextScope.h
View File

@ -18,16 +18,16 @@
namespace Moses
{
class ContextScope
{
protected:
class ContextScope
{
protected:
typedef std::map<void const*, boost::shared_ptr<void> > scratchpad_t;
typedef scratchpad_t::iterator iter_t;
typedef scratchpad_t::value_type entry_t;
typedef scratchpad_t::const_iterator const_iter_t;
scratchpad_t m_scratchpad;
mutable boost::shared_mutex m_lock;
public:
public:
// class write_access
// {
// boost::unique_lock<boost::shared_mutex> m_lock;
@ -50,26 +50,22 @@ namespace Moses
template<typename T>
boost::shared_ptr<void> const&
set(void const* const key, boost::shared_ptr<T> const& val)
{
set(void const* const key, boost::shared_ptr<T> const& val) {
boost::unique_lock<boost::shared_mutex> lock(m_lock);
return (m_scratchpad[key] = val);
}
template<typename T>
boost::shared_ptr<T> const
get(void const* key, bool CreateNewIfNecessary=false)
{
get(void const* key, bool CreateNewIfNecessary=false) {
using boost::shared_mutex;
using boost::upgrade_lock;
// T const* key = reinterpret_cast<T const*>(xkey);
upgrade_lock<shared_mutex> lock(m_lock);
iter_t m = m_scratchpad.find(key);
boost::shared_ptr< T > ret;
if (m != m_scratchpad.end())
{
if (m->second == NULL && CreateNewIfNecessary)
{
if (m != m_scratchpad.end()) {
if (m->second == NULL && CreateNewIfNecessary) {
boost::upgrade_to_unique_lock<shared_mutex> xlock(lock);
m->second.reset(new T);
}
@ -85,13 +81,12 @@ namespace Moses
ContextScope() { }
ContextScope(ContextScope const& other)
{
ContextScope(ContextScope const& other) {
boost::unique_lock<boost::shared_mutex> lock1(this->m_lock);
boost::unique_lock<boost::shared_mutex> lock2(other.m_lock);
m_scratchpad = other.m_scratchpad;
}
};
};
};

9
moses/DecodeStepTranslation.cpp
View File

@ -223,8 +223,7 @@ ProcessLEGACY(TranslationOption const& in,
TranslationOptionCollection *toc,
bool adhereTableLimit) const
{
if (in.GetTargetPhrase().GetSize() == 0)
{
if (in.GetTargetPhrase().GetSize() == 0) {
// word deletion
out.Add(new TranslationOption(in));
return;
@ -241,16 +240,14 @@ ProcessLEGACY(TranslationOption const& in,
TargetPhraseCollectionWithSourcePhrase const* phraseColl;
phraseColl = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange);
if (phraseColl != NULL)
{
if (phraseColl != NULL) {
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
iterEnd = ((adhereTableLimit && tableLimit && phraseColl->GetSize() >= tableLimit)
? phraseColl->begin() + tableLimit : phraseColl->end());
for (iterTargetPhrase = phraseColl->begin();
iterTargetPhrase != iterEnd;
++iterTargetPhrase)
{
++iterTargetPhrase) {
TargetPhrase const& targetPhrase = **iterTargetPhrase;
if (targetPhrase.GetSize() != currSize ||
(IsFilteringStep() && !in.IsCompatible(targetPhrase, m_conflictFactors)))

45
moses/ExportInterface.cpp
View File

@ -114,7 +114,9 @@ string SimpleTranslationInterface::translate(const string &inputString)
boost::shared_ptr<InputType> source = ioWrapper->ReadInput();
if (!source) return "Error: Source==null!!!";
IFVERBOSE(1) { ResetUserTime(); }
IFVERBOSE(1) {
ResetUserTime();
}
FeatureFunction::CallChangeSource(&*source);
@ -147,10 +149,14 @@ int
run_as_server()
{
#ifdef HAVE_XMLRPC_C
int port; params.SetParameter(port, "server-port", 8080);
bool isSerial; params.SetParameter(isSerial, "serial", false);
string logfile; params.SetParameter(logfile, "server-log", string(""));
size_t num_threads; params.SetParameter(num_threads, "threads", size_t(10));
int port;
params.SetParameter(port, "server-port", 8080);
bool isSerial;
params.SetParameter(isSerial, "serial", false);
string logfile;
params.SetParameter(logfile, "server-log", string(""));
size_t num_threads;
params.SetParameter(num_threads, "threads", size_t(10));
if (isSerial) VERBOSE(1,"Running server in serial mode." << endl);
xmlrpc_c::registry myRegistry;
@ -166,8 +172,9 @@ run_as_server()
xmlrpc_c::serverAbyss myAbyssServer(myRegistry, port, logfile);
XVERBOSE(1,"Listening on port " << port << endl);
if (isSerial) { while(1) myAbyssServer.runOnce(); }
else myAbyssServer.run();
if (isSerial) {
while(1) myAbyssServer.runOnce();
} else myAbyssServer.run();
std::cerr << "xmlrpc_c::serverAbyss.run() returned but should not." << std::endl;
// #pragma message("BUILDING MOSES WITH SERVER SUPPORT")
@ -197,8 +204,7 @@ batch_run()
// check on weights
const ScoreComponentCollection& weights = staticData.GetAllWeights();
IFVERBOSE(2)
{
IFVERBOSE(2) {
TRACE_ERR("The global weight vector looks like this: ");
TRACE_ERR(weights);
TRACE_ERR("\n");
@ -214,8 +220,7 @@ batch_run()
// main loop over set of input sentences
boost::shared_ptr<InputType> source;
while ((source = ioWrapper->ReadInput()) != NULL)
{
while ((source = ioWrapper->ReadInput()) != NULL) {
IFVERBOSE(1) ResetUserTime();
FeatureFunction::CallChangeSource(source.get());
@ -236,8 +241,7 @@ batch_run()
// simulated post-editing requires threads (within the dynamic phrase tables)
// but runs all sentences serially, to allow updating of the bitext.
bool spe = params.isParamSpecified("spe-src");
if (spe)
{
if (spe) {
// simulated post-editing: always run single-threaded!
task->Run();
string src,trg,aln;
@ -247,16 +251,14 @@ batch_run()
<< "missing update data for simulated post-editing.");
UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
<< "missing update data for simulated post-editing.");
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl())
{
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) {
Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
if (sapt) sapt->add(src,trg,aln);
VERBOSE(1,"[" << HERE << " added src] " << src << endl);
VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
}
}
else pool.Submit(task);
} else pool.Submit(task);
#else
pool.Submit(task);
@ -295,8 +297,7 @@ int decoder_main(int argc, char** argv)
#endif
// echo command line, if verbose
IFVERBOSE(1)
{
IFVERBOSE(1) {
TRACE_ERR("command: ");
for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
TRACE_ERR(endl);
@ -317,8 +318,7 @@ int decoder_main(int argc, char** argv)
exit(1);
// setting "-show-weights" -> just dump out weights and exit
if (params.isParamSpecified("show-weights"))
{
if (params.isParamSpecified("show-weights")) {
ShowWeights();
exit(0);
}
@ -330,8 +330,7 @@ int decoder_main(int argc, char** argv)
}
#ifdef NDEBUG
catch (const std::exception &e)
{
catch (const std::exception &e) {
std::cerr << "Exception: " << e.what() << std::endl;
return EXIT_FAILURE;
}

4
moses/ExportInterface.h
View File

@ -45,7 +45,9 @@ public:
~SimpleTranslationInterface();
std::string translate(const std::string &input);
Moses::StaticData& getStaticData();
Moses::Parameter& getParameters(){ return m_params; }
Moses::Parameter& getParameters() {
return m_params;
}
private:
SimpleTranslationInterface();
Moses::Parameter m_params;

16
moses/FF/Factory.cpp
View File

@ -157,20 +157,15 @@ FeatureFactory
std::vector<float> weights = static_data.GetParameter()->GetWeights(featureName);
if (feature->GetNumScoreComponents())
{
if (weights.size() == 0)
{
if (feature->GetNumScoreComponents()) {
if (weights.size() == 0) {
weights = feature->DefaultWeights();
if (weights.size() == 0)
{
if (weights.size() == 0) {
TRACE_ERR("WARNING: No weights specified in config file for FF "
<< featureName << ". This FF does not supply default values.\n"
<< "WARNING: Auto-initializing all weights for this FF to 1.0");
weights.assign(feature->GetNumScoreComponents(),1.0);
}
else
{
} else {
TRACE_ERR("WARNING: No weights specified in config file for FF "
<< featureName << ". Using default values supplied by FF.");
}
@ -181,8 +176,7 @@ FeatureFactory
<< " (features: " << feature->GetNumScoreComponents()
<< " vs. weights: " << weights.size() << ")");
static_data.SetWeights(feature, weights);
}
else if (feature->IsTuneable())
} else if (feature->IsTuneable())
static_data.SetWeights(feature, weights);
}

12
moses/FF/FeatureFunction.cpp
View File

@ -193,17 +193,23 @@ void FeatureFunction::SetTuneableComponents(const std::string& value)
void
FeatureFunction
::InitializeForInput(ttasksptr const& ttask)
{ InitializeForInput(*(ttask->GetSource().get())); }
{
InitializeForInput(*(ttask->GetSource().get()));
}
void
FeatureFunction
::CleanUpAfterSentenceProcessing(ttasksptr const& ttask)
{ CleanUpAfterSentenceProcessing(*(ttask->GetSource().get())); }
{
CleanUpAfterSentenceProcessing(*(ttask->GetSource().get()));
}
size_t
FeatureFunction
::GetIndex() const
{ return m_index; }
{
return m_index;
}
/// set index

4
moses/FF/FeatureFunction.h
View File

@ -136,7 +136,9 @@ public:
CleanUpAfterSentenceProcessing(ttasksptr const& ttask);
const std::string &
GetArgLine() const { return m_argLine; }
GetArgLine() const {
return m_argLine;
}
// given a target phrase containing only factors specified in mask
// return true if the feature function can be evaluated

3
moses/FF/InternalTree.cpp
View File

@ -147,8 +147,7 @@ void InternalTree::GetUnbinarizedChildren(std::vector<TreePointer> &ret) const
const std::string &label = (*itx)->GetLabel();
if (!label.empty() && label[0] == '^') {
(*itx)->GetUnbinarizedChildren(ret);
}
else {
} else {
ret.push_back(*itx);
}
}

9
moses/FF/InternalTree.h
View File

@ -96,8 +96,7 @@ public:
bool RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const;
// Python-like generator that yields next nonterminal leaf on every call
$generator(leafNT)
{
$generator(leafNT) {
std::vector<TreePointer>::iterator it;
InternalTree* tree;
leafNT(InternalTree* root = 0): tree(root) {}
@ -116,8 +115,7 @@ public:
// Python-like generator that yields the parent of the next nonterminal leaf on every call
$generator(leafNTParent)
{
$generator(leafNTParent) {
std::vector<TreePointer>::iterator it;
InternalTree* tree;
leafNTParent(InternalTree* root = 0): tree(root) {}
@ -135,8 +133,7 @@ public:
};
// Python-like generator that yields the next nonterminal leaf on every call, and also stores the path from the root of the tree to the nonterminal
$generator(leafNTPath)
{
$generator(leafNTPath) {
std::vector<TreePointer>::iterator it;
InternalTree* tree;
std::vector<InternalTree*> * path;

3
moses/FF/LexicalReordering/LexicalReorderingState.h
View File

@ -44,8 +44,7 @@ public:
static const ReorderingType L = 1; // left
static const ReorderingType MAX = 3; // largest possible
#else
enum ReorderingType
{
enum ReorderingType {
M = 0, // monotonic
NM = 1, // non-monotonic
S = 1, // swap

45
moses/FF/Model1Feature.cpp
View File

@ -71,21 +71,18 @@ void Model1Vocabulary::Load(const std::string& fileName)
std::string line;
unsigned i = 0;
if ( getline(inFile, line) ) // first line of MGIZA vocabulary files seems to be special : "1 UNK 0" -- skip if it's this
{
if ( getline(inFile, line) ) { // first line of MGIZA vocabulary files seems to be special : "1 UNK 0" -- skip if it's this
++i;
std::vector<std::string> tokens = Tokenize(line);
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
unsigned id = Scan<unsigned>(tokens[0]);
if (! ( (id == 1) && (tokens[1] == "UNK") ))
{
if (! ( (id == 1) && (tokens[1] == "UNK") )) {
const Factor* factor = factorCollection.AddFactor(tokens[1],false); // TODO: can we assume that the vocabulary is know and filter the model on loading?
bool stored = Store(factor, id);
UTIL_THROW_IF2(!stored, "Line " << i << " in " << fileName << " overwrites existing vocabulary entry.");
}
}
while ( getline(inFile, line) )
{
while ( getline(inFile, line) ) {
++i;
std::vector<std::string> tokens = Tokenize(line);
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
@ -104,8 +101,7 @@ void Model1LexicalTable::Load(const std::string &fileName, const Model1Vocabular
std::string line;
unsigned i = 0;
while ( getline(inFile, line) )
{
while ( getline(inFile, line) ) {
++i;
std::vector<std::string> tokens = Tokenize(line);
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
@ -193,25 +189,21 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
float score = 0.0;
float norm = TransformScore(1+sentence.GetSize());
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT)
{
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
const Word &wordT = targetPhrase.GetWord(posT);
if ( !wordT.IsNonTerminal() )
{
if ( !wordT.IsNonTerminal() ) {
float thisWordProb = m_model1.GetProbability(m_emptyWord,wordT[0]); // probability conditioned on empty word
// cache lookup
bool foundInCache = false;
{
#ifdef WITH_THREADS
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#endif
#endif
boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::const_iterator sentenceCache = m_cache.find(&input);
if (sentenceCache != m_cache.end())
{
if (sentenceCache != m_cache.end()) {
boost::unordered_map<const Factor*, float>::const_iterator cacheHit = sentenceCache->second.find(wordT[0]);
if (cacheHit != sentenceCache->second.end())
{
if (cacheHit != sentenceCache->second.end()) {
foundInCache = true;
score += cacheHit->second;
FEATUREVERBOSE(3, "Cached score( " << wordT << " ) = " << cacheHit->second << std::endl);
@ -219,10 +211,8 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
}
}
if (!foundInCache)
{
for (size_t posS=1; posS<sentence.GetSize()-1; ++posS) // ignore <s> and </s>
{
if (!foundInCache) {
for (size_t posS=1; posS<sentence.GetSize()-1; ++posS) { // ignore <s> and </s>
const Word &wordS = sentence.GetWord(posS);
float modelProb = m_model1.GetProbability(wordS[0],wordT[0]);
FEATUREVERBOSE(4, "p( " << wordT << " | " << wordS << " ) = " << modelProb << std::endl);
@ -231,10 +221,10 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
float thisWordScore = TransformScore(thisWordProb) - norm;
FEATUREVERBOSE(3, "score( " << wordT << " ) = " << thisWordScore << std::endl);
{
#ifdef WITH_THREADS
#ifdef WITH_THREADS
// need to update cache; write lock
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
#endif
m_cache[&input][wordT[0]] = thisWordScore;
}
score += thisWordScore;
@ -247,14 +237,13 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
void Model1Feature::CleanUpAfterSentenceProcessing(const InputType& source)
{
#ifdef WITH_THREADS
#ifdef WITH_THREADS
// need to update cache; write lock
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
#endif
// clear cache
boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::iterator sentenceCache = m_cache.find(&source);
if (sentenceCache != m_cache.end())
{
if (sentenceCache != m_cache.end()) {
sentenceCache->second.clear();
m_cache.erase(sentenceCache);
}

6
moses/FF/Model1Feature.h
View File

@ -17,7 +17,7 @@ class Model1Vocabulary
{
public:
#define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX
#define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX
static const std::string GIZANULL;
Model1Vocabulary();
@ -103,10 +103,10 @@ private:
// cache
mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache;
#ifdef WITH_THREADS
#ifdef WITH_THREADS
// reader-writer lock
mutable boost::shared_mutex m_accessLock;
#endif
#endif
};

9
moses/FF/RulePairUnlexicalizedSource.cpp
View File

@ -51,8 +51,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
return;
}
for (size_t posS=0; posS<source.GetSize(); ++posS)
{
for (size_t posS=0; posS<source.GetSize(); ++posS) {
const Word &wordS = source.GetWord(posS);
if ( !wordS.IsNonTerminal() ) {
return;
@ -61,8 +60,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
ostringstream namestr;
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT)
{
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
const Word &wordT = targetPhrase.GetWord(posT);
const Factor* factorT = wordT[0];
if ( wordT.IsNonTerminal() ) {
@ -78,8 +76,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
namestr << targetPhraseLHS->GetString() << "|";
for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
it!=targetPhrase.GetAlignNonTerm().end(); ++it)
{
it!=targetPhrase.GetAlignNonTerm().end(); ++it) {
namestr << "|" << it->first << "-" << it->second;
}

18
moses/FF/RuleScope.cpp
View File

@ -68,15 +68,12 @@ void RuleScope::EvaluateInIsolation(const Phrase &source
if (m_futureCostOnly) {
estimatedFutureScore.PlusEquals(this, scores);
}
else {
} else {
scoreBreakdown.PlusEquals(this, scores);
}
}
else if (m_futureCostOnly) {
} else if (m_futureCostOnly) {
estimatedFutureScore.PlusEquals(this, score);
}
else {
} else {
scoreBreakdown.PlusEquals(this, score);
}
}
@ -85,14 +82,11 @@ void RuleScope::SetParameter(const std::string& key, const std::string& value)
{
if (key == "source-syntax") {
m_sourceSyntax = Scan<bool>(value);
}
else if (key == "per-scope") {
} else if (key == "per-scope") {
m_perScope = Scan<bool>(value);
}
else if ("future-cost-only") {
} else if ("future-cost-only") {
m_futureCostOnly = Scan<bool>(value);
}
else {
} else {
StatelessFeatureFunction::SetParameter(key, value);
}
}

335
moses/Hypothesis.cpp
View File

@ -45,11 +45,11 @@ namespace Moses
{
#ifdef USE_HYPO_POOL
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
#endif
Hypothesis::
Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt)
Hypothesis::
Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt)
: m_prevHypo(NULL)
, m_sourceCompleted(source.GetSize(), manager.GetSource().m_sourceCompleted)
, m_sourceInput(source)
@ -65,7 +65,7 @@ namespace Moses
, m_transOpt(initialTransOpt)
, m_manager(manager)
, m_id(m_manager.GetNextHypoId())
{
{
// used for initial seeding of trans process
// initialize scores
//_hash_computed = false;
@ -74,13 +74,13 @@ namespace Moses
for (unsigned i = 0; i < ffs.size(); ++i)
m_ffStates[i] = ffs[i]->EmptyHypothesisState(source);
m_manager.GetSentenceStats().AddCreated();
}
}
/***
/***
* continue prevHypo by appending the phrases in transOpt
*/
Hypothesis::
Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt)
Hypothesis::
Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt)
: m_prevHypo(&prevHypo)
, m_sourceCompleted(prevHypo.m_sourceCompleted )
, m_sourceInput(prevHypo.m_sourceInput)
@ -96,7 +96,7 @@ namespace Moses
, m_transOpt(transOpt)
, m_manager(prevHypo.GetManager())
, m_id(m_manager.GetNextHypoId())
{
{
m_currScoreBreakdown.PlusEquals(transOpt.GetScoreBreakdown());
// assert that we are not extending our hypothesis by retranslating something
@ -107,11 +107,11 @@ namespace Moses
m_sourceCompleted.SetValue(m_currSourceWordsRange.GetStartPos(), m_currSourceWordsRange.GetEndPos(), true);
m_wordDeleted = transOpt.IsDeletionOption();
m_manager.GetSentenceStats().AddCreated();
}
}
Hypothesis::
~Hypothesis()
{
Hypothesis::
~Hypothesis()
{
for (unsigned i = 0; i < m_ffStates.size(); ++i)
delete m_ffStates[i];
@ -125,12 +125,12 @@ namespace Moses
delete m_arcList;
m_arcList = NULL;
}
}
}
void
Hypothesis::
AddArc(Hypothesis *loserHypo)
{
void
Hypothesis::
AddArc(Hypothesis *loserHypo)
{
if (!m_arcList) {
if (loserHypo->m_arcList) { // we don't have an arcList, but loser does
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
@ -151,25 +151,25 @@ namespace Moses
}
}
m_arcList->push_back(loserHypo);
}
}
/***
/***
* return the subclass of Hypothesis most appropriate to the given translation option
*/
Hypothesis*
Hypothesis::
CreateNext(const TranslationOption &transOpt) const
{
Hypothesis*
Hypothesis::
CreateNext(const TranslationOption &transOpt) const
{
return Create(*this, transOpt);
}
}
/***
/***
* return the subclass of Hypothesis most appropriate to the given translation option
*/
Hypothesis*
Hypothesis::
Create(const Hypothesis &prevHypo, const TranslationOption &transOpt)
{
Hypothesis*
Hypothesis::
Create(const Hypothesis &prevHypo, const TranslationOption &transOpt)
{
#ifdef USE_HYPO_POOL
Hypothesis *ptr = s_objectPool.getPtr();
@ -177,33 +177,33 @@ namespace Moses
#else
return new Hypothesis(prevHypo, transOpt);
#endif
}
/***
}
/***
* return the subclass of Hypothesis most appropriate to the given target phrase
*/
Hypothesis*
Hypothesis::
Create(Manager& manager, InputType const& m_source,
Hypothesis*
Hypothesis::
Create(Manager& manager, InputType const& m_source,
const TranslationOption &initialTransOpt)
{
{
#ifdef USE_HYPO_POOL
Hypothesis *ptr = s_objectPool.getPtr();
return new(ptr) Hypothesis(manager, m_source, initialTransOpt);
#else
return new Hypothesis(manager, m_source, initialTransOpt);
#endif
}
}
/** check, if two hypothesis can be recombined.
/** check, if two hypothesis can be recombined.
this is actually a sorting function that allows us to
keep an ordered list of hypotheses. This makes recombination
much quicker.
*/
int
Hypothesis::
RecombineCompare(const Hypothesis &compare) const
{
*/
int
Hypothesis::
RecombineCompare(const Hypothesis &compare) const
{
// -1 = this < compare
// +1 = this > compare
// 0 = this ==compare
@ -221,40 +221,39 @@ namespace Moses
}
return 0;
}
}
void
Hypothesis::
EvaluateWhenApplied(StatefulFeatureFunction const& sfff,
void
Hypothesis::
EvaluateWhenApplied(StatefulFeatureFunction const& sfff,
int state_idx)
{
{
const StaticData &staticData = StaticData::Instance();
if (! staticData.IsFeatureFunctionIgnored( sfff ))
{
if (! staticData.IsFeatureFunctionIgnored( sfff )) {
m_ffStates[state_idx]
= sfff.EvaluateWhenApplied
(*this, m_prevHypo ? m_prevHypo->m_ffStates[state_idx] : NULL,
&m_currScoreBreakdown);
}
}
}
void
Hypothesis::
EvaluateWhenApplied(const StatelessFeatureFunction& slff)
{
void
Hypothesis::
EvaluateWhenApplied(const StatelessFeatureFunction& slff)
{
const StaticData &staticData = StaticData::Instance();
if (! staticData.IsFeatureFunctionIgnored( slff )) {
slff.EvaluateWhenApplied(*this, &m_currScoreBreakdown);
}
}
}
/***
/***
* calculate the logarithm of our total translation score (sum up components)
*/
void
Hypothesis::
EvaluateWhenApplied(const SquareMatrix &futureScore)
{
void
Hypothesis::
EvaluateWhenApplied(const SquareMatrix &futureScore)
{
IFVERBOSE(2) {
m_manager.GetSentenceStats().StartTimeOtherScore();
}
@ -299,20 +298,20 @@ namespace Moses
IFVERBOSE(2) {
m_manager.GetSentenceStats().StopTimeEstimateScore();
}
}
}
const Hypothesis* Hypothesis::GetPrevHypo()const
{
const Hypothesis* Hypothesis::GetPrevHypo()const
{
return m_prevHypo;
}
}
/**
/**
* print hypothesis information for pharaoh-style logging
*/
void
Hypothesis::
PrintHypothesis() const
{
void
Hypothesis::
PrintHypothesis() const
{
if (!m_prevHypo) {
TRACE_ERR(endl << "NULL hypo" << endl);
return;
@ -344,12 +343,12 @@ namespace Moses
TRACE_ERR( "\tscore "<<m_totalScore - m_futureScore<<" + future cost "<<m_futureScore<<" = "<<m_totalScore<<endl);
TRACE_ERR( "\tunweighted feature scores: " << m_currScoreBreakdown << endl);
//PrintLMScores();
}
}
void
Hypothesis::
CleanupArcList()
{
void
Hypothesis::
CleanupArcList()
{
// point this hypo's main hypo to itself
SetWinningHypo(this);
@ -369,8 +368,7 @@ namespace Moses
staticData.GetOutputSearchGraphHypergraph() ||
staticData.UseLatticeMBR());
if (!distinctNBest && m_arcList->size() > nBestSize * 5)
{
if (!distinctNBest && m_arcList->size() > nBestSize * 5) {
// prune arc list only if there too many arcs
NTH_ELEMENT4(m_arcList->begin(), m_arcList->begin() + nBestSize - 1,
m_arcList->end(), CompareHypothesisTotalScore());
@ -388,27 +386,29 @@ namespace Moses
Hypothesis *arc = *iter;
arc->SetWinningHypo(this);
}
}
}
TargetPhrase const&
Hypothesis::
GetCurrTargetPhrase() const
{ return m_transOpt.GetTargetPhrase(); }
TargetPhrase const&
Hypothesis::
GetCurrTargetPhrase() const
{
return m_transOpt.GetTargetPhrase();
}
void
Hypothesis::
GetOutputPhrase(Phrase &out) const
{
void
Hypothesis::
GetOutputPhrase(Phrase &out) const
{
if (m_prevHypo != NULL)
m_prevHypo->GetOutputPhrase(out);
out.Append(GetCurrTargetPhrase());
}
}
TO_STRING_BODY(Hypothesis)
TO_STRING_BODY(Hypothesis)
// friend
ostream& operator<<(ostream& out, const Hypothesis& hypo)
{
// friend
ostream& operator<<(ostream& out, const Hypothesis& hypo)
{
hypo.ToStream(out);
// words bitmap
out << "[" << hypo.m_sourceCompleted << "] ";
@ -421,43 +421,47 @@ namespace Moses
out << " " << hypo.GetCurrTargetPhrase().GetAlignNonTerm();
return out;
}
}
std::string
Hypothesis::
GetSourcePhraseStringRep(const vector<FactorType> factorsToPrint) const
{ return m_transOpt.GetInputPath().GetPhrase().GetStringRep(factorsToPrint); }
std::string
Hypothesis::
GetSourcePhraseStringRep(const vector<FactorType> factorsToPrint) const
{
return m_transOpt.GetInputPath().GetPhrase().GetStringRep(factorsToPrint);
}
std::string
Hypothesis::
GetTargetPhraseStringRep(const vector<FactorType> factorsToPrint) const
{ return (m_prevHypo ? GetCurrTargetPhrase().GetStringRep(factorsToPrint) : ""); }
std::string
Hypothesis::
GetTargetPhraseStringRep(const vector<FactorType> factorsToPrint) const
{
return (m_prevHypo ? GetCurrTargetPhrase().GetStringRep(factorsToPrint) : "");
}
std::string
Hypothesis::
GetSourcePhraseStringRep() const
{
std::string
Hypothesis::
GetSourcePhraseStringRep() const
{
vector<FactorType> allFactors(MAX_NUM_FACTORS);
for(size_t i=0; i < MAX_NUM_FACTORS; i++)
allFactors[i] = i;
return GetSourcePhraseStringRep(allFactors);
}
}
std::string
Hypothesis::
GetTargetPhraseStringRep() const
{
std::string
Hypothesis::
GetTargetPhraseStringRep() const
{
vector<FactorType> allFactors(MAX_NUM_FACTORS);
for(size_t i=0; i < MAX_NUM_FACTORS; i++)
allFactors[i] = i;
return GetTargetPhraseStringRep(allFactors);
}
}
void
Hypothesis::
OutputAlignment(std::ostream &out) const
{
void
Hypothesis::
OutputAlignment(std::ostream &out) const
{
std::vector<const Hypothesis *> edges;
const Hypothesis *currentHypo = this;
while (currentHypo) {
@ -467,12 +471,12 @@ namespace Moses
OutputAlignment(out, edges);
}
}
void
Hypothesis::
OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges)
{
void
Hypothesis::
OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges)
{
size_t targetOffset = 0;
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
@ -485,13 +489,13 @@ namespace Moses
targetOffset += tp.GetSize();
}
// Used by --print-alignment-info, so no endl
}
}
void
Hypothesis::
OutputAlignment(ostream &out, const AlignmentInfo &ai,
void
Hypothesis::
OutputAlignment(ostream &out, const AlignmentInfo &ai,
size_t sourceOffset, size_t targetOffset)
{
{
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
AlignVec alignments = ai.GetSortedAlignments();
@ -501,51 +505,51 @@ namespace Moses
out << alignment.first + sourceOffset << "-" << alignment.second + targetOffset << " ";
}
}
}
void
Hypothesis::
OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
{
void
Hypothesis::
OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
{
if (!hypo->GetPrevHypo()) return;
OutputInput(map, hypo->GetPrevHypo());
map[hypo->GetCurrSourceWordsRange().GetStartPos()]
= &hypo->GetTranslationOption().GetInputPath().GetPhrase();
}
}
void
Hypothesis::
OutputInput(std::ostream& os) const
{
void
Hypothesis::
OutputInput(std::ostream& os) const
{
size_t len = this->GetInput().GetSize();
std::vector<const Phrase*> inp_phrases(len, 0);
OutputInput(inp_phrases, this);
for (size_t i=0; i<len; ++i)
if (inp_phrases[i]) os << *inp_phrases[i];
}
}
void
Hypothesis::
OutputBestSurface(std::ostream &out, const std::vector<FactorType> &outputFactorOrder,
void
Hypothesis::
OutputBestSurface(std::ostream &out, const std::vector<FactorType> &outputFactorOrder,
char reportSegmentation, bool reportAllFactors) const
{
if (m_prevHypo)
{ // recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence
{
if (m_prevHypo) {
// recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence
m_prevHypo->OutputBestSurface(out, outputFactorOrder, reportSegmentation, reportAllFactors);
}
OutputSurface(out, *this, outputFactorOrder, reportSegmentation, reportAllFactors);
}
}
//////////////////////////////////////////////////////////////////////////
/***
//////////////////////////////////////////////////////////////////////////
/***
* print surface factor only for the given phrase
*/
void
Hypothesis::
OutputSurface(std::ostream &out, const Hypothesis &edge,
void
Hypothesis::
OutputSurface(std::ostream &out, const Hypothesis &edge,
const std::vector<FactorType> &outputFactorOrder,
char reportSegmentation, bool reportAllFactors) const
{
{
UTIL_THROW_IF2(outputFactorOrder.size() == 0,
"Must specific at least 1 output factor");
const TargetPhrase& phrase = edge.GetCurrTargetPhrase();
@ -614,12 +618,12 @@ namespace Moses
}
out << "| ";
}
}
}
std::map<size_t, const Factor*>
Hypothesis::
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
{
std::map<size_t, const Factor*>
Hypothesis::
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
{
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
const Phrase &inputPhrase = inputPath.GetPhrase();
@ -636,13 +640,13 @@ namespace Moses
}
return ret;
}
}
#ifdef HAVE_XMLRPC_C
void
Hypothesis::
OutputLocalWordAlignment(vector<xmlrpc_c::value>& dest) const
{
void
Hypothesis::
OutputLocalWordAlignment(vector<xmlrpc_c::value>& dest) const
{
using namespace std;
WordsRange const& src = this->GetCurrSourceWordsRange();
WordsRange const& trg = this->GetCurrTargetWordsRange();
@ -651,24 +655,23 @@ namespace Moses
= this->GetCurrTargetPhrase().GetAlignTerm().GetSortedAlignments();
typedef pair<size_t,size_t> item;
map<string, xmlrpc_c::value> M;
BOOST_FOREACH(item const* p, a)
{
BOOST_FOREACH(item const* p, a) {
M["source-word"] = xmlrpc_c::value_int(src.GetStartPos() + p->first);
M["target-word"] = xmlrpc_c::value_int(trg.GetStartPos() + p->second);
dest.push_back(xmlrpc_c::value_struct(M));
}
}
}
void
Hypothesis::
OutputWordAlignment(vector<xmlrpc_c::value>& out) const
{
void
Hypothesis::
OutputWordAlignment(vector<xmlrpc_c::value>& out) const
{
vector<Hypothesis const*> tmp;
for (Hypothesis const* h = this; h; h = h->GetPrevHypo())
tmp.push_back(h);
for (size_t i = tmp.size(); i-- > 0;)
tmp[i]->OutputLocalWordAlignment(out);
}
}
#endif

2
moses/IOWrapper.h
View File

@ -161,7 +161,7 @@ public:
return m_detailTreeFragmentsOutputCollector.get();
}
void SetInputStreamFromString(std::istringstream &input){
void SetInputStreamFromString(std::istringstream &input) {
m_inputStream = &input;
}

63
moses/LM/RDLM.cpp
View File

@ -13,12 +13,14 @@ namespace Moses
typedef Eigen::Map<Eigen::Matrix<int,Eigen::Dynamic,1> > EigenMap;
RDLM::~RDLM() {
RDLM::~RDLM()
{
delete lm_head_base_instance_;
delete lm_label_base_instance_;
}
void RDLM::Load() {
void RDLM::Load()
{
lm_head_base_instance_ = new nplm::neuralTM();
lm_head_base_instance_->read(m_path_head_lm);
@ -202,8 +204,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
// ignore glue rules
if (root->GetLabel() == m_glueSymbol) {
// recursion
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it)
{
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels);
}
return;
@ -260,8 +261,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
}
if (ancestor_labels.size() >= m_context_up && !num_virtual) {
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
}
else {
} else {
boost::hash_combine(boundary_hash, ngram_head_null.back());
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
}
@ -276,8 +276,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
return;
}
rescoring_levels = m_context_up-1;
}
else {
} else {
return;
}
}
@ -305,16 +304,14 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
virtual_head = true;
if (m_binarized == 1 || (m_binarized == 3 && head_label[2] == 'l')) {
reached_end = 1; //indicate that we've seen the first symbol of the RHS
}
else if (m_binarized == 2 || (m_binarized == 3 && head_label[2] == 'r')) {
} else if (m_binarized == 2 || (m_binarized == 3 && head_label[2] == 'r')) {
reached_end = 2; // indicate that we've seen the last symbol of the RHS
}
// with 'full' binarization, direction is encoded in 2nd char
std::string clipped_label = (m_binarized == 3) ? head_label.substr(2,head_label.size()-2) : head_label.substr(1,head_label.size()-1);
label_idx = lm_label->lookup_input_word(clipped_label);
label_idx_out = lm_label->lookup_output_word(clipped_label);
}
else {
} else {
reached_end = 3; // indicate that we've seen first and last symbol of the RHS
label_idx = lm_label->lookup_input_word(head_label);
label_idx_out = lm_label->lookup_output_word(head_label);
@ -341,8 +338,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it);
it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it);
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
}
else {
} else {
boost::hash_combine(boundary_hash, ngram_head_null.back());
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
}
@ -362,8 +358,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it);
it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it);
score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size())));
}
else {
} else {
boost::hash_combine(boundary_hash, ngram_label_null.back());
score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size())));
}
@ -374,8 +369,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (virtual_head) {
num_virtual = m_context_up;
}
else if (num_virtual) {
} else if (num_virtual) {
--num_virtual;
}
@ -456,8 +450,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (reached_end == 2 || reached_end == 3) {
std::fill_n(it, right_padding, static_stop_head);
it += right_padding;
}
else {
} else {
std::copy(static_label_null.begin()+offset_up_head-m_context_right-right_padding, static_label_null.begin()-m_context_right+offset_up_head, it);
}
}
@ -468,8 +461,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (reached_end == 2 || reached_end == 3) {
std::fill_n(it, right_padding, static_stop_label);
it += right_padding;
}
else {
} else {
std::copy(static_label_null.begin()+offset_up_head-right_padding, static_label_null.begin()+offset_up_head, it);
}
}
@ -478,8 +470,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (ancestor_labels.size() >= m_context_up && !num_virtual) {
score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
}
else {
} else {
boost::hash_combine(boundary_hash, ngram.back());
score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
}
@ -492,8 +483,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (ancestor_labels.size() >= m_context_up && !num_virtual) {
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
}
else {
} else {
boost::hash_combine(boundary_hash, ngram.back());
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
}
@ -519,8 +509,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
return;
}
// recursion
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it)
{
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels - 1);
}
ancestor_heads.pop_back();
@ -531,12 +520,10 @@ InternalTree* RDLM::GetHead(InternalTree* root, const TreePointerMap & back_poin
{
InternalTree *tree;
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it)
{
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
if ((*it)->IsLeafNT()) {
tree = back_pointers.find(it->get())->second.get();
}
else {
} else {
tree = it->get();
}
@ -563,8 +550,7 @@ InternalTree* RDLM::GetHead(InternalTree* root, const TreePointerMap & back_poin
for (std::vector<TreePointer>::const_iterator it2 = tree->GetChildren().begin(); it2 != tree->GetChildren().end(); ++it2) {
if ((*it2)->IsLeafNT()) {
tree2 = back_pointers.find(it2->get())->second.get();
}
else {
} else {
tree2 = it2->get();
}
if (tree2->GetLabel() == "PTKVZ" && tree2->GetLength() == 1 && tree2->GetChildren()[0]->IsTerminal()) {
@ -659,8 +645,7 @@ void RDLM::GetIDs(const std::string & head, const std::string & preterminal, std
}
if (m_sharedVocab) {
IDs.second = IDs.first;
}
else {
} else {
IDs.second = lm_head_base_instance_->lookup_output_word(head);
if (m_isPretermBackoff && IDs.second == 0) {
IDs.second = lm_head_base_instance_->lookup_output_word(preterminal);
@ -694,8 +679,7 @@ RDLM::TreePointerMap RDLM::AssociateLeafNTs(InternalTree* root, const std::vecto
found = next_leafNT(it);
if (found) {
ret[it->get()] = *it_prev;
}
else {
} else {
std::cerr << "Warning: leaf nonterminal not found in rule; why did this happen?\n";
}
}
@ -822,8 +806,7 @@ FFState* RDLM::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
}
return new RDLMState(mytree, score[1], score[3], boundary_hash);
}
else {
} else {
UTIL_THROW2("Error: RDLM active, but no internal tree structure found");
}

29
moses/LM/RDLM.h
View File

@ -11,8 +11,9 @@
// Sennrich, Rico (2015). Modelling and Optimizing on Syntactic N-Grams for Statistical Machine Translation. Transactions of the Association for Computational Linguistics.
// see 'scripts/training/rdlm' for training scripts
namespace nplm {
class neuralTM;
namespace nplm
{
class neuralTM;
}
namespace Moses
@ -121,8 +122,7 @@ public:
, m_normalizeLabelLM(false)
, m_sharedVocab(false)
, m_binarized(0)
, m_cacheSize(1000000)
{
, m_cacheSize(1000000) {
ReadParameters();
}
@ -161,7 +161,9 @@ public:
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const {UTIL_THROW(util::Exception, "Not implemented");};
ScoreComponentCollection* accumulator) const {
UTIL_THROW(util::Exception, "Not implemented");
};
FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
@ -185,8 +187,7 @@ public:
UnbinarizedChildren(InternalTree* root, const TreePointerMap & pointers, bool binary):
current(root),
back_pointers(pointers),
binarized(binary)
{
binarized(binary) {
stack.reserve(10);
_end = current->GetChildren().end();
iter = current->GetChildren().begin();
@ -196,8 +197,7 @@ public:
// also go through trees or previous hypotheses to rescore nodes for which more context has become available
if ((*iter)->IsLeafNT()) {
current = back_pointers.find(iter->get())->second.get();
}
else {
} else {
current = iter->get();
}
iter = current->GetChildren().begin();
@ -205,8 +205,12 @@ public:
_begin = iter;
}
std::vector<TreePointer>::const_iterator begin() const { return _begin; }
std::vector<TreePointer>::const_iterator end() const { return _end; }
std::vector<TreePointer>::const_iterator begin() const {
return _begin;
}
std::vector<TreePointer>::const_iterator end() const {
return _end;
}
std::vector<TreePointer>::const_iterator operator++() {
iter++;
@ -230,8 +234,7 @@ public:
// also go through trees or previous hypotheses to rescore nodes for which more context has become available
if ((*iter)->IsLeafNT()) {
current = back_pointers.find(iter->get())->second.get();
}
else {
} else {
current = iter->get();
}
iter = current->GetChildren().begin();

4
moses/Manager.cpp
View File

@ -87,7 +87,9 @@ Manager::~Manager()
const InputType&
Manager::GetSource() const
{ return m_source ; }
{
return m_source ;
}
/**
* Main decoder loop that translates a sentence by expanding

2
moses/OutputCollector.h
View File

@ -110,7 +110,7 @@ private:
#endif
public:
void SetOutputStream(std::ostream* outStream){
void SetOutputStream(std::ostream* outStream) {
m_outStream = outStream;
}

6
moses/Parameter.cpp
View File

@ -368,8 +368,7 @@ AddParam(po::options_description& optgroup,
m_fullname[abbrevName] = paramName;
m_description[paramName] = description;
string optname = paramName;
if (abbrevName.size() == 1)
{
if (abbrevName.size() == 1) {
optname += string(",")+abbrevName;
// m_confusable[abbrevName[0]].insert(paramName);
}
@ -429,8 +428,7 @@ LoadParam(int argc, char* xargv[])
// legacy parameter handling: all parameters are expected
// to start with a single dash
char* argv[argc+1];
for (int i = 0; i < argc; ++i)
{
for (int i = 0; i < argc; ++i) {
argv[i] = xargv[i];
if (strlen(argv[i]) > 2 && argv[i][0] == '-' && argv[i][1] == '-')
++argv[i];

6
moses/ScoreComponentCollection.cpp
View File

@ -195,14 +195,12 @@ void ScoreComponentCollection::Save(ostream& out, bool multiline) const
std::vector<FeatureFunction*> const& all_ff
= FeatureFunction::GetFeatureFunctions();
BOOST_FOREACH(FeatureFunction const* ff, all_ff)
{
BOOST_FOREACH(FeatureFunction const* ff, all_ff) {
string name = ff->GetScoreProducerDescription();
size_t i = ff->GetIndex();
if (ff->GetNumScoreComponents() == 1)
out << name << sep << m_scores[i] << linesep;
else
{
else {
size_t stop = i + ff->GetNumScoreComponents();
boost::format fmt("%s_%d");
for (size_t k = 1; i < stop; ++i, ++k)

2
moses/ScoreComponentCollection.h
View File

@ -234,7 +234,7 @@ public:
const ScoreComponentCollection& scores) {
size_t i = sp->GetIndex();
size_t stop = i + sp->GetNumScoreComponents();
for (;i < stop; ++i) m_scores[i] += scores.m_scores[i];
for (; i < stop; ++i) m_scores[i] += scores.m_scores[i];
}
//! Add scores from a single FeatureFunction only

62
moses/Sentence.cpp
View File

@ -60,23 +60,18 @@ aux_init_partial_translation(string& line)
string sourceCompletedStr;
int loc1 = line.find( "|||", 0 );
int loc2 = line.find( "|||", loc1 + 3 );
if (loc1 > -1 && loc2 > -1)
{
if (loc1 > -1 && loc2 > -1) {
m_initialTargetPhrase = Trim(line.substr(0, loc1));
string scov = Trim(line.substr(loc1 + 3, loc2 - loc1 - 3));
line = line.substr(loc2 + 3);
m_sourceCompleted.resize(scov.size());
int contiguous = 1;
for (size_t i = 0; i < scov.size(); ++i)
{
if (sourceCompletedStr.at(i) == '1')
{
for (size_t i = 0; i < scov.size(); ++i) {
if (sourceCompletedStr.at(i) == '1') {
m_sourceCompleted[i] = true;
if (contiguous) m_frontSpanCoveredLength++;
}
else
{
} else {
m_sourceCompleted[i] = false;
contiguous = 0;
}
@ -94,38 +89,31 @@ aux_interpret_sgml_markup(string& line)
metamap::const_iterator i;
if ((i = meta.find("id")) != meta.end())
this->SetTranslationId(atol(i->second.c_str()));
if ((i = meta.find("docid")) != meta.end())
{
if ((i = meta.find("docid")) != meta.end()) {
this->SetDocumentId(atol(i->second.c_str()));
this->SetUseTopicId(false);
this->SetUseTopicIdAndProb(false);
}
if ((i = meta.find("topic")) != meta.end())
{
if ((i = meta.find("topic")) != meta.end()) {
vector<string> topic_params;
boost::split(topic_params, i->second, boost::is_any_of("\t "));
if (topic_params.size() == 1)
{
if (topic_params.size() == 1) {
this->SetTopicId(atol(topic_params[0].c_str()));
this->SetUseTopicId(true);
this->SetUseTopicIdAndProb(false);
}
else
{
} else {
this->SetTopicIdAndProb(topic_params);
this->SetUseTopicId(false);
this->SetUseTopicIdAndProb(true);
}
}
if ((i = meta.find("weight-setting")) != meta.end())
{
if ((i = meta.find("weight-setting")) != meta.end()) {
this->SetWeightSetting(i->second);
this->SetSpecifiesWeightSetting(true);
StaticData::Instance().SetWeightSetting(i->second);
// oh this is so horrible! Why does this have to be propagated globally?
// --- UG
}
else this->SetSpecifiesWeightSetting(false);
} else this->SetSpecifiesWeightSetting(false);
}
void
@ -135,21 +123,17 @@ aux_interpret_dlt(string& line) // whatever DLT means ... --- UG
using namespace std;
typedef map<string, string> str2str_map;
vector<str2str_map> meta = ProcessAndStripDLT(line);
BOOST_FOREACH(str2str_map const& M, meta)
{
BOOST_FOREACH(str2str_map const& M, meta) {
str2str_map::const_iterator i,j;
if ((i = M.find("type")) != M.end())
{
if ((i = M.find("type")) != M.end()) {
j = M.find("id");
string id = j == M.end() ? "default" : j->second;
if (i->second == "cbtm")
{
if (i->second == "cbtm") {
PhraseDictionaryDynamicCacheBased* cbtm;
cbtm = PhraseDictionaryDynamicCacheBased::InstanceNonConst(id);
if (cbtm) cbtm->ExecuteDlt(M);
}
if (i->second == "cblm")
{
if (i->second == "cblm") {
DynamicCacheBasedLanguageModel* cblm;
cblm = DynamicCacheBasedLanguageModel::InstanceNonConst(id);
if (cblm) cblm->ExecuteDlt(M);
@ -162,13 +146,13 @@ void
Sentence::
aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls,
std::vector<std::pair<size_t, std::string> >& placeholders)
{ // parse XML markup in translation line
{
// parse XML markup in translation line
const StaticData &SD = StaticData::Instance();
using namespace std;
if (SD.GetXmlInputType() != XmlPassThrough)
{
if (SD.GetXmlInputType() != XmlPassThrough) {
int offset = SD.IsSyntax() ? 1 : 0;
bool OK = ProcessAndStripXMLTags(line, m_xmlOptions,
m_reorderingConstraint,
@ -197,8 +181,7 @@ init(string line, std::vector<FactorType> const& factorOrder)
aux_interpret_dlt(line); // some poorly documented cache-based stuff
// if sentences is specified as "<passthrough tag1=""/>"
if (SD.IsPassthroughEnabled() || SD.IsPassthroughInNBestEnabled())
{
if (SD.IsPassthroughEnabled() || SD.IsPassthroughInNBestEnabled()) {
string pthru = PassthroughSGML(line,"passthrough");
this->SetPassthroughInformation(pthru);
}
@ -218,11 +201,9 @@ init(string line, std::vector<FactorType> const& factorOrder)
// our XmlOptions and create TranslationOptions
// only fill the vector if we are parsing XML
if (SD.GetXmlInputType() != XmlPassThrough)
{
if (SD.GetXmlInputType() != XmlPassThrough) {
m_xmlCoverageMap.assign(GetSize(), false);
BOOST_FOREACH(XmlOption* o, m_xmlOptions)
{
BOOST_FOREACH(XmlOption* o, m_xmlOptions) {
WordsRange const& r = o->range;
for(size_t j = r.GetStartPos(); j <= r.GetEndPos(); ++j)
m_xmlCoverageMap[j]=true;
@ -233,8 +214,7 @@ init(string line, std::vector<FactorType> const& factorOrder)
m_reorderingConstraint.InitializeWalls(GetSize());
// set reordering walls, if "-monotone-at-punction" is set
if (SD.UseReorderingConstraint() && GetSize())
{
if (SD.UseReorderingConstraint() && GetSize()) {
WordsRange r(0, GetSize()-1);
m_reorderingConstraint.SetMonotoneAtPunctuation(GetSubString(r));
}

33
moses/Sentence.h
View File

@ -32,22 +32,22 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace Moses
{
class WordsRange;
class PhraseDictionary;
class TranslationOption;
class TranslationOptionCollection;
class ChartTranslationOptions;
class TranslationTask;
struct XmlOption;
class WordsRange;
class PhraseDictionary;
class TranslationOption;
class TranslationOptionCollection;
class ChartTranslationOptions;
class TranslationTask;
struct XmlOption;
/**
/**
* A Phrase class with an ID. Used specifically as source input so contains functionality to read
* from IODevice and create trans opt
*/
class Sentence : public Phrase, public InputType
{
protected:
class Sentence : public Phrase, public InputType
{
protected:
/**
* Utility method that takes in a string representing an XML tag and the name of the attribute,
@ -61,7 +61,7 @@ namespace Moses
void ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders);
public:
public:
Sentence();
Sentence(size_t const transId, std::string const& stext,
std::vector<FactorType> const* IFO = NULL);
@ -106,14 +106,15 @@ namespace Moses
std::string const& phraseString);
const NonTerminalSet&
GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
{ return m_defaultLabelSet; }
GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const {
return m_defaultLabelSet;
}
void
init(std::string line, std::vector<FactorType> const& factorOrder);
private:
private:
// auxliliary functions for Sentence initialization
// void aux_interpret_sgml_markup(std::string& line);
// void aux_interpret_dlt(std::string& line);
@ -134,7 +135,7 @@ namespace Moses
void
aux_init_partial_translation(std::string& line);
};
};
}

8
moses/StaticData.h
View File

@ -476,8 +476,7 @@ public:
// m_searchAlgorithm == SyntaxF2S;
// }
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const
{
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const {
if (algo == DefaultSearchAlgorithm)
algo = m_searchAlgorithm;
return (algo == CYKPlus || algo == ChartIncremental ||
@ -486,8 +485,9 @@ public:
}
const ScoreComponentCollection&
GetAllWeights() const
{ return m_allWeights; }
GetAllWeights() const {
return m_allWeights;
}
void SetAllWeights(const ScoreComponentCollection& weights) {
m_allWeights = weights;

13
moses/TargetPhrase.cpp
View File

@ -232,11 +232,9 @@ mergescores(boost::shared_ptr<Scores> const& a,
if (!b) return a;
if (a->size() != b->size()) return ret;
ret.reset(new Scores(*a));
for (size_t i = 0; i < a->size(); ++i)
{
for (size_t i = 0; i < a->size(); ++i) {
if ((*a)[i] == 0) (*a)[i] = (*b)[i];
else if ((*b)[i])
{
else if ((*b)[i]) {
UTIL_THROW_IF2((*a)[i] != (*b)[i], "can't merge feature vectors");
}
}
@ -253,8 +251,7 @@ Merge(const TargetPhrase &copy, const std::vector<FactorType>& factorVec)
m_fullScore += copy.m_fullScore;
typedef ScoreCache_t::iterator iter;
typedef ScoreCache_t::value_type item;
BOOST_FOREACH(item const& s, copy.m_cached_scores)
{
BOOST_FOREACH(item const& s, copy.m_cached_scores) {
pair<iter,bool> foo = m_cached_scores.insert(s);
if (foo.second == false)
foo.first->second = mergescores(foo.first->second, s.second);
@ -280,7 +277,9 @@ void
TargetPhrase::
SetExtraScores(FeatureFunction const* ff,
boost::shared_ptr<Scores> const& s)
{ m_cached_scores[ff] = s; }
{
m_cached_scores[ff] = s;
}
void TargetPhrase::SetProperties(const StringPiece &str)

4
moses/TargetPhrase.h
View File

@ -51,7 +51,7 @@ class PhraseDictionary;
*/
class TargetPhrase: public Phrase
{
public:
public:
typedef std::map<FeatureFunction const*, boost::shared_ptr<Scores> >
ScoreCache_t;
ScoreCache_t const& GetExtraScores() const;
@ -59,7 +59,7 @@ class TargetPhrase: public Phrase
void SetExtraScores(FeatureFunction const* ff,
boost::shared_ptr<Scores> const& scores);
private:
private:
ScoreCache_t m_cached_scores;
private:

6
moses/TrainingTask.h
View File

@ -26,8 +26,7 @@ public:
// factory function
static boost::shared_ptr<TrainingTask>
create(boost::shared_ptr<InputType> const& source)
{
create(boost::shared_ptr<InputType> const& source) {
boost::shared_ptr<IOWrapper> nix;
boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, nix));
ret->m_self = ret;
@ -37,8 +36,7 @@ public:
// factory function
static boost::shared_ptr<TrainingTask>
create(boost::shared_ptr<InputType> const& source,
boost::shared_ptr<IOWrapper> const& ioWrapper)
{
boost::shared_ptr<IOWrapper> const& ioWrapper) {
boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper));
ret->m_self = ret;
return ret;

6
moses/TranslationModel/PhraseDictionary.h
View File

@ -117,8 +117,7 @@ public:
virtual
TargetPhraseCollection const *
GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src)
{
GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src) {
return GetTargetPhraseCollectionLEGACY(src);
}
@ -129,8 +128,7 @@ public:
virtual
void
GetTargetPhraseCollectionBatch(ttasksptr const& ttask,
const InputPathList &inputPathQueue) const
{
const InputPathList &inputPathQueue) const {
GetTargetPhraseCollectionBatch(inputPathQueue);
}

2
moses/TranslationOption.cpp
View File

@ -109,7 +109,7 @@ ostream& operator<<(ostream& out, const TranslationOption& possibleTranslation)
return out;
}
/** returns cached scores */
/** returns cached scores */
const Scores*
TranslationOption::
GetLexReorderingScores(LexicalReordering const* scoreProducer) const

3
moses/TranslationOptionCollection.cpp
View File

@ -626,8 +626,7 @@ CacheLexReordering()
{
size_t const stop = m_source.GetSize();
typedef StatefulFeatureFunction sfFF;
BOOST_FOREACH(sfFF const* ff, sfFF::GetStatefulFeatureFunctions())
{
BOOST_FOREACH(sfFF const* ff, sfFF::GetStatefulFeatureFunctions()) {
if (typeid(*ff) != typeid(LexicalReordering)) continue;
LexicalReordering const& lr = static_cast<const LexicalReordering&>(*ff);
for (size_t s = 0 ; s < stop ; s++)

3
moses/TranslationOptionCollection.h
View File

@ -177,8 +177,7 @@ public:
return m_inputPathQueue;
}
ttasksptr GetTranslationTask() const
{
ttasksptr GetTranslationTask() const {
return m_ttask.lock();
}
TO_STRING();

22
moses/TranslationTask.cpp
View File

@ -82,33 +82,29 @@ TranslationTask
if (!staticData.IsSyntax(algo))
manager.reset(new Manager(this->self())); // phrase-based
else if (algo == SyntaxF2S || algo == SyntaxT2S)
{ // STSG-based tree-to-string / forest-to-string decoding (ask Phil Williams)
else if (algo == SyntaxF2S || algo == SyntaxT2S) {
// STSG-based tree-to-string / forest-to-string decoding (ask Phil Williams)
typedef Syntax::F2S::RuleMatcherCallback Callback;
typedef Syntax::F2S::RuleMatcherHyperTree<Callback> RuleMatcher;
manager.reset(new Syntax::F2S::Manager<RuleMatcher>(this->self()));
}
else if (algo == SyntaxS2T)
{ // new-style string-to-tree decoding (ask Phil Williams)
else if (algo == SyntaxS2T) {
// new-style string-to-tree decoding (ask Phil Williams)
S2TParsingAlgorithm algorithm = staticData.GetS2TParsingAlgorithm();
if (algorithm == RecursiveCYKPlus)
{
if (algorithm == RecursiveCYKPlus) {
typedef Syntax::S2T::EagerParserCallback Callback;
typedef Syntax::S2T::RecursiveCYKPlusParser<Callback> Parser;
manager.reset(new Syntax::S2T::Manager<Parser>(this->self()));
}
else if (algorithm == Scope3)
{
} else if (algorithm == Scope3) {
typedef Syntax::S2T::StandardParserCallback Callback;
typedef Syntax::S2T::Scope3Parser<Callback> Parser;
manager.reset(new Syntax::S2T::Manager<Parser>(this->self()));
}
else UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
} else UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
}
else if (algo == SyntaxT2S_SCFG)
{ // SCFG-based tree-to-string decoding (ask Phil Williams)
else if (algo == SyntaxT2S_SCFG) {
// SCFG-based tree-to-string decoding (ask Phil Williams)
typedef Syntax::F2S::RuleMatcherCallback Callback;
typedef Syntax::T2S::RuleMatcherSCFG<Callback> RuleMatcher;
manager.reset(new Syntax::T2S::Manager<RuleMatcher>(this->self()));

19
moses/TranslationTask.h
View File

@ -40,7 +40,9 @@ class TranslationTask : public Moses::Task
TranslationTask(TranslationTask const& other) { }
TranslationTask const&
operator=(TranslationTask const& other) { return *this; }
operator=(TranslationTask const& other) {
return *this;
}
protected:
boost::weak_ptr<TranslationTask> m_self; // weak ptr to myself
@ -68,11 +70,15 @@ protected:
public:
boost::shared_ptr<TranslationTask>
self() { return m_self.lock(); }
self() {
return m_self.lock();
}
virtual
boost::shared_ptr<TranslationTask const>
self() const { return m_self.lock(); }
self() const {
return m_self.lock();
}
// creator functions
static boost::shared_ptr<TranslationTask> create();
@ -92,15 +98,16 @@ public:
virtual void Run();
boost::shared_ptr<Moses::InputType>
GetSource() const { return m_source; }
GetSource() const {
return m_source;
}
boost::shared_ptr<BaseManager>
SetupManager(SearchAlgorithm algo = DefaultSearchAlgorithm);
boost::shared_ptr<ContextScope> const&
GetScope() const
{
GetScope() const {
UTIL_THROW_IF2(m_scope == NULL, "No context scope!");
return m_scope;
}

2
moses/TreeInput.h
View File

@ -8,7 +8,7 @@
namespace Moses
{
class TranslationTask;
class TranslationTask;
//! @todo what is this?
class XMLParseOutput
{

66
moses/TypeDef.h
View File

@ -89,18 +89,18 @@ enum FactorDirection {
};
enum DecodeType {
Translate
,Generate
Translate,
Generate
};
namespace LexReorderType
{
enum LexReorderType { // explain values
Backward
,Forward
,Bidirectional
,Fe
,F
Backward,
Forward,
Bidirectional,
Fe,
F
};
}
@ -113,13 +113,13 @@ enum DistortionOrientationOptions {
}
enum InputTypeEnum {
SentenceInput = 0
,ConfusionNetworkInput = 1
,WordLatticeInput = 2
,TreeInputType = 3
//,WordLatticeInput2 = 4
, TabbedSentenceInput = 5
,ForestInputType = 6
SentenceInput = 0,
ConfusionNetworkInput = 1,
WordLatticeInput = 2,
TreeInputType = 3,
//,WordLatticeInput2 = 4,
TabbedSentenceInput = 5,
ForestInputType = 6
};
enum XmlInputType {
@ -131,41 +131,41 @@ enum XmlInputType {
};
enum DictionaryFind {
Best = 0
,All = 1
Best = 0,
All = 1
};
// Note: StaticData uses SearchAlgorithm to determine whether the translation
// model is phrase-based or syntax-based. If you add a syntax-based search
// algorithm here then you should also update StaticData::IsSyntax().
enum SearchAlgorithm {
Normal = 0
,CubePruning = 1
Normal = 0,
CubePruning = 1,
//,CubeGrowing = 2
,CYKPlus = 3
,NormalBatch = 4
,ChartIncremental = 5
,SyntaxS2T = 6
,SyntaxT2S = 7
,SyntaxT2S_SCFG = 8
,SyntaxF2S = 9
,DefaultSearchAlgorithm = 777 // means: use StaticData.m_searchAlgorithm
CYKPlus = 3,
NormalBatch = 4,
ChartIncremental = 5,
SyntaxS2T = 6,
SyntaxT2S = 7,
SyntaxT2S_SCFG = 8,
SyntaxF2S = 9,
DefaultSearchAlgorithm = 777 // means: use StaticData.m_searchAlgorithm
};
enum SourceLabelOverlap {
SourceLabelOverlapAdd = 0
,SourceLabelOverlapReplace = 1
,SourceLabelOverlapDiscard = 2
SourceLabelOverlapAdd = 0,
SourceLabelOverlapReplace = 1,
SourceLabelOverlapDiscard = 2
};
enum WordAlignmentSort {
NoSort = 0
,TargetOrder = 1
NoSort = 0,
TargetOrder = 1
};
enum FormatType {
MosesFormat
,HieroFormat
MosesFormat,
HieroFormat
};
enum S2TParsingAlgorithm {

7
moses/Util.cpp
View File

@ -90,13 +90,6 @@ bool FileExists(const std::string& filePath)
return !ifs.fail();
}
const std::string Trim(const std::string& str, const std::string dropChars)
{
std::string res = str;
res.erase(str.find_last_not_of(dropChars)+1);
return res.erase(0, res.find_first_not_of(dropChars));
}
void ResetUserTime()
{
g_timer.start();

15
moses/Util.h
View File

@ -19,8 +19,7 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef moses_Util_h
#define moses_Util_h
#pragma once
#include <iostream>
#include <fstream>
@ -89,10 +88,17 @@ namespace Moses
#define NTH_ELEMENT4(begin, middle, end, orderer) std::nth_element(begin, middle, end, orderer)
#endif
//! delete white spaces at beginning and end of string
const std::string Trim(const std::string& str, const std::string dropChars = " \t\n\r");
const std::string ToLower(const std::string& str);
//! delete white spaces at beginning and end of string
inline std::string Trim(const std::string& str, const std::string dropChars = " \t\n\r")
{
std::string res = str;
res.erase(str.find_last_not_of(dropChars)+1);
return res.erase(0, res.find_first_not_of(dropChars));
}
//! get string representation of any object/variable, as long as it can pipe to a stream
template<typename T>
inline std::string SPrint(const T &input)
@ -533,4 +539,3 @@ void ShowWeights();
} // namespace
#endif

31
moses/server/Optimizer.cpp
View File

@ -3,36 +3,34 @@
namespace MosesServer
{
using namespace std;
using namespace std;
Optimizer::
Optimizer()
{
Optimizer::
Optimizer()
{
// signature and help strings are documentation -- the client
// can query this information with a system.methodSignature and
// system.methodHelp RPC.
this->_signature = "S:S";
this->_help = "Optimizes multi-model translation model";
}
}
void
Optimizer::
execute(xmlrpc_c::paramList const& paramList,
void
Optimizer::
execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP)
{
{
#ifdef WITH_DLIB
const params_t params = paramList.getStruct(0);
params_t::const_iterator si;
if ((si = params.find("model_name")) == params.end())
{
if ((si = params.find("model_name")) == params.end()) {
string msg = "Missing name of model to be optimized";
msg += " (e.g. PhraseDictionaryMultiModelCounts0)";
throw xmlrpc_c::fault(msg, xmlrpc_c::fault::CODE_PARSE);
}
const string model_name = xmlrpc_c::value_string(si->second);
if ((si = params.find("phrase_pairs")) == params.end())
{
if ((si = params.find("phrase_pairs")) == params.end()) {
throw xmlrpc_c::fault("Missing list of phrase pairs",
xmlrpc_c::fault::CODE_PARSE);
}
@ -42,8 +40,7 @@ namespace MosesServer
xmlrpc_c::value_array pp_array = xmlrpc_c::value_array(si->second);
vector<xmlrpc_c::value> ppValVec(pp_array.vectorValueValue());
for (size_t i = 0; i < ppValVec.size(); ++i)
{
for (size_t i = 0; i < ppValVec.size(); ++i) {
xmlrpc_c::value_array pp_array
= xmlrpc_c::value_array(ppValVec[i]);
vector<xmlrpc_c::value> pp(pp_array.vectorValueValue());
@ -58,7 +55,7 @@ namespace MosesServer
vector<float> weight_vector = pdmm->MinimizePerplexity(phrase_pairs);
vector<xmlrpc_c::value> weight_vector_ret;
for (size_t i=0;i < weight_vector.size();i++)
for (size_t i=0; i < weight_vector.size(); i++)
weight_vector_ret.push_back(xmlrpc_c::value_double(weight_vector[i]));
*retvalP = xmlrpc_c::value_array(weight_vector_ret);
@ -68,5 +65,5 @@ namespace MosesServer
std::cerr << errmsg << std::endl;
*retvalP = xmlrpc_c::value_string(errmsg);
#endif
}
}
}

8
moses/server/Optimizer.h
View File

@ -6,12 +6,12 @@
namespace MosesServer
{
class
class
Optimizer : public xmlrpc_c::method
{
public:
{
public:
Optimizer();
void execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP);
};
};
}

237
moses/server/TranslationRequest.cpp
View File

@ -3,40 +3,40 @@
namespace MosesServer
{
using namespace std;
using Moses::Hypothesis;
using Moses::StaticData;
using Moses::WordsRange;
using Moses::ChartHypothesis;
using Moses::Phrase;
using Moses::Manager;
using Moses::SearchGraphNode;
using Moses::TrellisPathList;
using Moses::TranslationOptionCollection;
using Moses::TranslationOptionList;
using Moses::TranslationOption;
using Moses::TargetPhrase;
using Moses::FValue;
using Moses::PhraseDictionaryMultiModel;
using Moses::FindPhraseDictionary;
using Moses::Sentence;
using namespace std;
using Moses::Hypothesis;
using Moses::StaticData;
using Moses::WordsRange;
using Moses::ChartHypothesis;
using Moses::Phrase;
using Moses::Manager;
using Moses::SearchGraphNode;
using Moses::TrellisPathList;
using Moses::TranslationOptionCollection;
using Moses::TranslationOptionList;
using Moses::TranslationOption;
using Moses::TargetPhrase;
using Moses::FValue;
using Moses::PhraseDictionaryMultiModel;
using Moses::FindPhraseDictionary;
using Moses::Sentence;
boost::shared_ptr<TranslationRequest>
TranslationRequest::
create(xmlrpc_c::paramList const& paramList,
boost::shared_ptr<TranslationRequest>
TranslationRequest::
create(xmlrpc_c::paramList const& paramList,
boost::condition_variable& cond,
boost::mutex& mut)
{
{
boost::shared_ptr<TranslationRequest> ret;
ret.reset(new TranslationRequest(paramList,cond, mut));
ret->m_self = ret;
return ret;
}
}
void
TranslationRequest::
Run()
{
void
TranslationRequest::
Run()
{
parse_request(m_paramList.getStruct(0));
Moses::StaticData const& SD = Moses::StaticData::Instance();
@ -60,13 +60,13 @@ namespace MosesServer
}
m_cond.notify_one();
}
}
/// add phrase alignment information from a Hypothesis
void
TranslationRequest::
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
{
/// add phrase alignment information from a Hypothesis
void
TranslationRequest::
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
{
if (!m_withAlignInfo) return;
WordsRange const& trg = h.GetCurrTargetWordsRange();
WordsRange const& src = h.GetCurrSourceWordsRange();
@ -76,12 +76,12 @@ namespace MosesServer
pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
pAlnInfo["src-end"] = xmlrpc_c::value_int(src.GetEndPos());
aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
}
}
void
TranslationRequest::
outputChartHypo(ostream& out, const ChartHypothesis* hypo)
{
void
TranslationRequest::
outputChartHypo(ostream& out, const ChartHypothesis* hypo)
{
Phrase outPhrase(20);
hypo->GetOutputPhrase(outPhrase);
@ -91,18 +91,20 @@ namespace MosesServer
outPhrase.RemoveWord(outPhrase.GetSize() - 1);
for (size_t pos = 0 ; pos < outPhrase.GetSize() ; pos++)
out << *outPhrase.GetFactor(pos, 0) << " ";
}
}
bool
TranslationRequest::
compareSearchGraphNode(const Moses::SearchGraphNode& a,
bool
TranslationRequest::
compareSearchGraphNode(const Moses::SearchGraphNode& a,
const Moses::SearchGraphNode& b)
{ return a.hypo->GetId() < b.hypo->GetId(); }
{
return a.hypo->GetId() < b.hypo->GetId();
}
void
TranslationRequest::
insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
{
void
TranslationRequest::
insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
{
using xmlrpc_c::value_int;
using xmlrpc_c::value_double;
using xmlrpc_c::value_struct;
@ -111,16 +113,14 @@ namespace MosesServer
vector<SearchGraphNode> searchGraph;
manager.GetSearchGraph(searchGraph);
std::sort(searchGraph.begin(), searchGraph.end());
BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph)
{
BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph) {
map<string, xmlrpc_c::value> x; // search graph xml node
x["forward"] = value_double(n.forward);
x["fscore"] = value_double(n.fscore);
const Hypothesis* hypo = n.hypo;
x["hyp"] = value_int(hypo->GetId());
x["stack"] = value_int(hypo->GetWordsBitmap().GetNumWordsCovered());
if (hypo->GetId() != 0)
{
if (hypo->GetId() != 0) {
const Hypothesis *prevHypo = hypo->GetPrevHypo();
x["back"] = value_int(prevHypo->GetId());
x["score"] = value_double(hypo->GetScore());
@ -134,36 +134,32 @@ namespace MosesServer
searchGraphXml.push_back(value_struct(x));
}
retData["sg"] = xmlrpc_c::value_array(searchGraphXml);
}
}
void
TranslationRequest::
output_phrase(ostream& out, Phrase const& phrase) const
{
if (!m_reportAllFactors)
{
void
TranslationRequest::
output_phrase(ostream& out, Phrase const& phrase) const
{
if (!m_reportAllFactors) {
for (size_t i = 0 ; i < phrase.GetSize(); ++i)
out << *phrase.GetFactor(i, 0) << " ";
}
else out << phrase;
}
} else out << phrase;
}
void
TranslationRequest::
outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
{
void
TranslationRequest::
outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
{
TrellisPathList nBestList;
vector<xmlrpc_c::value> nBestXml;
manager.CalcNBest(m_nbestSize, nBestList, m_nbestDistinct);
BOOST_FOREACH(Moses::TrellisPath const* path, nBestList)
{
BOOST_FOREACH(Moses::TrellisPath const* path, nBestList) {
vector<const Hypothesis *> const& E = path->GetEdges();
if (!E.size()) continue;
std::map<std::string, xmlrpc_c::value> nBestXmlItem;
pack_hypothesis(E, "hyp", nBestXmlItem);
if (m_withScoreBreakdown)
{
if (m_withScoreBreakdown) {
// should the score breakdown be reported in a more structured manner?
ostringstream buf;
path->GetScoreBreakdown()->OutputAllFeatureScores(buf);
@ -175,26 +171,23 @@ namespace MosesServer
nBestXml.push_back(xmlrpc_c::value_struct(nBestXmlItem));
}
retData["nbest"] = xmlrpc_c::value_array(nBestXml);
}
}
void
TranslationRequest::
insertTranslationOptions(Moses::Manager& manager,
void
TranslationRequest::
insertTranslationOptions(Moses::Manager& manager,
std::map<std::string, xmlrpc_c::value>& retData)
{
{
const TranslationOptionCollection* toptsColl
= manager.getSntTranslationOptions();
vector<xmlrpc_c::value> toptsXml;
size_t const stop = toptsColl->GetSource().GetSize();
TranslationOptionList const* tol;
for (size_t s = 0 ; s < stop ; ++s)
{
for (size_t s = 0 ; s < stop ; ++s) {
for (size_t e = s;
(tol = toptsColl->GetTranslationOptionList(s,e)) != NULL;
++e)
{
BOOST_FOREACH(TranslationOption const* topt, *tol)
{
++e) {
BOOST_FOREACH(TranslationOption const* topt, *tol) {
std::map<std::string, xmlrpc_c::value> toptXml;
TargetPhrase const& tp = topt->GetTargetPhrase();
StaticData const& GLOBAL = StaticData::Instance();
@ -215,25 +208,26 @@ namespace MosesServer
}
}
retData["topt"] = xmlrpc_c::value_array(toptsXml);
}
}
bool
check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key)
{
bool
check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key)
{
std::map<std::string, xmlrpc_c::value>::const_iterator m;
return (params.find(key) != params.end());
}
}
TranslationRequest::
TranslationRequest(xmlrpc_c::paramList const& paramList,
TranslationRequest::
TranslationRequest(xmlrpc_c::paramList const& paramList,
boost::condition_variable& cond, boost::mutex& mut)
: m_cond(cond), m_mutex(mut), m_done(false), m_paramList(paramList)
{ }
{ }
void
TranslationRequest::
parse_request(std::map<std::string, xmlrpc_c::value> const& params)
{ // parse XMLRPC request
void
TranslationRequest::
parse_request(std::map<std::string, xmlrpc_c::value> const& params)
{
// parse XMLRPC request
// params_t const params = m_paramList.getStruct(0);
m_paramList.verifyEnd(1); // ??? UG
@ -254,16 +248,14 @@ namespace MosesServer
m_withScoreBreakdown = check(params, "add-score-breakdown");
m_source.reset(new Sentence(0,m_source_string));
si = params.find("lambda");
if (si != params.end())
{
if (si != params.end()) {
// muMo = multiModel
xmlrpc_c::value_array muMoArray = xmlrpc_c::value_array(si->second);
vector<xmlrpc_c::value> muMoValVec(muMoArray.vectorValueValue());
vector<float> w(muMoValVec.size());
for (size_t i = 0; i < muMoValVec.size(); ++i)
w[i] = xmlrpc_c::value_double(muMoValVec[i]);
if (w.size() && (si = params.find("model_name")) != params.end())
{
if (w.size() && (si = params.find("model_name")) != params.end()) {
string const model_name = xmlrpc_c::value_string(si->second);
PhraseDictionaryMultiModel* pdmm
= (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
@ -281,13 +273,13 @@ namespace MosesServer
// for (size_t i = 1; i < tmp.size(); i += 2)
// m_bias[xmlrpc_c::value_int(tmp[i-1])] = xmlrpc_c::value_double(tmp[i]);
// }
} // end of Translationtask::parse_request()
} // end of Translationtask::parse_request()
void
TranslationRequest::
run_chart_decoder()
{
void
TranslationRequest::
run_chart_decoder()
{
Moses::TreeInput tinput;
istringstream buf(m_source_string + "\n");
tinput.Read(buf, StaticData::Instance().GetInputFactorOrder());
@ -302,27 +294,26 @@ namespace MosesServer
m_target_string = out.str();
m_retData["text"] = xmlrpc_c::value_string(m_target_string);
if (m_withGraphInfo)
{
if (m_withGraphInfo) {
std::ostringstream sgstream;
manager.OutputSearchGraphMoses(sgstream);
m_retData["sg"] = xmlrpc_c::value_string(sgstream.str());
}
} // end of TranslationRequest::run_chart_decoder()
} // end of TranslationRequest::run_chart_decoder()
void
TranslationRequest::
pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key,
void
TranslationRequest::
pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key,
map<string, xmlrpc_c::value> & dest) const
{
{
// target string
ostringstream target;
BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
output_phrase(target, e->GetCurrTargetPhrase());
dest[key] = xmlrpc_c::value_string(target.str());
if (m_withAlignInfo)
{ // phrase alignment, if requested
if (m_withAlignInfo) {
// phrase alignment, if requested
vector<xmlrpc_c::value> p_aln;
BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
@ -330,32 +321,32 @@ namespace MosesServer
dest["align"] = xmlrpc_c::value_array(p_aln);
}
if (m_withWordAlignInfo)
{ // word alignment, if requested
if (m_withWordAlignInfo) {
// word alignment, if requested
vector<xmlrpc_c::value> w_aln;
BOOST_FOREACH(Hypothesis const* e, edges)
e->OutputLocalWordAlignment(w_aln);
dest["word-align"] = xmlrpc_c::value_array(w_aln);
}
}
}
void
TranslationRequest::
pack_hypothesis(Hypothesis const* h, string const& key,
void
TranslationRequest::
pack_hypothesis(Hypothesis const* h, string const& key,
map<string, xmlrpc_c::value>& dest) const
{
{
using namespace std;
vector<Hypothesis const*> edges;
for (;h; h = h->GetPrevHypo())
for (; h; h = h->GetPrevHypo())
edges.push_back(h);
pack_hypothesis(edges, key, dest);
}
}
void
TranslationRequest::
run_phrase_decoder()
{
void
TranslationRequest::
run_phrase_decoder()
{
Manager manager(this->self());
// if (m_bias.size()) manager.SetBias(&m_bias);
manager.Decode();
@ -370,5 +361,5 @@ namespace MosesServer
.SetOutputSearchGraph(false);
// WTF? one more reason not to have this as global variable! --- UG
}
}
}

22
moses/server/TranslationRequest.h
View File

@ -23,9 +23,9 @@
#include <xmlrpc-c/base.hpp>
namespace MosesServer
{
class
class
TranslationRequest : public virtual Moses::TranslationTask
{
{
boost::condition_variable& m_cond;
boost::mutex& m_mutex;
bool m_done;
@ -90,12 +90,12 @@ namespace MosesServer
void
insertTranslationOptions(Moses::Manager& manager,
std::map<std::string, xmlrpc_c::value>& retData);
protected:
protected:
TranslationRequest(xmlrpc_c::paramList const& paramList,
boost::condition_variable& cond,
boost::mutex& mut);
public:
public:
static
boost::shared_ptr<TranslationRequest>
@ -105,18 +105,24 @@ namespace MosesServer
virtual bool
DeleteAfterExecution() { return false; }
DeleteAfterExecution() {
return false;
}
bool
IsDone() const { return m_done; }
IsDone() const {
return m_done;
}
std::map<std::string, xmlrpc_c::value> const&
GetRetData() { return m_retData; }
GetRetData() {
return m_retData;
}
void
Run();
};
};
}

22
moses/server/Translator.cpp
View File

@ -4,25 +4,25 @@
namespace MosesServer
{
using namespace std;
using namespace Moses;
using namespace std;
using namespace Moses;
Translator::
Translator(size_t numThreads)
Translator::
Translator(size_t numThreads)
: m_threadPool(numThreads)
{
{
// signature and help strings are documentation -- the client
// can query this information with a system.methodSignature and
// system.methodHelp RPC.
this->_signature = "S:S";
this->_help = "Does translation";
}
}
void
Translator::
execute(xmlrpc_c::paramList const& paramList,
void
Translator::
execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP)
{
{
boost::condition_variable cond;
boost::mutex mut;
boost::shared_ptr<TranslationRequest> task
@ -32,6 +32,6 @@ namespace MosesServer
while (!task->IsDone())
cond.wait(lock);
*retvalP = xmlrpc_c::value_struct(task->GetRetData());
}
}
}

10
moses/server/Translator.h
View File

@ -10,17 +10,17 @@
#endif
namespace MosesServer
{
class
class
// MosesServer::
Translator : public xmlrpc_c::method
{
public:
{
public:
Translator(size_t numThreads = 10);
void execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP);
private:
private:
Moses::ThreadPool m_threadPool;
};
};
}

32
moses/server/Updater.cpp
View File

@ -2,24 +2,24 @@
namespace MosesServer
{
using namespace Moses;
using namespace std;
using namespace Moses;
using namespace std;
Updater::
Updater()
{
Updater::
Updater()
{
// signature and help strings are documentation -- the client
// can query this information with a system.methodSignature and
// system.methodHelp RPC.
this->_signature = "S:S";
this->_help = "Updates stuff";
}
}
void
Updater::
execute(xmlrpc_c::paramList const& paramList,
void
Updater::
execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP)
{
{
#if PT_UG
const params_t params = paramList.getStruct(0);
breakOutParams(params);
@ -28,12 +28,12 @@ namespace MosesServer
XVERBOSE(1,"Done inserting\n");
*retvalP = xmlrpc_c::value_string("Phrase table updated");
#endif
};
};
void
Updater::
breakOutParams(const params_t& params)
{
void
Updater::
breakOutParams(const params_t& params)
{
params_t::const_iterator si = params.find("source");
if(si == params.end())
throw xmlrpc_c::fault("Missing source sentence",
@ -52,6 +52,6 @@ namespace MosesServer
XVERBOSE(1,"alignment = " << m_aln << endl);
m_bounded = ((si = params.find("bounded")) != params.end());
m_add2ORLM = ((si = params.find("updateORLM")) != params.end());
};
};
}

8
moses/server/Updater.h
View File

@ -19,9 +19,9 @@
namespace MosesServer
{
class
class
Updater: public xmlrpc_c::method
{
{
typedef std::map<std::string, xmlrpc_c::value> params_t;
@ -29,7 +29,7 @@ namespace MosesServer
std::string m_src, m_trg, m_aln;
bool m_bounded, m_add2ORLM;
public:
public:
Updater();
void
@ -39,6 +39,6 @@ namespace MosesServer
void
breakOutParams(const params_t& params);
};
};
}

50
moses/thread_safe_container.h
View File

@ -18,20 +18,20 @@
namespace Moses
{
// todo: replace this with thread lock-free containers, if a stable library can
// be found somewhere
// todo: replace this with thread lock-free containers, if a stable library can
// be found somewhere
template<typename KEY, typename VAL, class CONTAINER = std::map<KEY,VAL> >
class
template<typename KEY, typename VAL, class CONTAINER = std::map<KEY,VAL> >
class
ThreadSafeContainer
{
protected:
{
protected:
mutable boost::shared_mutex m_lock;
CONTAINER m_container;
typedef typename CONTAINER::iterator iter_t;
typedef typename CONTAINER::const_iterator const_iter_t;
typedef typename CONTAINER::value_type entry_t;
public:
public:
class locking_iterator
{
@ -49,8 +49,7 @@ namespace Moses
: m_lock(lock), m_container(container), m_iter(iter)
{ }
entry_t const& operator->()
{
entry_t const& operator->() {
UTIL_THROW_IF2(m_container == NULL, "This locking iterator is invalid "
<< "or has not been assigned.");
return m_iter.operator->();
@ -58,21 +57,22 @@ namespace Moses
// locking operators transfer the lock upon assignment and become invalid
locking_iterator const&
operator=(locking_iterator& other)
{
operator=(locking_iterator& other) {
m_lock.swap(other.m_lock);
m_iter = other.m_iter;
other.m_iter = other.m_container.end();
}
bool
operator==(const_iter_t const& other)
{
operator==(const_iter_t const& other) {
return m_iter == other;
}
locking_iterator const&
operator++() { ++m_iter; return *this; }
operator++() {
++m_iter;
return *this;
}
// DO NOT DEFINE THE POST-INCREMENT OPERATOR!
// locking_operators are non-copyable,
@ -82,16 +82,15 @@ namespace Moses
operator++(int);
};
const_iter_t const& end() const
{ return m_container.end(); }
const_iter_t const& end() const {
return m_container.end();
}
locking_iterator begin() const
{
locking_iterator begin() const {
return locking_iterator(m_lock, this, m_container.begin());
}
VAL const& set(KEY const& key, VAL const& val)
{
VAL const& set(KEY const& key, VAL const& val) {
boost::unique_lock< boost::shared_mutex > lock(m_lock);
entry_t entry(key,val);
iter_t foo = m_container.insert(entry).first;
@ -99,27 +98,24 @@ namespace Moses
return foo->second;
}
VAL const* get(KEY const& key, VAL const& default_val)
{
VAL const* get(KEY const& key, VAL const& default_val) {
boost::shared_lock< boost::shared_mutex > lock(m_lock);
entry_t entry(key, default_val);
iter_t foo = m_container.insert(entry).first;
return &(foo->second);
}
VAL const* get(KEY const& key) const
{
VAL const* get(KEY const& key) const {
boost::shared_lock< boost::shared_mutex > lock(m_lock);
const_iter_t m = m_container.find(key);
if (m == m_container.end()) return NULL;
return &m->second;
}
size_t erase(KEY const& key)
{
size_t erase(KEY const& key) {
boost::unique_lock< boost::shared_mutex > lock(m_lock);
return m_container.erase(key);
}
};
};
}
#endif

4
phrase-extract/PropertiesConsolidator.cpp
View File

@ -116,7 +116,7 @@ void PropertiesConsolidator::ProcessPropertiesString(const std::string &properti
} else if ( !keyValue[0].compare("POS") ) {
/* DO NOTHING (property is not registered in the decoder at the moment)
/* DO NOTHING (property is not registered in the decoder at the moment)
if ( m_partsOfSpeechFlag ) {
// POS property: replace strings with vocabulary indices
@ -127,7 +127,7 @@ void PropertiesConsolidator::ProcessPropertiesString(const std::string &properti
} else { // don't process POS property
out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
}
*/
*/
} else {

12
phrase-extract/extract-ghkm/ExtractGHKM.cpp
View File

@ -699,12 +699,12 @@ void ExtractGHKM::WriteGlueGrammar(
// const size_t partOfSpeechSentenceStart = 0;
// const size_t partOfSpeechSentenceEnd = 1;
#ifndef BOS_
#define BOS_ "<s>" //Beginning of sentence symbol
#endif
#ifndef EOS_
#define EOS_ "</s>" //End of sentence symbol
#endif
#ifndef BOS_
#define BOS_ "<s>" //Beginning of sentence symbol
#endif
#ifndef EOS_
#define EOS_ "</s>" //End of sentence symbol
#endif
std::string sentenceStartSource = BOS_;
std::string sentenceEndSource = EOS_;

5
phrase-extract/filter-rule-table/TreeCfgFilter.h
View File

@ -25,8 +25,9 @@ namespace FilterRuleTable
// Filters a rule table, discarding rules that cannot be applied to a given
// test set. The rule table must have a TSG source-side and the test sentences
// must be parse trees.
class TreeCfgFilter : public CfgFilter {
public:
class TreeCfgFilter : public CfgFilter
{
public:
// Initialize the filter for a given set of test sentences.
TreeCfgFilter(const std::vector<boost::shared_ptr<StringTree> > &);

4
phrase-extract/postprocess-egret-forests/Forest.h
View File

@ -15,7 +15,7 @@ namespace PostprocessEgretForests
class Forest
{
public:
public:
struct Vertex;
struct Hyperedge {
@ -35,7 +35,7 @@ class Forest
std::vector<boost::shared_ptr<Vertex> > vertices;
private:
private:
// Copying is not allowed.
Forest(const Forest &);
Forest &operator=(const Forest &);

15
phrase-extract/postprocess-egret-forests/ForestParser.cpp
View File

@ -17,15 +17,18 @@ namespace PostprocessEgretForests
{
ForestParser::ForestParser()
: m_input(0) {
: m_input(0)
{
}
ForestParser::ForestParser(std::istream &input)
: m_input(&input) {
: m_input(&input)
{
++(*this);
}
ForestParser &ForestParser::operator++() {
ForestParser &ForestParser::operator++()
{
if (!m_input) {
return *this;
}
@ -132,12 +135,14 @@ boost::shared_ptr<Forest::Vertex> ForestParser::ParseVertex(
return v;
}
bool operator==(const ForestParser &lhs, const ForestParser &rhs) {
bool operator==(const ForestParser &lhs, const ForestParser &rhs)
{
// TODO Is this right? Compare values of istreams if non-zero?
return lhs.m_input == rhs.m_input;
}
bool operator!=(const ForestParser &lhs, const ForestParser &rhs) {
bool operator!=(const ForestParser &lhs, const ForestParser &rhs)
{
return !(lhs == rhs);
}

15
phrase-extract/postprocess-egret-forests/ForestParser.h
View File

@ -20,8 +20,9 @@ namespace Syntax
namespace PostprocessEgretForests
{
class ForestParser {
public:
class ForestParser
{
public:
struct Entry {
std::size_t sentNum;
std::string sentence;
@ -31,15 +32,19 @@ class ForestParser {
ForestParser();
ForestParser(std::istream &);
Entry &operator*() { return m_entry; }
Entry *operator->() { return &m_entry; }
Entry &operator*() {
return m_entry;
}
Entry *operator->() {
return &m_entry;
}
ForestParser &operator++();
friend bool operator==(const ForestParser &, const ForestParser &);
friend bool operator!=(const ForestParser &, const ForestParser &);
private:
private:
typedef boost::shared_ptr<Forest::Vertex> VertexSP;
typedef boost::shared_ptr<Forest::Hyperedge> HyperedgeSP;

6
phrase-extract/postprocess-egret-forests/ForestWriter.cpp
View File

@ -61,7 +61,8 @@ void ForestWriter::WriteVertex(const Forest::Vertex &v)
}
}
std::string ForestWriter::PossiblyEscape(const std::string &s) const {
std::string ForestWriter::PossiblyEscape(const std::string &s) const
{
if (m_options.escape) {
return Escape(s);
} else {
@ -70,7 +71,8 @@ std::string ForestWriter::PossiblyEscape(const std::string &s) const {
}
// Escapes XML special characters.
std::string ForestWriter::Escape(const std::string &s) const {
std::string ForestWriter::Escape(const std::string &s) const
{
std::string t;
std::size_t len = s.size();
t.reserve(len);

4
phrase-extract/postprocess-egret-forests/ForestWriter.h
View File

@ -15,13 +15,13 @@ namespace PostprocessEgretForests
class ForestWriter
{
public:
public:
ForestWriter(const Options &options, std::ostream &out)
: m_options(options), m_out(out) {}
void Write(const std::string &, const Forest &, std::size_t);
private:
private:
std::string Escape(const std::string &) const;
std::string PossiblyEscape(const std::string &) const;
void WriteHyperedgeLine(const Forest::Hyperedge &);

15
phrase-extract/postprocess-egret-forests/SplitPointFileParser.cpp
View File

@ -16,15 +16,18 @@ namespace PostprocessEgretForests
{
SplitPointFileParser::SplitPointFileParser()
: m_input(0) {
: m_input(0)
{
}
SplitPointFileParser::SplitPointFileParser(std::istream &input)
: m_input(&input) {
: m_input(&input)
{
++(*this);
}
SplitPointFileParser &SplitPointFileParser::operator++() {
SplitPointFileParser &SplitPointFileParser::operator++()
{
if (!m_input) {
return *this;
}
@ -66,13 +69,15 @@ void SplitPointFileParser::ParseLine(const std::string &line,
}
bool operator==(const SplitPointFileParser &lhs,
const SplitPointFileParser &rhs) {
const SplitPointFileParser &rhs)
{
// TODO Is this right? Compare values of istreams if non-zero?
return lhs.m_input == rhs.m_input;
}
bool operator!=(const SplitPointFileParser &lhs,
const SplitPointFileParser &rhs) {
const SplitPointFileParser &rhs)
{
return !(lhs == rhs);
}

15
phrase-extract/postprocess-egret-forests/SplitPointFileParser.h
View File

@ -13,8 +13,9 @@ namespace Syntax
namespace PostprocessEgretForests
{
class SplitPointFileParser {
public:
class SplitPointFileParser
{
public:
struct Entry {
std::vector<SplitPoint> splitPoints;
};
@ -22,8 +23,12 @@ class SplitPointFileParser {
SplitPointFileParser();
SplitPointFileParser(std::istream &);
const Entry &operator*() const { return m_entry; }
const Entry *operator->() const { return &m_entry; }
const Entry &operator*() const {
return m_entry;
}
const Entry *operator->() const {
return &m_entry;
}
SplitPointFileParser &operator++();
@ -33,7 +38,7 @@ class SplitPointFileParser {
friend bool operator!=(const SplitPointFileParser &,
const SplitPointFileParser &);
private:
private:
void ParseLine(const std::string &, std::vector<SplitPoint> &);
Entry m_entry;

4
phrase-extract/postprocess-egret-forests/Symbol.h
View File

@ -30,14 +30,14 @@ inline bool operator==(const Symbol &s, const Symbol &t)
}
struct SymbolHasher {
public:
public:
std::size_t operator()(const Symbol &s) const {
return hash_value(s);
}
};
struct SymbolEqualityPred {
public:
public:
bool operator()(const Symbol &s, const Symbol &t) const {
return s.value == t.value && s.isNonTerminal == t.isNonTerminal;
}

4
phrase-extract/postprocess-egret-forests/TopologicalSorter.h
View File

@ -16,10 +16,10 @@ namespace PostprocessEgretForests
class TopologicalSorter
{
public:
public:
void Sort(const Forest &, std::vector<const Forest::Vertex *> &);
private:
private:
typedef boost::unordered_set<const Forest::Vertex *> VertexSet;
void BuildPredSets(const Forest &);

1
phrase-extract/score-main.cpp
View File

@ -315,7 +315,6 @@ int main(int argc, char* argv[])
// loop through all extracted phrase translations
std::string line, lastLine;
lastLine[0] = '\0';
ExtractionPhrasePair *phrasePair = NULL;
std::vector< ExtractionPhrasePair* > phrasePairsWithSameSource;
std::vector< ExtractionPhrasePair* > phrasePairsWithSameSourceAndTarget; // required for hierarchical rules only, as non-terminal alignments might make the phrases incompatible

48
scripts/ems/experiment.meta
View File

@ -233,6 +233,8 @@ train
template: $lm-training -order $order $settings -text IN -lm OUT
error: cannot execute binary file
error: unrecognised option
not-error: BadDiscountException
not-error: To override this error
randomize
in: lm
out: rlm
@ -309,8 +311,14 @@ split-tuning
default-name: lm/interpolate-tuning.split
pass-unless: output-splitter
template: $output-splitter -model IN1.$output-extension < IN > OUT
strip-tuning
in: split-tuning
out: stripped-tuning
default-name: lm/interpolate-tuning.stripped
pass-unless: mock-output-parser-lm
template: $moses-script-dir/training/strip-xml.perl < IN > OUT && $moses-script-dir/training/wrappers/mosesxml2brackets.py < IN > OUT.trees
interpolate
in: script split-tuning LM:lm
in: script stripped-tuning LM:lm
rerun-on-change: srilm-dir group weights
out: lm
default-name: lm/interpolated-lm
@ -466,14 +474,32 @@ fast-align
in: prepared-data-fast-align
out: fast-alignment
rerun-on-change: fast-align-settings
ignore-if: fast-align-max-lines
template: $external-bin-dir/fast_align -i IN $fast-align-settings > OUT
default-name: fast-align
fast-align-inverse
in: prepared-data-fast-align
out: fast-alignment-inverse
rerun-on-change: fast-align-settings
ignore-if: fast-align-max-lines
template: $external-bin-dir/fast_align -i IN -r $fast-align-settings > OUT
default-name: fast-align-inverse
fast-align-in-parts
in: prepared-data-fast-align
out: fast-alignment
rerun-on-change: fast-align-settings fast-align-max-lines
ignore-unless: fast-align-max-lines
tmp-name: training/tmp.fast-align
template: $moses-script-dir/ems/support/fast-align-in-parts.perl -bin $external-bin-dir/fast_align -i IN -max-lines $fast-align-max-lines -tmp TMP -settings '$fast-align-settings' > OUT
default-name: fast-align
fast-align-in-parts-inverse
in: prepared-data-fast-align
out: fast-alignment-inverse
rerun-on-change: fast-align-settings fast-align-max-lines
ignore-unless: fast-align-max-lines
tmp-name: training/tmp.fast-align-inverse
template: $moses-script-dir/ems/support/fast-align-in-parts.perl -bin $external-bin-dir/fast_align -i IN -r -max-lines $fast-align-max-lines -tmp TMP -settings '$fast-align-settings' > OUT
default-name: fast-align
symmetrize-fast-align
in: fast-alignment fast-alignment-inverse corpus-mml-prefilter=OR=corpus
out: word-alignment
@ -1324,6 +1350,24 @@ multi-bleu-c
rerun-on-change: multi-bleu-c
template: $multi-bleu-c IN1 < IN > OUT
final-model: yes
multi-bleu-detok
in: detokenized-output tokenized-reference
out: multi-bleu-detok-score
default-name: evaluation/multi-bleu-detok
ignore-unless: multi-bleu-detok
rerun-on-change: multi-bleu-detok
template: $multi-bleu-detok IN1 < IN > OUT
final-model: yes
multi-bleu-c-detok
in: detokenized-output tokenized-reference
out: multi-bleu-c-detok-score
default-name: evaluation/multi-bleu-c-detok
ignore-unless: multi-bleu-c-detok
rerun-on-change: multi-bleu-c-detok
template: $multi-bleu-c-detok IN1 < IN > OUT
final-model: yes
ter
in: wrapped-output reference-sgm
out: ter-score
@ -1371,6 +1415,6 @@ analysis-precision
[REPORTING] single
report
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:multi-bleu-detok-score EVALUATION:multi-bleu-c-detok-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis
out: report
default-name: evaluation/report

56
scripts/ems/experiment.perl
View File

@ -312,10 +312,10 @@ sub read_meta {
$ONLY_FACTOR_0{"$module:$step"}++;
}
elsif ($1 eq "error") {
@{$ERROR{"$module:$step"}} = split(/,/,$2);
push @{$ERROR{"$module:$step"}}, $2;
}
elsif ($1 eq "not-error") {
@{$NOT_ERROR{"$module:$step"}} = split(/,/,$2);
push @{$NOT_ERROR{"$module:$step"}}, $2;
}
else {
die("META ERROR unknown parameter: $1");
@ -1282,10 +1282,10 @@ sub execute_steps {
&write_info($i);
# cluster job submission
if ($CLUSTER && ! &is_qsub_script($i)) {
if ($CLUSTER && (!&is_qsub_script($i) || (&backoff_and_get($DO_STEP[$i].":jobs") && (&backoff_and_get($DO_STEP[$i].":jobs")==1)))) {
$DO{$i}++;
my $qsub_args = &get_qsub_args($DO_STEP[$i]);
print "\texecuting $step via qsub ($active active)\n";
print "\texecuting $step via qsub $qsub_args ($active active)\n";
my $qsub_command="qsub $qsub_args -S /bin/bash -e $step.STDERR -o $step.STDOUT $step";
print "\t$qsub_command\n" if $VERBOSE;
`$qsub_command`;
@ -1338,15 +1338,15 @@ sub execute_steps {
sub get_qsub_args {
my ($step) = @_;
my $qsub_args = &get("$step:qsub-settings");
$qsub_args = &get("GENERAL:qsub-settings") unless defined($qsub_args);
my $qsub_args = &backoff_and_get("$step:qsub-settings");
$qsub_args = "" unless defined($qsub_args);
my $memory = &get("$step:qsub-memory");
$qsub_args .= " -pe memory $memory" if defined($memory);
my $hours = &get("$step:qsub-hours");
$qsub_args .= " -l h_rt=$hours:0:0" if defined($hours);
my $project = &backoff_and_get("$step:qsub-project");
$qsub_args = "-P $project" if defined($project);
$qsub_args .= " -P $project" if defined($project);
$qsub_args =~ s/^ //;
print "qsub args: $qsub_args\n" if $VERBOSE;
return $qsub_args;
}
@ -1880,7 +1880,7 @@ sub define_tuning_tune {
my $decoder_settings = &backoff_and_get("TUNING:decoder-settings");
$decoder_settings = "" unless $decoder_settings;
$decoder_settings .= " -v 0 " unless $CLUSTER && $jobs;
$decoder_settings .= " -v 0 " unless $CLUSTER && $jobs && $jobs>1;
my $tuning_settings = &backoff_and_get("TUNING:tuning-settings");
$tuning_settings = "" unless $tuning_settings;
@ -1891,9 +1891,9 @@ sub define_tuning_tune {
$cmd .= " --skip-decoder" if $skip_decoder;
$cmd .= " --inputtype $tune_inputtype" if defined($tune_inputtype);
my $qsub_args = &get_qsub_args("TUNING");
my $qsub_args = &get_qsub_args($DO_STEP[$step_id]);
$cmd .= " --queue-flags=\"$qsub_args\"" if ($CLUSTER && $qsub_args);
$cmd .= " --jobs $jobs" if $CLUSTER && $jobs;
$cmd .= " --jobs $jobs" if $CLUSTER && $jobs && $jobs>1;
my $tuning_dir = $tuned_config;
$tuning_dir =~ s/\/[^\/]+$//;
$cmd .= "\nmkdir -p $tuning_dir";
@ -2576,6 +2576,7 @@ sub define_training_create_config {
my $set = shift @LM_SETS;
next if defined($INTERPOLATED_AWAY{$set});
my $order = &check_backoff_and_get("LM:$set:order");
my $lm_file = "$lm";
my $type = 0; # default: SRILM
@ -2591,6 +2592,13 @@ sub define_training_create_config {
# manually set type
$type = &backoff_and_get("LM:$set:type") if (&backoff_and_get("LM:$set:type"));
# binarized by INTERPOLATED-LM
if (&get("INTERPOLATED-LM:lm-binarizer")) {
$lm_file =~ s/\.lm/\.binlm/;
$type = 1;
$type = &get("INTERPOLATED-LM:type") if &get("INTERPOLATED-LM:type");
}
# which factor is the model trained on?
my $factor = 0;
if (&backoff_and_get("TRAINING:output-factors") &&
@ -2696,7 +2704,7 @@ sub define_interpolated_lm_interpolate {
sub define_interpolated_lm_process {
my ($step_id) = @_;
my ($processed_lm, $interpolatd_lm) = &get_output_and_input($step_id);
my ($processed_lm, $interpolated_lm) = &get_output_and_input($step_id);
my ($module,$set,$stepname) = &deconstruct_name($DO_STEP[$step_id]);
my $tool = &check_backoff_and_get("INTERPOLATED-LM:lm-${stepname}r");
my $FACTOR = &backoff_and_get_array("TRAINING:output-factors");
@ -2706,11 +2714,23 @@ sub define_interpolated_lm_process {
my $cmd = "";
foreach my $factor (keys %{$ILM_SETS}) {
foreach my $order (keys %{$$ILM_SETS{$factor}}) {
next unless scalar(@{$$ILM_SETS{$factor}{$order}}) > 1;
my ($name,$name_processed);
if (scalar(@{$$ILM_SETS{$factor}{$order}}) == 1) {
# not interpolated -> get name from LM version of these steps
my($id,$set) = split(/ /,$$ILM_SETS{$factor}{$order}[0]);
$name = &get_default_file("LM",$set,"train"); # well... works for now;
$name_processed = $STEP_OUTNAME{"LM:$stepname"};
$name_processed =~ s/^(.+\/)([^\/]+)$/$1$set.$2/;
$name_processed = &versionize(&long_file_name($name_processed,"lm",""));
}
else {
my $suffix = "";
$suffix = ".$$FACTOR[$factor]" if $icount > 1 && defined($FACTOR);
$suffix .= ".order$order" if $icount > 1;
$cmd .= "$tool $interpolatd_lm$suffix $processed_lm$suffix\n";
$name = "$interpolated_lm$suffix";
$name_processed = "$processed_lm$suffix";
}
$cmd .= "$tool $name $name_processed\n";
}
}
@ -3072,7 +3092,7 @@ sub define_evaluation_decode {
my $nbest_size;
$nbest_size = $nbest if $nbest;
$nbest_size =~ s/[^\d]//g if $nbest;
if ($jobs && $CLUSTER) {
if ($jobs && $jobs>1 && $CLUSTER) {
$cmd .= "mkdir -p $dir/evaluation/tmp.$set.$VERSION\n";
$cmd .= "cd $dir/evaluation/tmp.$set.$VERSION\n";
if (defined $moses_parallel) {
@ -3496,9 +3516,15 @@ sub check_backoff_and_get_array {
return $CONFIG{$parameter} if defined($CONFIG{$parameter});
# remove set -> find setting for module
$parameter =~ s/:.*:/:/;
$parameter =~ s/:[^:]+:/:/;
return $CONFIG{$parameter} if defined($CONFIG{$parameter});
# remove step (if exists)
if ($parameter =~ /:[^:]+:/) {
$parameter =~ s/:[^:]+:/:/;
return $CONFIG{$parameter} if defined($CONFIG{$parameter});
}
# remove model -> find global setting
$parameter =~ s/^[^:]+:/GENERAL:/;
return $CONFIG{$parameter} if defined($CONFIG{$parameter});

16
scripts/ems/support/build-sparse-features.perl
View File

@ -12,15 +12,17 @@ use strict;
my ($corpus,$input_extension,$output_extension,$outfile_prefix,$specification) = @ARGV;
my $ini = "[feature]\n";
my %ALREADY;
my %ID;
foreach my $feature_spec (split(/,\s*/,$specification)) {
my @SPEC = split(/\s+/,$feature_spec);
my $factor = ($SPEC[0] eq 'word-translation') ? "0-0" : "0";
$factor = $1 if $feature_spec =~ / factor ([\d\-]+)/;
$feature_spec =~ s/ factor ([\d\-]+)//;
if ($SPEC[0] eq 'target-word-insertion') {
$ini .= "TargetWordInsertionFeature name=TWI factor=$factor";
$ini .= "TargetWordInsertionFeature name=TWI".&get_id($SPEC[0])." factor=$factor";
if ($SPEC[1] eq 'top' && $SPEC[2] =~ /^\d+$/) {
my $file = &create_top_words($output_extension, $SPEC[2]);
@ -34,7 +36,7 @@ foreach my $feature_spec (split(/,\s*/,$specification)) {
$ini .= "\n";
}
elsif ($SPEC[0] eq 'source-word-deletion') {
$ini .= "SourceWordDeletionFeature name=SWD factor=$factor";
$ini .= "SourceWordDeletionFeature name=SWD".&get_id($SPEC[0])." factor=$factor";
if ($SPEC[1] eq 'top' && $SPEC[2] =~ /^\d+$/) {
my $file = &create_top_words($input_extension, $SPEC[2]);
$ini .= " path=$file";
@ -60,7 +62,7 @@ foreach my $feature_spec (split(/,\s*/,$specification)) {
die("ERROR: Unknown parameter specification in '$SPEC[1]'\n");
}
my ($input_factor,$output_factor) = split(/\-/,$factor);
$ini .= "WordTranslationFeature name=WT input-factor=$input_factor output-factor=$output_factor simple=1 source-context=0 target-context=0$extra_ini\n";
$ini .= "WordTranslationFeature name=WT".&get_id($SPEC[0])." input-factor=$input_factor output-factor=$output_factor simple=1 source-context=0 target-context=0$extra_ini\n";
}
elsif ($SPEC[0] eq 'phrase-length') {
$ini .= "PhraseLengthFeature name=PL\n";
@ -111,3 +113,11 @@ sub create_top_words {
return $file;
}
sub get_id {
my ($name) = @_;
$ID{$name}++;
return "" if $ID{$name} == 1;
return $ID{$name};
}

91
scripts/ems/support/fast-align-in-parts.perl Executable file
View File

@ -0,0 +1,91 @@
#!/usr/bin/env perl
#######################
# Revision history
#
# 28 Apr 2015 first version
use warnings;
use strict;
use Getopt::Long qw(:config pass_through no_ignore_case permute);
my ($BIN,$IN,$MAX_LINES,$SETTINGS,$REVERSE,$TMP);
GetOptions('bin=s' => \$BIN,
'i=s' => \$IN,
'max-lines=i' => \$MAX_LINES,
'settings=s' => \$SETTINGS,
'r' => \$REVERSE,
'tmp=s' => \$TMP,
) or exit(1);
die("ERROR - usage: fast-align-in-parts.perl -bin FAST_ALIGN_BIN -i PARALLEL_CORPUS -max-lines COUNT -settings CONFIG [-r] -tmp TMPDIR")
unless defined($BIN) && defined($IN) && defined($SETTINGS) && defined($TMP) && defined($MAX_LINES)
&& $MAX_LINES > 0;
die("ERROR - input file does not exist: $IN") unless -e $IN;
die("ERROR - fast_align binary does not exist: $BIN") unless -e $BIN;
chomp(my $line_count = `cat $IN | wc -l`);
# not more than maximal number of lines -> just run it regulary
if ($MAX_LINES > $line_count) {
my $cmd = "$BIN -i $IN $SETTINGS";
$cmd .= " -r" if defined($REVERSE);
safesystem($cmd) or die;
exit(0);
}
my $cmd = "mkdir -p $TMP";
safesystem($cmd) or die;
# split input
$cmd = "split -a 2 -l $MAX_LINES $IN $TMP/prepared-";
safesystem($cmd) or die;
# process
my @INPUT_FILES = `ls $TMP/prepared-*`;
chop(@INPUT_FILES);
foreach my $input_file (@INPUT_FILES) {
# create output file name
die("ERROR") unless $input_file =~ /prepared-(..)$/;
my $output_file = "$TMP/aligned-$1";
# process part
my $cmd = "$BIN -i $input_file $SETTINGS";
$cmd .= " -r" if defined($REVERSE);
$cmd .= " >$output_file";
safesystem($cmd) or die;
die("ERROR: no output produced from command $cmd") unless -e $output_file;
# check line count
chomp(my $input_line_count = `cat $input_file | wc -l`);
chomp(my $output_line_count = `cat $output_file | wc -l`);
die("ERROR: mismatched number of lines in part $1\n\t$input_line_count\t$input_file\n\t$output_line_count\t$output_file\n") unless $input_line_count == $output_line_count;
}
# join output
$cmd = "cat $TMP/aligned-*";
safesystem($cmd) or die;
$cmd = "rm -r $TMP/* ; rmdir $TMP";
safesystem($cmd);
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit 1;
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}

2
scripts/ems/support/generic-parallelizer.perl
View File

@ -4,7 +4,7 @@ use warnings;
use strict;
my $jobs = 20;
my ($infile,$outfile,$cmd,$tmpdir);
my ($infile,$outfile,$cmd,$tmpdir,$qflags);
use Getopt::Long qw(:config pass_through no_ignore_case);
GetOptions('jobs=i' => \$jobs,

10
scripts/ems/support/lmplz-wrapper.perl
View File

@ -7,11 +7,12 @@ use Getopt::Long "GetOptions";
Getopt::Long::config("no_auto_abbrev");
Getopt::Long::config("pass_through");
my ($TEXT,$ORDER,$BIN,$LM);
my ($TEXT,$ORDER,$BIN,$LM,$MEMORY,$TMPDIR);
&GetOptions('text=s' => \$TEXT,
'lm=s' => \$LM,
'S=s' => \$MEMORY,
'T=s' => \$TMPDIR,
'bin=s' => \$BIN,
'order=i' => \$ORDER);
@ -19,8 +20,9 @@ die("ERROR: specify at least --bin BIN --text CORPUS --lm LM and --order N!")
unless defined($BIN) && defined($TEXT) && defined($LM) && defined($ORDER);
my $settings = join(' ', @ARGV);
#print STDERR "settngs=$settings \n";
my $cmd = "$BIN --text $TEXT --order $ORDER --arpa $LM $settings";
$cmd .= " -T $TMPDIR" if defined($TMPDIR);
$cmd .= " -S $MEMORY" if defined($MEMORY);
$cmd .= " " . join(' ', @ARGV) if scalar(@ARGV); # Pass remaining args through.
print "exec: $cmd\n";
`$cmd`;

35
scripts/generic/extract-parallel.perl
View File

@ -6,6 +6,7 @@
use warnings;
use strict;
use File::Basename;
use Cwd 'abs_path';
sub RunFork($);
sub systemCheck($);
@ -109,20 +110,20 @@ else
{
my $numStr = NumStr(0);
$cmd = "ln -s $target $TMPDIR/target.$numStr";
$cmd = "ln -s ".abs_path($target)." $TMPDIR/target.$numStr";
print STDERR "Executing: $cmd \n";
`$cmd`;
$cmd = "ln -s $source $TMPDIR/source.$numStr";
$cmd = "ln -s ".abs_path($source)." $TMPDIR/source.$numStr";
print STDERR "Executing: $cmd \n";
`$cmd`;
$cmd = "ln -s $align $TMPDIR/align.$numStr";
$cmd = "ln -s ".abs_path($align)." $TMPDIR/align.$numStr";
print STDERR "Executing: $cmd \n";
`$cmd`;
if ($weights) {
$cmd = "ln -s $weights $TMPDIR/weights.$numStr";
$cmd = "ln -s ".abs_path($weights)." $TMPDIR/weights.$numStr";
print STDERR "Executing: $cmd \n";
`$cmd`;
}
@ -149,9 +150,8 @@ for (my $i = 0; $i < $numParallel; ++$i)
print "glueArg=$glueArg \n";
my $cmd = "$extractCmd $TMPDIR/target.$numStr $TMPDIR/source.$numStr $TMPDIR/align.$numStr $TMPDIR/extract.$numStr $glueArg $otherExtractArgs $weightsCmd --SentenceOffset ".($i*$linesPerSplit)." 2>> /dev/stderr \n";
print STDERR $cmd;
`$cmd`;
safesystem($cmd) or die;
exit();
}
else
@ -163,6 +163,10 @@ for (my $i = 0; $i < $numParallel; ++$i)
# wait for everything is finished
foreach (@children) {
waitpid($_, 0);
if($? != 0) {
print STDERR "ERROR: Failed to execute: @_\n $!\n";
exit(1);
}
}
# merge
@ -325,3 +329,22 @@ sub NumStr($)
return $numStr;
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "ERROR: Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}

38
scripts/training/mert-moses.pl
View File

@ -76,7 +76,7 @@ my $___N_BEST_LIST_SIZE = 100;
my $___LATTICE_SAMPLES = 0;
my $queue_flags = "-hard"; # extra parameters for parallelizer
# the -l ws0ssmt was relevant only to JHU 2006 workshop
my $___JOBS = undef; # if parallel, number of jobs to use (undef or 0 -> serial)
my $___JOBS = undef; # if parallel, number of jobs to use (undef or <= 0 -> serial)
my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder
my $continue = 0; # should we try to continue from the last saved step?
my $skip_decoder = 0; # and should we skip the first decoder run (assuming we got interrupted during mert)
@ -544,7 +544,7 @@ if ($__PROMIX_TRAINING) {
my $___FILTER_F = $___DEV_F;
$___FILTER_F = $filterfile if (defined $filterfile);
my $cmd = "$filtercmd ./$filtered_path $filtered_config $___FILTER_F";
&submit_or_exec($cmd, "filterphrases_$i.out", "filterphrases_$i.err");
&submit_or_exec($cmd, "filterphrases_$i.out", "filterphrases_$i.err", 1);
push (@_PROMIX_TABLES_BIN,"$filtered_path/phrase-table.0-0.1.1");
}
}
@ -559,7 +559,7 @@ if ($___FILTER_PHRASE_TABLE) {
my $___FILTER_F = $___DEV_F;
$___FILTER_F = $filterfile if (defined $filterfile);
my $cmd = "$filtercmd ./$outdir $___CONFIG $___FILTER_F";
&submit_or_exec($cmd, "filterphrases.out", "filterphrases.err");
&submit_or_exec($cmd, "filterphrases.out", "filterphrases.err", 1);
}
# make a backup copy of startup ini filepath
@ -829,7 +829,7 @@ while (1) {
# remove segmentation
$cmd .= " -l $__REMOVE_SEGMENTATION" if $__PROMIX_TRAINING;
$cmd = &create_extractor_script($cmd, $___WORKING_DIR);
&submit_or_exec($cmd, "extract.out","extract.err");
&submit_or_exec($cmd, "extract.out","extract.err", 1);
}
# Create the initial weights file for mert: init.opt
@ -919,11 +919,11 @@ while (1) {
my $pro_optimizer_cmd = "$pro_optimizer $megam_default_options run$run.pro.data";
if ($___PAIRWISE_RANKED_OPTIMIZER) { # pro optimization
$cmd = "$mert_pro_cmd $proargs $seed_settings $pro_file_settings -o run$run.pro.data ; echo 'not used' > $weights_out_file; $pro_optimizer_cmd";
&submit_or_exec($cmd, $mert_outfile, $mert_logfile);
&submit_or_exec($cmd, $mert_outfile, $mert_logfile, 1);
} elsif ($___PRO_STARTING_POINT) { # First, run pro, then mert
# run pro...
my $pro_cmd = "$mert_pro_cmd $proargs $seed_settings $pro_file_settings -o run$run.pro.data ; $pro_optimizer_cmd";
&submit_or_exec($pro_cmd, "run$run.pro.out", "run$run.pro.err");
&submit_or_exec($pro_cmd, "run$run.pro.out", "run$run.pro.err", 1);
# ... get results ...
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.pro.out","run$run.pro.err",scalar @{$featlist->{"names"}},\%sparse_weights, \@promix_weights);
# Get the pro outputs ready for mert. Add the weight ranges,
@ -951,11 +951,11 @@ while (1) {
# ... and run mert
$cmd =~ s/(--ifile \S+)/$1,run$run.init.pro/;
&submit_or_exec($cmd . $mert_settings, $mert_outfile, $mert_logfile);
&submit_or_exec($cmd . $mert_settings, $mert_outfile, $mert_logfile, ($__THREADS ? $__THREADS : 1) );
} elsif ($___BATCH_MIRA) { # batch MIRA optimization
safesystem("echo 'not used' > $weights_out_file") or die;
$cmd = "$mert_mira_cmd $mira_settings $seed_settings $pro_file_settings -o $mert_outfile";
&submit_or_exec($cmd, "run$run.mira.out", $mert_logfile);
&submit_or_exec($cmd, "run$run.mira.out", $mert_logfile, 1);
} elsif ($___HG_MIRA) {
safesystem("echo 'not used' > $weights_out_file") or die;
$mira_settings .= " --type hypergraph ";
@ -963,7 +963,7 @@ while (1) {
$mira_settings .= " --hgdir $hypergraph_dir ";
#$mira_settings .= "--verbose ";
$cmd = "$mert_mira_cmd $mira_settings $seed_settings -o $mert_outfile";
&submit_or_exec($cmd, "run$run.mira.out", $mert_logfile);
&submit_or_exec($cmd, "run$run.mira.out", $mert_logfile, 1);
} elsif ($__PROMIX_TRAINING) {
# PRO trained mixture model
safesystem("echo 'not used' > $weights_out_file") or die;
@ -972,10 +972,10 @@ while (1) {
$cmd .= join(" ", map {"-p $_"} @_PROMIX_TABLES_BIN);
$cmd .= " -i $___DEV_F";
print "Starting promix optimisation at " . `date`;
&submit_or_exec($cmd, "$mert_outfile", $mert_logfile);
&submit_or_exec($cmd, "$mert_outfile", $mert_logfile, 1);
print "Finished promix optimisation at " . `date`;
} else { # just mert
&submit_or_exec($cmd . $mert_settings, $mert_outfile, $mert_logfile);
&submit_or_exec($cmd . $mert_settings, $mert_outfile, $mert_logfile, ($__THREADS ? $__THREADS : 1) );
}
die "Optimization failed, file $weights_out_file does not exist or is empty"
@ -1283,7 +1283,7 @@ sub run_decoder {
$lsamp_cmd = " -lattice-samples $lsamp_filename $___LATTICE_SAMPLES ";
}
if (defined $___JOBS && $___JOBS > 0) {
if (defined $___JOBS && $___JOBS > 1) {
die "Hypergraph mira not supported by moses-parallel" if $___HG_MIRA;
$decoder_cmd = "$moses_parallel_cmd $pass_old_sge -config $___CONFIG";
$decoder_cmd .= " -inputtype $___INPUTTYPE" if defined($___INPUTTYPE);
@ -1378,9 +1378,9 @@ sub get_featlist_from_moses {
print STDERR "Asking moses for feature names and values from $___CONFIG\n";
my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn";
$cmd .= " -inputtype $___INPUTTYPE" if defined($___INPUTTYPE);
$cmd .= " -show-weights > $featlistfn";
$cmd .= " -show-weights";
print STDERR "Executing: $cmd\n";
safesystem($cmd) or die "Failed to run moses with the config $configfn";
&submit_or_exec($cmd, $featlistfn, "/dev/null", 1);
}
return get_featlist_from_file($featlistfn);
}
@ -1706,10 +1706,14 @@ sub ensure_full_path {
}
sub submit_or_exec {
my ($cmd, $stdout, $stderr) = @_;
my ($cmd, $stdout, $stderr, $threads) = @_;
print STDERR "exec: $cmd\n";
if (defined $___JOBS && $___JOBS > 0) {
safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameter=\"$queue_flags\" -stdout=$stdout -stderr=$stderr" )
if (defined $___JOBS && $___JOBS > 1) {
# request fewer CPU slots, if not needed
my $queue_flags_for_this_command = $queue_flags;
$threads = 1 unless defined($threads);
$queue_flags_for_this_command =~ s/(\-pe smp) \d+/$1 $threads/;
safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameter=\"$queue_flags_for_this_command\" -stdout=$stdout -stderr=$stderr" )
or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)";
} else {
safesystem("$cmd > $stdout 2> $stderr") or die "ERROR: Failed to run '$cmd'.";

110
scripts/training/wrappers/madamira-tok.perl Executable file
View File

@ -0,0 +1,110 @@
#!/usr/bin/env perl
use warnings;
use strict;
use File::Temp qw/tempfile/;
use Getopt::Long "GetOptions";
use File::Basename;
use FindBin qw($RealBin);
use Cwd 'abs_path';
sub GetFactors;
my $TMPDIR = "tmp";
my $KEEP_TMP = 0;
my $MADA_DIR;
my $CONFIG;
my $SCHEME;
my $USE_PARALLEL = 1;
my $FACTORS_STR;
my @FACTORS;
GetOptions(
"tmpdir=s" => \$TMPDIR,
"keep-tmp" => \$KEEP_TMP,
"mada-dir=s" => \$MADA_DIR,
"factors=s" => \$FACTORS_STR,
"config=s" => \$CONFIG,
"scheme=s" => \$SCHEME,
"use-parallel=i" => \$USE_PARALLEL
) or die("ERROR: unknown options");
die("must have -scheme arg") unless defined($SCHEME);
if (!defined($CONFIG)) {
$CONFIG = "$MADA_DIR/samples/sampleConfigFile.xml";
}
$TMPDIR = abs_path($TMPDIR);
print STDERR "TMPDIR=$TMPDIR \n";
if (defined($FACTORS_STR)) {
@FACTORS = split(",", $FACTORS_STR);
}
#binmode(STDIN, ":utf8");
#binmode(STDOUT, ":utf8");
$TMPDIR = "$TMPDIR/madamira.$$";
`mkdir -p $TMPDIR`;
`mkdir -p $TMPDIR/split`;
`mkdir -p $TMPDIR/out`;
my $infile = "$TMPDIR/input";
print STDERR $infile."\n";
open(TMP,">$infile");
while(<STDIN>) {
print TMP $_;
}
close(TMP);
my $cmd;
if ($USE_PARALLEL) {
# split input file
my $SPLIT_EXEC = `gsplit --help 2>/dev/null`;
if($SPLIT_EXEC) {
$SPLIT_EXEC = 'gsplit';
}
else {
$SPLIT_EXEC = 'split';
}
$cmd = "$SPLIT_EXEC -l 10000 -a 7 -d $TMPDIR/input $TMPDIR/split/x";
`$cmd`;
$cmd = "cd $MADA_DIR && parallel --jobs 4 java -Xmx2500m -Xms2500m -XX:NewRatio=3 -jar $MADA_DIR/MADAMIRA.jar -rawinput {} -rawoutdir $TMPDIR/out -rawconfig $CONFIG ::: $TMPDIR/split/x*";
print STDERR "Executing: $cmd\n";
`$cmd`;
$cmd = "cat $TMPDIR/out/x*.$SCHEME.tok > $infile.mada";
print STDERR "Executing: $cmd\n";
`$cmd`;
}
else {
$cmd = "cd $MADA_DIR && java -Xmx2500m -Xms2500m -XX:NewRatio=3 -jar $MADA_DIR/MADAMIRA.jar -rawinput $infile -rawoutdir $TMPDIR/out -rawconfig $CONFIG";
print STDERR "Executing: $cmd\n";
`$cmd`;
$cmd = "cat $TMPDIR/out/input.$SCHEME.tok > $infile.mada";
print STDERR "Executing: $cmd\n";
`$cmd`;
}
# get stuff out of mada output
open(MADA_OUT,"<$infile.mada");
#binmode(MADA_OUT, ":utf8");
while(my $line = <MADA_OUT>) {
chomp($line);
print "$line\n";
}
close (MADA_OUT);
if ($KEEP_TMP == 0) {
# `rm -rf $TMPDIR`;
}

2
scripts/training/wrappers/madamira-wrapper.perl
View File

@ -12,7 +12,6 @@ sub GetFactors;
my $TMPDIR = "tmp";
my $SCHEME = "D2";
my $KEEP_TMP = 0;
my $MADA_DIR;
my $CONFIG;
@ -21,7 +20,6 @@ my $FACTORS_STR;
my @FACTORS;
GetOptions(
"scheme=s" => \$SCHEME,
"tmpdir=s" => \$TMPDIR,
"keep-tmp" => \$KEEP_TMP,
"mada-dir=s" => \$MADA_DIR,

12
scripts/training/wrappers/make-factor-brown-cluster-mkcls.perl
View File

@ -3,11 +3,18 @@
use warnings;
use strict;
my ($lowercase, $cluster_file,$in,$out,$tmp) = @ARGV;
my ($lowercase,$cluster_file,$in,$out,$tmp) = @ARGV;
my $CLUSTER = &read_cluster_from_mkcls($cluster_file);
open(IN,$in) || die("ERROR: could not open input");
# is $lowercase a script?
if ($lowercase =~ /\//) {
open(IN,"$lowercase < $in|") || die("ERROR: could not open input");
$lowercase = 0;
}
else {
open(IN,$in) || die("ERROR: could not open input");
}
binmode(IN, ":utf8");
open(OUT,">$out");
binmode(OUT, ":utf8");
@ -18,6 +25,7 @@ while(<IN>) {
s/ $//;
my $first = 1;
foreach my $word (split) {
# if lowercase is a flag
if ($lowercase) {
$word = lc($word);
}

33
scripts/training/wrappers/make-factor-de-lemma.perl Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/perl -w
use strict;
use Encode;
use FindBin qw($RealBin);
my ($in,$out,$tmpdir) = @ARGV;
`mkdir -p $tmpdir`;
`$RealBin/../../tokenizer/deescape-special-chars.perl < $in | /home/pkoehn/statmt/bin/unicode2latin1.perl > $tmpdir/tok.$$`;
`/home/pkoehn/statmt/bin/run-lopar-tagger.perl $tmpdir/tok.$$ $tmpdir/lopar.$$`;
open(LOPAR,"$tmpdir/lopar.$$");
open(OUT,"|$RealBin/../../tokenizer/escape-special-chars.perl > $out");
while(<LOPAR>) {
chomp;
s/ +/ /g;
s/^ //;
s/ $//;
my $first = 1;
foreach (split) {
die("ERROR: choked on token '$_'") unless /^(.+)_([^_]+)_(.+)$/;
my ($word,$pos,$lemma) = ($1,$2,$3);
print OUT " " unless $first;
$first = 0;
$lemma =~ s/\|.+$//;
$lemma = $word if $lemma =~ /^\<.+\>$/;
print OUT encode('utf8', decode('iso-8859-1', $lemma));
}
print OUT "\n";
}
close(LOPAR);
close(OUT);

10
scripts/training/wrappers/make-factor-en-porter.perl Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/perl -w
use strict;
use FindBin qw($RealBin);
my ($in,$out,$tmpdir) = @ARGV;
my $porter_in = "$tmpdir/porter-in.$$";
`$RealBin/../../tokenizer/deescape-special-chars.perl < $in > $porter_in`;
`/home/pkoehn/statmt/bin/porter-stemmer $porter_in | $RealBin/../../tokenizer/escape-special-chars.perl > $out`;