moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-25 12:52:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

skelete pt

Browse Source
This commit is contained in:
Hieu Hoang 2013-09-27 20:45:20 +01:00
commit 06c5621d97
97 changed files with 1088 additions and 1117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
misc/GenerateSequences.cpp
View File

@ -22,39 +22,34 @@ int stringToInteger(string s)
void loadInput(const char * fileName, vector <string> & input)
{
ifstream sr (fileName);
string line;
if(sr.is_open())
{
while(getline(sr , line ))
{
input.push_back(line);
}
sr.close();
}
else
{
cout<<"Unable to read "<<fileName<<endl;
exit(1);
}
ifstream sr (fileName);
string line;
if(sr.is_open()) {
while(getline(sr , line )) {
input.push_back(line);
}
sr.close();
} else {
cout<<"Unable to read "<<fileName<<endl;
exit(1);
}
}
void getWords(string s, vector <string> & currInput)
{
istringstream iss(s);
currInput.clear();
do
{
string sub;
iss >> sub;
currInput.push_back(sub);
istringstream iss(s);
currInput.clear();
do {
string sub;
iss >> sub;
currInput.push_back(sub);
} while (iss);
} while (iss);
currInput.pop_back();
currInput.pop_back();
}
string getTranslation(int index, vector < pair <string , vector <int> > > & gCepts , vector <string> & currF , map <string,int> & singletons)
@ -463,58 +458,57 @@ void constructCepts(vector < pair < set <int> , set <int> > > & ceptsInPhrase, s
int main(int argc, char * argv[])
{
vector <string> e;
vector <string> f;
vector <string> a;
vector <string> singletons;
map <string,int> sTons;
vector < pair < set <int> , set <int> > > ceptsInPhrase;
vector < pair < string , vector <int> > > gCepts;
set <int> sourceNullWords;
set <int> targetNullWords;
vector <string> currE;
vector <string> currF;
vector <string> currA;
vector <string> e;
vector <string> f;
vector <string> a;
vector <string> singletons;
map <string,int> sTons;
vector < pair < set <int> , set <int> > > ceptsInPhrase;
vector < pair < string , vector <int> > > gCepts;
loadInput(argv[4],singletons);
set <int> sourceNullWords;
set <int> targetNullWords;
for(int i=0; i<singletons.size(); i++)
sTons[singletons[i]]=i;
vector <string> currE;
vector <string> currF;
vector <string> currA;
loadInput(argv[1],e);
loadInput(argv[2],f);
loadInput(argv[3],a);
loadInput(argv[4],singletons);
for (int i=0; i<a.size(); i++)
{
getWords(e[i],currE);
getWords(f[i],currF);
getWords(a[i],currA);
constructCepts(ceptsInPhrase, sourceNullWords , targetNullWords, currA , currE.size(), currF.size());
//cout<<"CC done"<<endl;
ceptsInGenerativeStoryFormat(ceptsInPhrase , gCepts , sourceNullWords, currE);
//cout<<"format done"<<endl;
// printCepts(gCepts, currF);
generateStory(gCepts, targetNullWords ,currF,sTons);
for(int i=0; i<singletons.size(); i++)
sTons[singletons[i]]=i;
/*
cout<<"________________________________________"<<endl;
cout<<"Press any integer to continue ..."<<endl;
int xx;
cin>>xx;
*/
}
loadInput(argv[1],e);
loadInput(argv[2],f);
loadInput(argv[3],a);
return 0;
for (int i=0; i<a.size(); i++) {
getWords(e[i],currE);
getWords(f[i],currF);
getWords(a[i],currA);
constructCepts(ceptsInPhrase, sourceNullWords , targetNullWords, currA , currE.size(), currF.size());
//cout<<"CC done"<<endl;
ceptsInGenerativeStoryFormat(ceptsInPhrase , gCepts , sourceNullWords, currE);
//cout<<"format done"<<endl;
// printCepts(gCepts, currF);
generateStory(gCepts, targetNullWords ,currF,sTons);
/*
cout<<"________________________________________"<<endl;
cout<<"Press any integer to continue ..."<<endl;
int xx;
cin>>xx;
*/
}
return 0;
}

12
moses-chart-cmd/Main.cpp
View File

@ -130,12 +130,12 @@ public:
//DIMw
if (staticData.IsDetailedAllTranslationReportingEnabled()) {
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
size_t nBestSize = staticData.GetNBestSize();
ChartTrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList, staticData.GetDistinctNBest());
m_ioWrapper.OutputDetailedAllTranslationReport(nBestList, manager, sentence, translationId);
}
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
size_t nBestSize = staticData.GetNBestSize();
ChartTrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList, staticData.GetDistinctNBest());
m_ioWrapper.OutputDetailedAllTranslationReport(nBestList, manager, sentence, translationId);
}
// n-best
size_t nBestSize = staticData.GetNBestSize();

26
moses-cmd/IOWrapper.cpp
View File

@ -199,12 +199,12 @@ std::map<size_t, const Factor*> GetPlaceholders(const Hypothesis &hypo, FactorTy
std::map<size_t, const Factor*> ret;
for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) {
const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
if (factor) {
std::set<size_t> targetPos = hypo.GetTranslationOption().GetTargetPhrase().GetAlignTerm().GetAlignmentsForSource(sourcePos);
CHECK(targetPos.size() == 1);
ret[*targetPos.begin()] = factor;
}
const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
if (factor) {
std::set<size_t> targetPos = hypo.GetTranslationOption().GetTargetPhrase().GetAlignTerm().GetAlignmentsForSource(sourcePos);
CHECK(targetPos.size() == 1);
ret[*targetPos.begin()] = factor;
}
}
return ret;
@ -226,8 +226,8 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<
std::map<size_t, const Factor*> placeholders;
if (placeholderFactor != NOT_FOUND) {
// creates map of target position -> factor for placeholders
placeholders = GetPlaceholders(edge, placeholderFactor);
// creates map of target position -> factor for placeholders
placeholders = GetPlaceholders(edge, placeholderFactor);
}
size_t size = phrase.GetSize();
@ -235,11 +235,11 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
if (placeholders.size()) {
// do placeholders
std::map<size_t, const Factor*>::const_iterator iter = placeholders.find(pos);
if (iter != placeholders.end()) {
factor = iter->second;
}
// do placeholders
std::map<size_t, const Factor*>::const_iterator iter = placeholders.find(pos);
if (iter != placeholders.end()) {
factor = iter->second;
}
}
CHECK(factor);

5
moses-cmd/Main.cpp
View File

@ -303,9 +303,8 @@ public:
IFVERBOSE(1) {
debug << "BEST TRANSLATION: " << *bestHypo << endl;
}
}
else {
VERBOSE(1, "NO BEST TRANSLATION" << endl);
} else {
VERBOSE(1, "NO BEST TRANSLATION" << endl);
}
out << endl;

10
moses/ChartCell.cpp
View File

@ -127,11 +127,11 @@ const ChartHypothesis *ChartCell::GetBestHypothesis() const
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
const HypoList &sortedList = iter->second.GetSortedHypotheses();
if (sortedList.size() > 0) {
const ChartHypothesis *hypo = sortedList[0];
if (hypo->GetTotalScore() > bestScore) {
bestScore = hypo->GetTotalScore();
ret = hypo;
}
const ChartHypothesis *hypo = sortedList[0];
if (hypo->GetTotalScore() > bestScore) {
bestScore = hypo->GetTotalScore();
ret = hypo;
}
}
}

24
moses/ChartHypothesis.cpp
View File

@ -102,19 +102,19 @@ void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
outPhrase.AddWord(word);
if (placeholderFactor != NOT_FOUND) {
std::set<size_t> sourcePosSet = GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
CHECK(ruleSourceFromInputPath);
std::set<size_t> sourcePosSet = GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
CHECK(ruleSourceFromInputPath);
size_t sourcePos = *sourcePosSet.begin();
const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
CHECK(sourceWord);
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor) {
outPhrase.Back()[0] = factor;
}
}
size_t sourcePos = *sourcePosSet.begin();
const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
CHECK(sourceWord);
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor) {
outPhrase.Back()[0] = factor;
}
}
}
}

8
moses/ChartHypothesisCollection.cpp
View File

@ -61,10 +61,10 @@ ChartHypothesisCollection::~ChartHypothesisCollection()
bool ChartHypothesisCollection::AddHypothesis(ChartHypothesis *hypo, ChartManager &manager)
{
if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) {
manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, -inf score" << std::endl);
ChartHypothesis::Delete(hypo);
return false;
manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, -inf score" << std::endl);
ChartHypothesis::Delete(hypo);
return false;
}
if (hypo->GetTotalScore() < m_bestScore + m_beamWidth) {

2
moses/ChartParser.cpp
View File

@ -221,7 +221,7 @@ void ChartParser::CreateInputPaths(const InputType &input)
const InputPath &ChartParser::GetInputPath(WordsRange &range) const
{
return GetInputPath(range.GetStartPos(), range.GetEndPos());
return GetInputPath(range.GetStartPos(), range.GetEndPos());
}
const InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos) const

4
moses/ChartTranslationOption.cpp
View File

@ -23,8 +23,8 @@ void ChartTranslationOption::Evaluate(const InputType &input, const InputPath &i
std::ostream& operator<<(std::ostream &out, const ChartTranslationOption &transOpt)
{
out << transOpt.m_targetPhrase << " " << transOpt.m_scoreBreakdown;
return out;
out << transOpt.m_targetPhrase << " " << transOpt.m_scoreBreakdown;
return out;
}
}

22
moses/ChartTranslationOption.h
View File

@ -10,7 +10,7 @@ class InputType;
class ChartTranslationOption
{
friend std::ostream& operator<<(std::ostream&, const ChartTranslationOption&);
friend std::ostream& operator<<(std::ostream&, const ChartTranslationOption&);
protected:
const TargetPhrase &m_targetPhrase;
@ -25,15 +25,19 @@ public:
return m_targetPhrase;
}
const InputPath *GetInputPath() const
{ return m_inputPath; }
void SetInputPath(const InputPath *inputPath)
{ m_inputPath = inputPath; }
const InputPath *GetInputPath() const {
return m_inputPath;
}
void SetInputPath(const InputPath *inputPath) {
m_inputPath = inputPath;
}
const std::vector<const Word*> *GetSourceRuleFromInputPath() const
{ return m_ruleSourceFromInputPath; }
void SetSourceRuleFromInputPath(const std::vector<const Word*> *obj)
{ m_ruleSourceFromInputPath = obj; }
const std::vector<const Word*> *GetSourceRuleFromInputPath() const {
return m_ruleSourceFromInputPath;
}
void SetSourceRuleFromInputPath(const std::vector<const Word*> *obj) {
m_ruleSourceFromInputPath = obj;
}
const ScoreComponentCollection &GetScores() const {
return m_scoreBreakdown;

10
moses/ChartTranslationOptionList.cpp
View File

@ -68,11 +68,11 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &tpc,
}
for (size_t i = 0; i < stackVec.size(); ++i) {
const ChartCellLabel &chartCellLabel = *stackVec[i];
size_t numHypos = chartCellLabel.GetStack().cube->size();
if (numHypos == 0) {
return; // empty stack. These rules can't be used
}
const ChartCellLabel &chartCellLabel = *stackVec[i];
size_t numHypos = chartCellLabel.GetStack().cube->size();
if (numHypos == 0) {
return; // empty stack. These rules can't be used
}
}
float score = ChartTranslationOptions::CalcEstimateOfBestScore(tpc, stackVec);

69
moses/ChartTranslationOptions.cpp
View File

@ -72,7 +72,7 @@ void ChartTranslationOptions::Evaluate(const InputType &input, const InputPath &
{
SetInputPath(&inputPath);
if (StaticData::Instance().GetPlaceholderFactor() != NOT_FOUND) {
CreateSourceRuleFromInputPath();
CreateSourceRuleFromInputPath();
}
CollType::iterator iter;
@ -88,52 +88,49 @@ void ChartTranslationOptions::SetInputPath(const InputPath *inputPath)
{
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
ChartTranslationOption &transOpt = **iter;
transOpt.SetInputPath(inputPath);
ChartTranslationOption &transOpt = **iter;
transOpt.SetInputPath(inputPath);
}
}
void ChartTranslationOptions::CreateSourceRuleFromInputPath()
{
if (m_collection.size() == 0) {
return;
}
if (m_collection.size() == 0) {
return;
}
const InputPath *inputPath = m_collection.front()->GetInputPath();
CHECK(inputPath);
std::vector<const Word*> &ruleSourceFromInputPath = inputPath->AddRuleSourceFromInputPath();
const InputPath *inputPath = m_collection.front()->GetInputPath();
CHECK(inputPath);
std::vector<const Word*> &ruleSourceFromInputPath = inputPath->AddRuleSourceFromInputPath();
size_t chartCellIndex = 0;
const ChartCellLabel *chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
size_t chartCellIndex = 0;
const ChartCellLabel *chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
size_t ind = 0;
for (size_t sourcePos = m_wordsRange->GetStartPos(); sourcePos <= m_wordsRange->GetEndPos(); ++sourcePos, ++ind) {
if (chartCellLabel) {
if (sourcePos == chartCellLabel->GetCoverage().GetEndPos()) {
// end of child range. push an empty word to denote non-term
ruleSourceFromInputPath.push_back(NULL);
++chartCellIndex;
chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
}
else if (sourcePos >= chartCellLabel->GetCoverage().GetStartPos()) {
// in the range of child hypo. do nothing
}
else {
// not yet reached child range. add word
ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
}
}
else {
// no child in sight. add word
ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
}
}
size_t ind = 0;
for (size_t sourcePos = m_wordsRange->GetStartPos(); sourcePos <= m_wordsRange->GetEndPos(); ++sourcePos, ++ind) {
if (chartCellLabel) {
if (sourcePos == chartCellLabel->GetCoverage().GetEndPos()) {
// end of child range. push an empty word to denote non-term
ruleSourceFromInputPath.push_back(NULL);
++chartCellIndex;
chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
} else if (sourcePos >= chartCellLabel->GetCoverage().GetStartPos()) {
// in the range of child hypo. do nothing
} else {
// not yet reached child range. add word
ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
}
} else {
// no child in sight. add word
ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
}
}
// save it to each trans opt
// save it to each trans opt
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
ChartTranslationOption &transOpt = **iter;
transOpt.SetSourceRuleFromInputPath(&ruleSourceFromInputPath);
ChartTranslationOption &transOpt = **iter;
transOpt.SetSourceRuleFromInputPath(&ruleSourceFromInputPath);
}
}

24
moses/ChartTrellisNode.cpp
View File

@ -93,19 +93,19 @@ Phrase ChartTrellisNode::GetOutputPhrase() const
ret.AddWord(word);
if (placeholderFactor != NOT_FOUND) {
std::set<size_t> sourcePosSet = m_hypo.GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = m_hypo.GetTranslationOption().GetSourceRuleFromInputPath();
CHECK(ruleSourceFromInputPath);
std::set<size_t> sourcePosSet = m_hypo.GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = m_hypo.GetTranslationOption().GetSourceRuleFromInputPath();
CHECK(ruleSourceFromInputPath);
size_t sourcePos = *sourcePosSet.begin();
const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
CHECK(sourceWord);
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor) {
ret.Back()[0] = factor;
}
}
size_t sourcePos = *sourcePosSet.begin();
const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
CHECK(sourceWord);
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor) {
ret.Back()[0] = factor;
}
}
}
}
}

4
moses/ConfusionNet.cpp
View File

@ -81,8 +81,8 @@ ConfusionNet::ConfusionNet(Sentence const& s)
{
data.resize(s.GetSize());
for(size_t i=0; i<s.GetSize(); ++i) {
ScorePair scorePair;
std::pair<Word, ScorePair > temp = std::make_pair(s.GetWord(i), scorePair);
ScorePair scorePair;
std::pair<Word, ScorePair > temp = std::make_pair(s.GetWord(i), scorePair);
data[i].push_back(temp);
}
}

12
moses/FF/BleuScoreFeature.h
View File

@ -119,14 +119,14 @@ public:
int featureID,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
bool Enabled() const {

159
moses/FF/ConstrainedDecoding.cpp
View File

@ -14,75 +14,74 @@ namespace Moses
{
ConstrainedDecodingState::ConstrainedDecodingState(const Hypothesis &hypo)
{
hypo.GetOutputPhrase(m_outputPhrase);
hypo.GetOutputPhrase(m_outputPhrase);
}
ConstrainedDecodingState::ConstrainedDecodingState(const ChartHypothesis &hypo)
{
hypo.GetOutputPhrase(m_outputPhrase);
hypo.GetOutputPhrase(m_outputPhrase);
}
int ConstrainedDecodingState::Compare(const FFState& other) const
{
const ConstrainedDecodingState &otherFF = static_cast<const ConstrainedDecodingState&>(other);
bool ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
return ret;
const ConstrainedDecodingState &otherFF = static_cast<const ConstrainedDecodingState&>(other);
bool ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
return ret;
}
//////////////////////////////////////////////////////////////////
void ConstrainedDecoding::Load()
{
const StaticData &staticData = StaticData::Instance();
bool addBeginEndWord = (staticData.GetSearchAlgorithm() == ChartDecoding) || (staticData.GetSearchAlgorithm() == ChartIncremental);
const StaticData &staticData = StaticData::Instance();
bool addBeginEndWord = (staticData.GetSearchAlgorithm() == ChartDecoding) || (staticData.GetSearchAlgorithm() == ChartIncremental);
InputFileStream constraintFile(m_path);
std::string line;
long sentenceID = staticData.GetStartTranslationId() - 1;
while (getline(constraintFile, line)) {
vector<string> vecStr = Tokenize(line, "\t");
InputFileStream constraintFile(m_path);
std::string line;
long sentenceID = staticData.GetStartTranslationId() - 1;
while (getline(constraintFile, line)) {
vector<string> vecStr = Tokenize(line, "\t");
Phrase phrase(0);
if (vecStr.size() == 1) {
sentenceID++;
phrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), vecStr[0], staticData.GetFactorDelimiter(), NULL);
} else if (vecStr.size() == 2) {
sentenceID = Scan<long>(vecStr[0]);
phrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), vecStr[1], staticData.GetFactorDelimiter(), NULL);
} else {
CHECK(false);
}
Phrase phrase(0);
if (vecStr.size() == 1) {
sentenceID++;
phrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), vecStr[0], staticData.GetFactorDelimiter(), NULL);
} else if (vecStr.size() == 2) {
sentenceID = Scan<long>(vecStr[0]);
phrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), vecStr[1], staticData.GetFactorDelimiter(), NULL);
} else {
CHECK(false);
}
if (addBeginEndWord) {
phrase.InitStartEndWord();
}
m_constraints.insert(make_pair(sentenceID,phrase));
if (addBeginEndWord) {
phrase.InitStartEndWord();
}
m_constraints.insert(make_pair(sentenceID,phrase));
}
}
}
std::vector<float> ConstrainedDecoding::DefaultWeights() const
{
CHECK(m_numScoreComponents == 1);
vector<float> ret(1, 1);
return ret;
CHECK(m_numScoreComponents == 1);
vector<float> ret(1, 1);
return ret;
}
template <class H, class M>
const Phrase *GetConstraint(const std::map<long,Phrase> &constraints, const H &hypo)
{
const M &mgr = hypo.GetManager();
const InputType &input = mgr.GetSource();
long id = input.GetTranslationId();
const M &mgr = hypo.GetManager();
const InputType &input = mgr.GetSource();
long id = input.GetTranslationId();
map<long,Phrase>::const_iterator iter;
iter = constraints.find(id);
map<long,Phrase>::const_iterator iter;
iter = constraints.find(id);
if (iter == constraints.end()) {
return NULL;
}
else {
return &iter->second;
}
if (iter == constraints.end()) {
return NULL;
} else {
return &iter->second;
}
}
FFState* ConstrainedDecoding::Evaluate(
@ -90,27 +89,26 @@ FFState* ConstrainedDecoding::Evaluate(
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
const Phrase *ref = GetConstraint<Hypothesis, Manager>(m_constraints, hypo);
CHECK(ref);
const Phrase *ref = GetConstraint<Hypothesis, Manager>(m_constraints, hypo);
CHECK(ref);
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
const Phrase &outputPhrase = ret->GetPhrase();
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
const Phrase &outputPhrase = ret->GetPhrase();
size_t searchPos = ref->Find(outputPhrase, m_maxUnknowns);
size_t searchPos = ref->Find(outputPhrase, m_maxUnknowns);
float score;
if (hypo.IsSourceCompleted()) {
// translated entire sentence.
score = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize())
? 0 : - std::numeric_limits<float>::infinity();
}
else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
}
float score;
if (hypo.IsSourceCompleted()) {
// translated entire sentence.
score = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize())
? 0 : - std::numeric_limits<float>::infinity();
} else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
}
accumulator->PlusEquals(this, score);
accumulator->PlusEquals(this, score);
return ret;
return ret;
}
FFState* ConstrainedDecoding::EvaluateChart(
@ -118,41 +116,38 @@ FFState* ConstrainedDecoding::EvaluateChart(
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
const Phrase *ref = GetConstraint<ChartHypothesis, ChartManager>(m_constraints, hypo);
CHECK(ref);
const Phrase *ref = GetConstraint<ChartHypothesis, ChartManager>(m_constraints, hypo);
CHECK(ref);
const ChartManager &mgr = hypo.GetManager();
const Sentence &source = static_cast<const Sentence&>(mgr.GetSource());
const ChartManager &mgr = hypo.GetManager();
const Sentence &source = static_cast<const Sentence&>(mgr.GetSource());
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
const Phrase &outputPhrase = ret->GetPhrase();
size_t searchPos = ref->Find(outputPhrase, m_maxUnknowns);
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
const Phrase &outputPhrase = ret->GetPhrase();
size_t searchPos = ref->Find(outputPhrase, m_maxUnknowns);
float score;
if (hypo.GetCurrSourceRange().GetStartPos() == 0 &&
hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) {
// translated entire sentence.
score = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize())
? 0 : - std::numeric_limits<float>::infinity();
}
else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
}
float score;
if (hypo.GetCurrSourceRange().GetStartPos() == 0 &&
hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) {
// translated entire sentence.
score = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize())
? 0 : - std::numeric_limits<float>::infinity();
} else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
}
accumulator->PlusEquals(this, score);
accumulator->PlusEquals(this, score);
return ret;
return ret;
}
void ConstrainedDecoding::SetParameter(const std::string& key, const std::string& value)
{
if (key == "path") {
m_path = value;
}
else if (key == "max-unknowns") {
m_maxUnknowns = Scan<int>(value);
}
else {
m_path = value;
} else if (key == "max-unknowns") {
m_maxUnknowns = Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}

90
moses/FF/ConstrainedDecoding.h
View File

@ -11,19 +11,20 @@ namespace Moses
class ConstrainedDecodingState : public FFState
{
public:
ConstrainedDecodingState()
{}
ConstrainedDecodingState()
{}
ConstrainedDecodingState(const Hypothesis &hypo);
ConstrainedDecodingState(const ChartHypothesis &hypo);
ConstrainedDecodingState(const Hypothesis &hypo);
ConstrainedDecodingState(const ChartHypothesis &hypo);
int Compare(const FFState& other) const;
int Compare(const FFState& other) const;
const Phrase &GetPhrase() const
{ return m_outputPhrase; }
const Phrase &GetPhrase() const {
return m_outputPhrase;
}
protected:
Phrase m_outputPhrase;
Phrase m_outputPhrase;
};
//////////////////////////////////////////////////////////////////
@ -32,52 +33,51 @@ protected:
class ConstrainedDecoding : public StatefulFeatureFunction
{
public:
ConstrainedDecoding(const std::string &line)
:StatefulFeatureFunction("ConstrainedDecoding", 1, line)
,m_maxUnknowns(0)
{
m_tuneable = false;
ReadParameters();
}
ConstrainedDecoding(const std::string &line)
:StatefulFeatureFunction("ConstrainedDecoding", 1, line)
,m_maxUnknowns(0) {
m_tuneable = false;
ReadParameters();
}
void Load();
void Load();
bool IsUseable(const FactorMask &mask) const
{ return true; }
bool IsUseable(const FactorMask &mask) const {
return true;
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const
{
return new ConstrainedDecodingState();
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new ConstrainedDecodingState();
}
std::vector<float> DefaultWeights() const;
std::vector<float> DefaultWeights() const;
void SetParameter(const std::string& key, const std::string& value);
void SetParameter(const std::string& key, const std::string& value);
protected:
std::string m_path;
std::map<long,Phrase> m_constraints;
int m_maxUnknowns;
std::string m_path;
std::map<long,Phrase> m_constraints;
int m_maxUnknowns;
};

70
moses/FF/ControlRecombination.cpp
View File

@ -13,48 +13,45 @@ using namespace std;
namespace Moses
{
ControlRecombinationState::ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff)
:m_ff(ff)
:m_ff(ff)
{
if (ff.GetType() == SameOutput) {
UTIL_THROW(util::Exception, "Implemented not yet completed for phrase-based model. Need to take into account the coverage");
hypo.GetOutputPhrase(m_outputPhrase);
}
else {
m_hypo = &hypo;
}
if (ff.GetType() == SameOutput) {
UTIL_THROW(util::Exception, "Implemented not yet completed for phrase-based model. Need to take into account the coverage");
hypo.GetOutputPhrase(m_outputPhrase);
} else {
m_hypo = &hypo;
}
}
ControlRecombinationState::ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff)
:m_ff(ff)
:m_ff(ff)
{
if (ff.GetType() == SameOutput) {
hypo.GetOutputPhrase(m_outputPhrase);
}
else {
m_hypo = &hypo;
}
if (ff.GetType() == SameOutput) {
hypo.GetOutputPhrase(m_outputPhrase);
} else {
m_hypo = &hypo;
}
}
int ControlRecombinationState::Compare(const FFState& other) const
{
const ControlRecombinationState &otherFF = static_cast<const ControlRecombinationState&>(other);
if (m_ff.GetType() == SameOutput) {
bool ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
return ret;
}
else {
// compare hypo address. Won't be equal unless they're actually the same hypo
if (m_hypo == otherFF.m_hypo)
return 0;
return (m_hypo < otherFF.m_hypo) ? -1 : +1;
}
const ControlRecombinationState &otherFF = static_cast<const ControlRecombinationState&>(other);
if (m_ff.GetType() == SameOutput) {
bool ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
return ret;
} else {
// compare hypo address. Won't be equal unless they're actually the same hypo
if (m_hypo == otherFF.m_hypo)
return 0;
return (m_hypo < otherFF.m_hypo) ? -1 : +1;
}
}
std::vector<float> ControlRecombination::DefaultWeights() const
{
CHECK(m_numScoreComponents == 0);
vector<float> ret(0);
return ret;
CHECK(m_numScoreComponents == 0);
vector<float> ret(0);
return ret;
}
FFState* ControlRecombination::Evaluate(
@ -62,7 +59,7 @@ FFState* ControlRecombination::Evaluate(
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
return new ControlRecombinationState(hypo, *this);
return new ControlRecombinationState(hypo, *this);
}
FFState* ControlRecombination::EvaluateChart(
@ -70,17 +67,16 @@ FFState* ControlRecombination::EvaluateChart(
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new ControlRecombinationState(hypo, *this);
return new ControlRecombinationState(hypo, *this);
}
void ControlRecombination::SetParameter(const std::string& key, const std::string& value)
{
if (key == "type") {
m_type = (ControlRecombinationType) Scan<int>(value);
}
else {
StatefulFeatureFunction::SetParameter(key, value);
}
if (key == "type") {
m_type = (ControlRecombinationType) Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
}

100
moses/FF/ControlRecombination.h
View File

@ -8,8 +8,8 @@
namespace Moses
{
enum ControlRecombinationType
{ // when to recombine
enum ControlRecombinationType {
// when to recombine
SameOutput = 1,
Never = 2
};
@ -19,22 +19,23 @@ class ControlRecombination;
class ControlRecombinationState : public FFState
{
public:
ControlRecombinationState(const ControlRecombination &ff)
:m_ff(ff)
{}
ControlRecombinationState(const ControlRecombination &ff)
:m_ff(ff)
{}
ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
int Compare(const FFState& other) const;
int Compare(const FFState& other) const;
const Phrase &GetPhrase() const
{ return m_outputPhrase; }
const Phrase &GetPhrase() const {
return m_outputPhrase;
}
protected:
Phrase m_outputPhrase;
const ControlRecombination &m_ff;
const void *m_hypo;
Phrase m_outputPhrase;
const ControlRecombination &m_ff;
const void *m_hypo;
};
//////////////////////////////////////////////////////////////////
@ -43,51 +44,52 @@ protected:
class ControlRecombination : public StatefulFeatureFunction
{
public:
ControlRecombination(const std::string &line)
:StatefulFeatureFunction("ControlRecombination", 0, line)
,m_type(SameOutput)
ControlRecombination(const std::string &line)
:StatefulFeatureFunction("ControlRecombination", 0, line)
,m_type(SameOutput)
{
m_tuneable = false;
ReadParameters();
}
{
m_tuneable = false;
ReadParameters();
}
bool IsUseable(const FactorMask &mask) const
{ return true; }
bool IsUseable(const FactorMask &mask) const {
return true;
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const
{
return new ControlRecombinationState(*this);
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new ControlRecombinationState(*this);
}
std::vector<float> DefaultWeights() const;
std::vector<float> DefaultWeights() const;
void SetParameter(const std::string& key, const std::string& value);
void SetParameter(const std::string& key, const std::string& value);
ControlRecombinationType GetType() const
{ return m_type; }
ControlRecombinationType GetType() const {
return m_type;
}
protected:
ControlRecombinationType m_type;
ControlRecombinationType m_type;
};

16
moses/FF/DecodeFeature.h
View File

@ -64,20 +64,20 @@ public:
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
protected:

12
moses/FF/DistortionScoreProducer.h
View File

@ -42,14 +42,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

34
moses/FF/ExternalFeature.cpp
View File

@ -7,9 +7,9 @@ namespace Moses
{
ExternalFeatureState::ExternalFeatureState(int stateSize, void *data)
{
m_stateSize = stateSize;
m_data = malloc(stateSize);
memcpy(m_data, data, stateSize);
m_stateSize = stateSize;
m_data = malloc(stateSize);
memcpy(m_data, data, stateSize);
}
void ExternalFeature::Load()
@ -17,27 +17,28 @@ void ExternalFeature::Load()
string nparam = "testing";
if (m_path.size() < 1) {
cerr << "External requires a path to a dynamic library!\n";
abort();
cerr << "External requires a path to a dynamic library!\n";
abort();
}
lib_handle = dlopen(m_path.c_str(), RTLD_LAZY);
if (!lib_handle) {
cerr << "dlopen reports: " << dlerror() << endl;
cerr << "Did you provide a full path to the dynamic library?\n";
abort();
cerr << "dlopen reports: " << dlerror() << endl;
cerr << "Did you provide a full path to the dynamic library?\n";
abort();
}
CdecFF* (*fn)(const string&) =
(CdecFF* (*)(const string&))(dlsym(lib_handle, "create_ff"));
(CdecFF* (*)(const string&))(dlsym(lib_handle, "create_ff"));
if (!fn) {
cerr << "dlsym reports: " << dlerror() << endl;
abort();
cerr << "dlsym reports: " << dlerror() << endl;
abort();
}
ff_ext = (*fn)(nparam);
m_stateSize = ff_ext->StateSize();
}
ExternalFeature::~ExternalFeature() {
ExternalFeature::~ExternalFeature()
{
delete ff_ext;
dlclose(lib_handle);
}
@ -45,9 +46,8 @@ ExternalFeature::~ExternalFeature() {
void ExternalFeature::SetParameter(const std::string& key, const std::string& value)
{
if (key == "path") {
m_path = value;
}
else {
m_path = value;
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
@ -57,7 +57,7 @@ FFState* ExternalFeature::Evaluate(
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
return new ExternalFeatureState(m_stateSize);
return new ExternalFeatureState(m_stateSize);
}
FFState* ExternalFeature::EvaluateChart(
@ -65,7 +65,7 @@ FFState* ExternalFeature::EvaluateChart(
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new ExternalFeatureState(m_stateSize);
return new ExternalFeatureState(m_stateSize);
}

105
moses/FF/ExternalFeature.h
View File

@ -12,82 +12,79 @@ class CdecFF;
class ExternalFeatureState : public FFState
{
protected:
int m_stateSize;
void *m_data;
int m_stateSize;
void *m_data;
public:
ExternalFeatureState(int stateSize)
:m_stateSize(stateSize)
,m_data(NULL)
{}
ExternalFeatureState(int stateSize, void *data);
ExternalFeatureState(int stateSize)
:m_stateSize(stateSize)
,m_data(NULL)
{}
ExternalFeatureState(int stateSize, void *data);
~ExternalFeatureState()
{
free(m_data);
}
~ExternalFeatureState() {
free(m_data);
}
int Compare(const FFState& other) const
{
const ExternalFeatureState &otherFF = static_cast<const ExternalFeatureState&>(other);
int ret = memcmp(m_data, otherFF.m_data, m_stateSize);
return ret;
}
int Compare(const FFState& other) const {
const ExternalFeatureState &otherFF = static_cast<const ExternalFeatureState&>(other);
int ret = memcmp(m_data, otherFF.m_data, m_stateSize);
return ret;
}
};
// copied from cdec
class ExternalFeature : public StatefulFeatureFunction
{
public:
ExternalFeature(const std::string &line)
:StatefulFeatureFunction("ExternalFeature", line)
{
ReadParameters();
}
~ExternalFeature();
ExternalFeature(const std::string &line)
:StatefulFeatureFunction("ExternalFeature", line) {
ReadParameters();
}
~ExternalFeature();
void Load();
void Load();
bool IsUseable(const FactorMask &mask) const
{ return true; }
bool IsUseable(const FactorMask &mask) const {
return true;
}
void SetParameter(const std::string& key, const std::string& value);
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const
{
return new ExternalFeatureState(m_stateSize);
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new ExternalFeatureState(m_stateSize);
}
protected:
std::string m_path;
void* lib_handle;
CdecFF *ff_ext;
int m_stateSize;
std::string m_path;
void* lib_handle;
CdecFF *ff_ext;
int m_stateSize;
};
class CdecFF
{
public:
virtual int StateSize() const = 0;
virtual int StateSize() const = 0;
};
}

12
moses/FF/FeatureFunction.cpp
View File

@ -30,8 +30,8 @@ FeatureFunction &FeatureFunction::FindFeatureFunction(const std::string& name)
}
FeatureFunction::
FeatureFunction(const std::string& description,
const std::string& line)
FeatureFunction(const std::string& description,
const std::string& line)
: m_tuneable(true)
, m_numScoreComponents(1)
{
@ -39,16 +39,16 @@ FeatureFunction(const std::string& description,
}
FeatureFunction::
FeatureFunction(const std::string& description,
size_t numScoreComponents,
const std::string& line)
FeatureFunction(const std::string& description,
size_t numScoreComponents,
const std::string& line)
: m_tuneable(true)
, m_numScoreComponents(numScoreComponents)
{
Initialize(description, line);
}
void
void
FeatureFunction::
Initialize(const std::string& description, const std::string &line)
{

12
moses/FF/GlobalLexicalModel.h
View File

@ -81,14 +81,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}

2
moses/FF/GlobalLexicalModelUnlimited.cpp
View File

@ -134,7 +134,7 @@ void GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, ScoreComp
boost::unordered_set<uint64_t> alreadyScored;
for(size_t sourceIndex = 0; sourceIndex < input.GetSize(); sourceIndex++ ) {
const StringPiece sourceString = input.GetWord(sourceIndex).GetString(0);
const StringPiece sourceString = input.GetWord(sourceIndex).GetString(0);
// TODO: change for other factors
if (m_ignorePunctuation) {

12
moses/FF/GlobalLexicalModelUnlimited.h
View File

@ -91,14 +91,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void AddFeature(ScoreComponentCollection* accumulator,

20
moses/FF/InputFeature.cpp
View File

@ -19,11 +19,11 @@ InputFeature::InputFeature(const std::string &line)
void InputFeature::Load()
{
const StaticData &staticData = StaticData::Instance();
const PhraseDictionary *pt = staticData.GetTranslationScoreProducer(0);
const PhraseDictionaryTreeAdaptor *ptBin = dynamic_cast<const PhraseDictionaryTreeAdaptor*>(pt);
const StaticData &staticData = StaticData::Instance();
const PhraseDictionary *pt = staticData.GetTranslationScoreProducer(0);
const PhraseDictionaryTreeAdaptor *ptBin = dynamic_cast<const PhraseDictionaryTreeAdaptor*>(pt);
m_legacy = (ptBin != NULL);
m_legacy = (ptBin != NULL);
}
void InputFeature::SetParameter(const std::string& key, const std::string& value)
@ -43,16 +43,16 @@ void InputFeature::Evaluate(const InputType &input
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{
if (m_legacy) {
//binary phrase-table does input feature itself
return;
}
/*
if (m_legacy) {
//binary phrase-table does input feature itself
return;
}
/*
const ScorePair *scores = inputPath.GetInputScore();
if (scores) {
scoreBreakdown.PlusEquals(this, *scores);
}
*/
*/
}
} // namespace

16
moses/FF/InputFeature.h
View File

@ -33,20 +33,20 @@ public:
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}

12
moses/FF/LexicalReordering/LexicalReordering.h
View File

@ -52,14 +52,14 @@ public:
UTIL_THROW(util::Exception, "LexicalReordering is not valid for chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
private:

13
moses/FF/OSM-Feature/OpSequenceModel.cpp
View File

@ -235,13 +235,14 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va
if (key == "path") {
m_lmPath = value;
} else if (key == "numFeatures") {
numFeatures = Scan<int>(value);
} else if (key == "order") {
lmOrder = Scan<int>(value);
} else if (key == "sFactor") {
} else if (key == "support-features") {
if(value == "no")
numFeatures = 1;
else
numFeatures = 5;
} else if (key == "input-factor") {
sFactor = Scan<int>(value);
} else if (key == "tFactor") {
} else if (key == "output-factor") {
tFactor = Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);

8
moses/FF/OSM-Feature/OpSequenceModel.h
View File

@ -18,8 +18,6 @@ public:
lm::ngram::Model * OSM;
int lmOrder;
float unkOpProb;
int sFactor; // Source Factor ...
int tFactor; // Target Factor ...
@ -41,9 +39,9 @@ public:
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase

12
moses/FF/PhraseBoundaryFeature.h
View File

@ -54,14 +54,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);

8
moses/FF/PhraseLengthFeature.h
View File

@ -25,7 +25,7 @@ public:
}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis& hypo,
@ -34,9 +34,9 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
virtual void Evaluate(const Phrase &source

12
moses/FF/PhrasePairFeature.h
View File

@ -46,14 +46,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Load();

10
moses/FF/PhrasePenalty.h
View File

@ -20,16 +20,16 @@ public:
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
};

60
moses/FF/SkeletonStatefulFF.cpp
View File

@ -9,51 +9,51 @@ namespace Moses
{
int SkeletonState::Compare(const FFState& other) const
{
const SkeletonState &otherState = static_cast<const SkeletonState&>(other);
const SkeletonState &otherState = static_cast<const SkeletonState&>(other);
if (m_targetLen == otherState.m_targetLen)
return 0;
return (m_targetLen < otherState.m_targetLen) ? -1 : +1;
if (m_targetLen == otherState.m_targetLen)
return 0;
return (m_targetLen < otherState.m_targetLen) ? -1 : +1;
}
void SkeletonStatefulFF::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SkeletonStatefulFF::Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* SkeletonStatefulFF::Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
// dense scores
vector<float> newScores(m_numScoreComponents);
newScores[0] = 1.5;
newScores[1] = 0.3;
newScores[2] = 0.4;
accumulator->PlusEquals(this, newScores);
// dense scores
vector<float> newScores(m_numScoreComponents);
newScores[0] = 1.5;
newScores[1] = 0.3;
newScores[2] = 0.4;
accumulator->PlusEquals(this, newScores);
// sparse scores
accumulator->PlusEquals(this, "sparse-name", 2.4);
// sparse scores
accumulator->PlusEquals(this, "sparse-name", 2.4);
int targetLen = cur_hypo.GetCurrTargetPhrase().GetSize();
return new SkeletonState(0);
int targetLen = cur_hypo.GetCurrTargetPhrase().GetSize();
return new SkeletonState(0);
}
FFState* SkeletonStatefulFF::EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState(0);
}
FFState* SkeletonStatefulFF::EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState(0);
}
}

58
moses/FF/SkeletonStatefulFF.h
View File

@ -9,45 +9,45 @@ namespace Moses
class SkeletonState : public FFState
{
int m_targetLen;
int m_targetLen;
public:
SkeletonState(int targetLen)
:m_targetLen(targetLen)
{}
SkeletonState(int targetLen)
:m_targetLen(targetLen)
{}
int Compare(const FFState& other) const;
int Compare(const FFState& other) const;
};
class SkeletonStatefulFF : public StatefulFeatureFunction
{
public:
SkeletonStatefulFF(const std::string &line)
:StatefulFeatureFunction("SkeletonStatefulFF", 3, line)
{}
SkeletonStatefulFF(const std::string &line)
:StatefulFeatureFunction("SkeletonStatefulFF", 3, line)
{}
bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual const FFState* EmptyHypothesisState(const InputType &input) const
{
return new SkeletonState(0);
bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new SkeletonState(0);
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
};

30
moses/FF/SkeletonStatelessFF.cpp
View File

@ -7,33 +7,33 @@ using namespace std;
namespace Moses
{
void SkeletonStatelessFF::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// dense scores
vector<float> newScores(m_numScoreComponents);
newScores[0] = 1.5;
newScores[1] = 0.3;
scoreBreakdown.PlusEquals(this, newScores);
// dense scores
vector<float> newScores(m_numScoreComponents);
newScores[0] = 1.5;
newScores[1] = 0.3;
scoreBreakdown.PlusEquals(this, newScores);
// sparse scores
scoreBreakdown.PlusEquals(this, "sparse-name", 2.4);
// sparse scores
scoreBreakdown.PlusEquals(this, "sparse-name", 2.4);
}
void SkeletonStatelessFF::Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void SkeletonStatelessFF::Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void SkeletonStatelessFF::EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
}

35
moses/FF/SkeletonStatelessFF.h
View File

@ -9,25 +9,26 @@ namespace Moses
class SkeletonStatelessFF : public StatelessFeatureFunction
{
public:
SkeletonStatelessFF(const std::string &line)
:StatelessFeatureFunction("SkeletonStatelessFF", 2, line)
{}
SkeletonStatelessFF(const std::string &line)
:StatelessFeatureFunction("SkeletonStatelessFF", 2, line)
{}
bool IsUseable(const FactorMask &mask) const
{ return true; }
bool IsUseable(const FactorMask &mask) const {
return true;
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const;
};

16
moses/FF/SourceWordDeletionFeature.h
View File

@ -29,19 +29,19 @@ public:
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,

12
moses/FF/TargetBigramFeature.h
View File

@ -48,14 +48,14 @@ public:
throw std::logic_error("TargetBigramFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);

12
moses/FF/TargetNgramFeature.h
View File

@ -193,14 +193,14 @@ public:
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);

10
moses/FF/TargetWordInsertionFeature.h
View File

@ -33,15 +33,15 @@ public:
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,

16
moses/FF/UnknownWordPenaltyProducer.h
View File

@ -23,20 +23,20 @@ public:
std::vector<float> DefaultWeights() const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

10
moses/FF/WordPenaltyProducer.h
View File

@ -23,15 +23,15 @@ public:
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
};

12
moses/FF/WordTranslationFeature.h
View File

@ -54,14 +54,14 @@ public:
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

6
moses/Hypothesis.cpp
View File

@ -159,10 +159,10 @@ Hypothesis* Hypothesis::Create(const Hypothesis &prevHypo, const TranslationOpti
{
#ifdef USE_HYPO_POOL
Hypothesis *ptr = s_objectPool.getPtr();
return new(ptr) Hypothesis(prevHypo, transOpt);
Hypothesis *ptr = s_objectPool.getPtr();
return new(ptr) Hypothesis(prevHypo, transOpt);
#else
return new Hypothesis(prevHypo, transOpt);
return new Hypothesis(prevHypo, transOpt);
#endif
}
/***

8
moses/HypothesisStackCubePruning.cpp
View File

@ -82,10 +82,10 @@ pair<HypothesisStackCubePruning::iterator, bool> HypothesisStackCubePruning::Add
bool HypothesisStackCubePruning::AddPrune(Hypothesis *hypo)
{
if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) {
m_manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, constraint" << std::endl);
FREEHYPO(hypo);
return false;
m_manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, constraint" << std::endl);
FREEHYPO(hypo);
return false;
}
if (hypo->GetTotalScore() < m_worstScore) {

8
moses/HypothesisStackNormal.cpp
View File

@ -89,10 +89,10 @@ pair<HypothesisStackNormal::iterator, bool> HypothesisStackNormal::Add(Hypothesi
bool HypothesisStackNormal::AddPrune(Hypothesis *hypo)
{
if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) {
m_manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, constraint" << std::endl);
FREEHYPO(hypo);
return false;
m_manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, constraint" << std::endl);
FREEHYPO(hypo);
return false;
}
// too bad for stack. don't bother adding hypo into collection

8
moses/InputPath.cpp
View File

@ -10,10 +10,10 @@ using namespace std;
namespace Moses
{
InputPath::
InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms,
const WordsRange &range, const InputPath *prevNode,
const ScorePair *inputScore)
InputPath::
InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms,
const WordsRange &range, const InputPath *prevNode,
const ScorePair *inputScore)
:m_prevNode(prevNode)
,m_phrase(phrase)
,m_range(range)

7
moses/InputPath.h
View File

@ -81,10 +81,9 @@ public:
return m_inputScore;
}
std::vector<const Word*> &AddRuleSourceFromInputPath() const
{
m_ruleSourceFromInputPath.push_back(std::vector<const Word*>());
return m_ruleSourceFromInputPath.back();
std::vector<const Word*> &AddRuleSourceFromInputPath() const {
m_ruleSourceFromInputPath.push_back(std::vector<const Word*>());
return m_ruleSourceFromInputPath.back();
}
};

6
moses/LM/Base.h
View File

@ -91,9 +91,9 @@ public:
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
};

7
moses/LM/Ken.cpp
View File

@ -138,7 +138,7 @@ private:
};
} // namespace
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
:LanguageModel("KENLM", line)
,m_factorType(factorType)
@ -351,10 +351,11 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateChart(const Cha
return newState;
}
template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Incremental::Manager &manager) const {
template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Incremental::Manager &manager) const
{
manager.LMCallback(*m_ngram, m_lmIdLookup);
}
template <class Model>
bool LanguageModelKen<Model>::IsUseable(const FactorMask &mask) const
{

60
moses/LM/Ken.h
View File

@ -36,7 +36,7 @@ namespace Moses
{
//class LanguageModel;
class FFState;
class FFState;
LanguageModel *ConstructKenLM(const std::string &line);
@ -49,37 +49,37 @@ LanguageModel *ConstructKenLM(const std::string &line, const std::string &file,
template <class Model> class LanguageModelKen : public LanguageModel
{
public:
LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
virtual FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
virtual FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual bool IsUseable(const FactorMask &mask) const;
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
virtual FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
virtual FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual bool IsUseable(const FactorMask &mask) const;
protected:
boost::shared_ptr<Model> m_ngram;
const Factor *m_beginSentenceFactor;
FactorType m_factorType;
lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
boost::shared_ptr<Model> m_ngram;
const Factor *m_beginSentenceFactor;
FactorType m_factorType;
lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
private:
LanguageModelKen(const LanguageModelKen<Model> &copy_from);
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
LanguageModelKen(const LanguageModelKen<Model> &copy_from);
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
int position = hypo.GetCurrTargetWordsRange().GetEndPos();
@ -93,8 +93,8 @@ private:
}
}
std::vector<lm::WordIndex> m_lmIdLookup;
std::vector<lm::WordIndex> m_lmIdLookup;
};
} // namespace Moses

4
moses/LM/LDHT.cpp
View File

@ -2,9 +2,9 @@
// Oliver Wilson <oliver.wilson@ed.ac.uk>
//
// This file should be compiled only when the LM_RAND flag is enabled.
// This file should be compiled only when the LM_RAND flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when LM_RAND is disabled.
//
#ifdef LM_RAND

4
moses/LM/ParallelBackoff.cpp
View File

@ -36,14 +36,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/Util.h"
// By default, SRILM defines a function called zopen.
//
//
// However, on Mac OS X (and possibly other BSDs),
// <stdio.h> already defines a zopen function.
//
// To resolve this conflict, SRILM checks to see if HAVE_ZOPEN is defined.
// If it is, SRILM will rename its zopen function as my_zopen.
//
// So, before importing any SRILM headers,
// So, before importing any SRILM headers,
// it is important to define HAVE_ZOPEN if we are on an Apple OS:
//
#ifdef __APPLE__

4
moses/LM/Rand.cpp
View File

@ -17,9 +17,9 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
// This file should be compiled only when the LM_RAND flag is enabled.
// This file should be compiled only when the LM_RAND flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when LM_RAND is disabled.
//
#ifdef LM_RAND

4
moses/LM/SRI.cpp
View File

@ -32,14 +32,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/StaticData.h"
// By default, SRILM defines a function called zopen.
//
//
// However, on Mac OS X (and possibly other BSDs),
// <stdio.h> already defines a zopen function.
//
// To resolve this conflict, SRILM checks to see if HAVE_ZOPEN is defined.
// If it is, SRILM will rename its zopen function as my_zopen.
//
// So, before importing any SRILM headers,
// So, before importing any SRILM headers,
// it is important to define HAVE_ZOPEN if we are on an Apple OS:
//
#ifdef __APPLE__

9
moses/LM/SkeletonLM.cpp
View File

@ -12,7 +12,7 @@ SkeletonLM::SkeletonLM(const std::string &line)
ReadParameters();
if (m_factorType == NOT_FOUND) {
m_factorType = 0;
m_factorType = 0;
}
FactorCollection &factorCollection = FactorCollection::Instance();
@ -39,10 +39,9 @@ LMResult SkeletonLM::GetValue(const vector<const Word*> &contextFactor, State* f
const Factor *factor;
size_t hash_value(const Factor &f);
if (contextFactor.size()) {
factor = contextFactor.back()->GetFactor(m_factorType);
}
else {
factor = NULL;
factor = contextFactor.back()->GetFactor(m_factorType);
} else {
factor = NULL;
}
(*finalState) = (State*) factor;

2
moses/LM/SkeletonLM.h
View File

@ -12,7 +12,7 @@ class SkeletonLM : public LanguageModelSingleFactor
protected:
public:
SkeletonLM(const std::string &line);
SkeletonLM(const std::string &line);
~SkeletonLM();
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;

2
moses/Manager.cpp
View File

@ -60,7 +60,7 @@ Manager::Manager(size_t lineNumber, InputType const& source, SearchAlgorithm sea
,m_lineNumber(lineNumber)
,m_source(source)
{
StaticData::Instance().InitializeForInput(m_source);
StaticData::Instance().InitializeForInput(m_source);
}
Manager::~Manager()

29
moses/PCNTools.cpp
View File

@ -33,7 +33,7 @@ std::string getString(const std::string& in, int &c)
std::string ret;
eatws(in,c);
while (c < (int)in.size() && get(in,c) != ' ' && get(in,c) != ')' && get(in,c) != ',') {
ret += get(in,c++);
ret += get(in,c++);
}
eatws(in,c);
return ret;
@ -111,26 +111,25 @@ CNAlt getCNAlt(const std::string& in, int &c)
// dense scores
size_t ind;
for (ind = 0; ind < toks.size() - 1; ++ind) {
const string &tok = toks[ind];
const string &tok = toks[ind];
if (tok.find('=') == tok.npos) {
float val = Moses::Scan<float>(tok);
probs.push_back(val);
}
else {
// beginning of sparse feature
break;
}
if (tok.find('=') == tok.npos) {
float val = Moses::Scan<float>(tok);
probs.push_back(val);
} else {
// beginning of sparse feature
break;
}
}
// sparse features
std::map<string, float> sparseFeatures;
for (; ind < toks.size() - 1; ++ind) {
const string &tok = toks[ind];
vector<string> keyValue = Moses::Tokenize(tok, "=");
CHECK(keyValue.size() == 2);
float prob = Moses::Scan<float>(keyValue[1]);
sparseFeatures[ keyValue[0] ] = prob;
const string &tok = toks[ind];
vector<string> keyValue = Moses::Tokenize(tok, "=");
CHECK(keyValue.size() == 2);
float prob = Moses::Scan<float>(keyValue[1]);
sparseFeatures[ keyValue[0] ] = prob;
}
//last item is column increment

33
moses/PCNTools.h
View File

@ -34,24 +34,23 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace PCN
{
struct CNAlt
{
CNAlt()
{}
CNAlt(const std::string &word,
const std::vector<float> &denseFeatures,
const std::map<std::string, float> &sparseFeatures,
size_t next)
:m_word(word)
,m_denseFeatures(denseFeatures)
,m_sparseFeatures(sparseFeatures)
,m_next(next)
{}
struct CNAlt {
CNAlt()
{}
CNAlt(const std::string &word,
const std::vector<float> &denseFeatures,
const std::map<std::string, float> &sparseFeatures,
size_t next)
:m_word(word)
,m_denseFeatures(denseFeatures)
,m_sparseFeatures(sparseFeatures)
,m_next(next)
{}
std::string m_word;
std::vector<float> m_denseFeatures;
std::map<std::string, float> m_sparseFeatures;
size_t m_next;
std::string m_word;
std::vector<float> m_denseFeatures;
std::map<std::string, float> m_sparseFeatures;
size_t m_next;
};
//typedef std::pair<std::pair<std::string, std::vector<float> >, size_t> CNAlt;

28
moses/Phrase.cpp
View File

@ -120,9 +120,9 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const
stringstream strme;
for (size_t pos = 0 ; pos < GetSize() ; pos++) {
if(markUnknown && GetWord(pos).IsOOV()) {
strme << "UNK";
}
if(markUnknown && GetWord(pos).IsOOV()) {
strme << "UNK";
}
strme << GetWord(pos).GetString(factorsToPrint, (pos != GetSize()-1));
}
@ -388,25 +388,23 @@ size_t Phrase::Find(const Phrase &sought, int maxUnknown) const
for (soughtPos = 0; soughtPos < sought.GetSize(); ++soughtPos) {
const Word &soughtWord = sought.GetWord(soughtPos);
const Word &thisWord = GetWord(thisPos);
if (soughtWord == thisWord) {
++thisPos;
}
else if (soughtWord.IsOOV() && (maxUnknown < 0 || currUnknowns < maxUnknown)) {
// the output has an OOV word. Allow a certain number of OOVs
++currUnknowns;
++thisPos;
}
else {
break;
++thisPos;
} else if (soughtWord.IsOOV() && (maxUnknown < 0 || currUnknowns < maxUnknown)) {
// the output has an OOV word. Allow a certain number of OOVs
++currUnknowns;
++thisPos;
} else {
break;
}
}
if (soughtPos == sought.GetSize()) {
return startThisPos;
}
}
return NOT_FOUND;
}

83
moses/ScoreComponentCollection.cpp
View File

@ -10,64 +10,63 @@ namespace Moses
{
void ScorePair::PlusEquals(const ScorePair &other)
{
PlusEquals(other.denseScores);
std::map<StringPiece, float>::const_iterator iter;
for (iter = other.sparseScores.begin(); iter != other.sparseScores.end(); ++iter) {
PlusEquals(iter->first, iter->second);
}
PlusEquals(other.denseScores);
std::map<StringPiece, float>::const_iterator iter;
for (iter = other.sparseScores.begin(); iter != other.sparseScores.end(); ++iter) {
PlusEquals(iter->first, iter->second);
}
}
void ScorePair::PlusEquals(const StringPiece &key, float value)
{
std::map<StringPiece, float>::iterator iter;
iter = sparseScores.find(key);
if (iter == sparseScores.end()) {
sparseScores[key] = value;
}
else {
float &existingval = iter->second;
existingval += value;
}
std::map<StringPiece, float>::iterator iter;
iter = sparseScores.find(key);
if (iter == sparseScores.end()) {
sparseScores[key] = value;
} else {
float &existingval = iter->second;
existingval += value;
}
}
std::ostream& operator<<(std::ostream& os, const ScorePair& rhs)
{
for (size_t i = 0; i < rhs.denseScores.size(); ++i) {
os << rhs.denseScores[i] << ",";
}
for (size_t i = 0; i < rhs.denseScores.size(); ++i) {
os << rhs.denseScores[i] << ",";
}
std::map<StringPiece, float>::const_iterator iter;
for (iter = rhs.sparseScores.begin(); iter != rhs.sparseScores.end(); ++iter) {
os << iter->first << "=" << iter->second << ",";
}
std::map<StringPiece, float>::const_iterator iter;
for (iter = rhs.sparseScores.begin(); iter != rhs.sparseScores.end(); ++iter) {
os << iter->first << "=" << iter->second << ",";
}
return os;
return os;
}
ScoreComponentCollection::ScoreIndexMap ScoreComponentCollection::s_scoreIndexes;
size_t ScoreComponentCollection::s_denseVectorSize = 0;
ScoreComponentCollection::
ScoreComponentCollection()
ScoreComponentCollection()
: m_scores(s_denseVectorSize)
{}
void
void
ScoreComponentCollection::
RegisterScoreProducer(const FeatureFunction* scoreProducer)
{
size_t start = s_denseVectorSize;
size_t end = start + scoreProducer->GetNumScoreComponents();
VERBOSE(1, "FeatureFunction: "
<< scoreProducer->GetScoreProducerDescription()
<< " start: " << start << " end: " << (end-1) << endl);
VERBOSE(1, "FeatureFunction: "
<< scoreProducer->GetScoreProducerDescription()
<< " start: " << start << " end: " << (end-1) << endl);
s_scoreIndexes[scoreProducer] = pair<size_t,size_t>(start,end);
s_denseVectorSize = end;
}
float
float
ScoreComponentCollection::
GetWeightedScore() const
{
@ -213,7 +212,7 @@ void ScoreComponentCollection::Save(const string& filename) const
out.close();
}
void
void
ScoreComponentCollection::
Assign(const FeatureFunction* sp, const string line)
{
@ -229,18 +228,18 @@ Assign(const FeatureFunction* sp, const string line)
}
}
void
void
ScoreComponentCollection::
Assign(const FeatureFunction* sp, const std::vector<float>& scores)
Assign(const FeatureFunction* sp, const std::vector<float>& scores)
{
IndexPair indexes = GetIndexes(sp);
size_t numScores = indexes.second - indexes.first;
if (scores.size() != numScores) {
UTIL_THROW(util::Exception, "Feature function "
<< sp->GetScoreProducerDescription() << " specified "
<< numScores << " dense scores or weights. Actually has "
<< scores.size());
UTIL_THROW(util::Exception, "Feature function "
<< sp->GetScoreProducerDescription() << " specified "
<< numScores << " dense scores or weights. Actually has "
<< scores.size());
}
for (size_t i = 0; i < scores.size(); ++i) {
@ -286,14 +285,14 @@ FVector ScoreComponentCollection::GetVectorForProducer(const FeatureFunction* sp
void ScoreComponentCollection::PlusEquals(const FeatureFunction* sp, const ScorePair &scorePair)
{
PlusEquals(sp, scorePair.denseScores);
PlusEquals(sp, scorePair.denseScores);
std::map<StringPiece, float>::const_iterator iter;
for (iter = scorePair.sparseScores.begin(); iter != scorePair.sparseScores.end(); ++iter) {
const StringPiece &key = iter->first;
float value = iter->second;
PlusEquals(sp, key, value);
}
std::map<StringPiece, float>::const_iterator iter;
for (iter = scorePair.sparseScores.begin(); iter != scorePair.sparseScores.end(); ++iter) {
const StringPiece &key = iter->first;
float value = iter->second;
PlusEquals(sp, key, value);
}
}
}

40
moses/ScoreComponentCollection.h
View File

@ -44,31 +44,29 @@ namespace Moses
/**
* Smaller version for just 1 FF.
*/
struct ScorePair
{
friend std::ostream& operator<<(std::ostream& os, const ScorePair& rhs);
struct ScorePair {
friend std::ostream& operator<<(std::ostream& os, const ScorePair& rhs);
std::vector<float> denseScores;
std::map<StringPiece, float> sparseScores;
std::vector<float> denseScores;
std::map<StringPiece, float> sparseScores;
ScorePair()
{}
ScorePair(const std::vector<float> &other)
:denseScores(other)
{}
ScorePair()
{}
ScorePair(const std::vector<float> &other)
:denseScores(other)
{}
void PlusEquals(const ScorePair &other);
void PlusEquals(const StringPiece &key, float value);
void PlusEquals(const ScorePair &other);
void PlusEquals(const StringPiece &key, float value);
void PlusEquals(const std::vector<float> &other)
{
CHECK(denseScores.size() == other.size());
std::transform(denseScores.begin(),
denseScores.end(),
other.begin(),
denseScores.begin(),
std::plus<float>());
}
void PlusEquals(const std::vector<float> &other) {
CHECK(denseScores.size() == other.size());
std::transform(denseScores.begin(),
denseScores.end(),
other.begin(),
denseScores.begin(),
std::plus<float>());
}
};
/*** An unweighted collection of scores for a translation or step in a translation.

12
moses/ScoreComponentCollectionTest.cpp
View File

@ -37,14 +37,14 @@ public:
void Evaluate(const Hypothesis&, ScoreComponentCollection*) const {}
void EvaluateChart(const ChartHypothesis&, ScoreComponentCollection*) const {}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

2
moses/SearchNormal.cpp
View File

@ -32,7 +32,7 @@ SearchNormal::SearchNormal(Manager& manager, const InputType &source, const Tran
for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind) {
HypothesisStackNormal *sourceHypoColl = new HypothesisStackNormal(m_manager);
sourceHypoColl->SetMaxHypoStackSize(staticData.GetMaxHypoStackSize(),
staticData.GetMinHypoStackDiversity());
staticData.GetMinHypoStackDiversity());
sourceHypoColl->SetBeamWidth(staticData.GetBeamWidth());
m_hypoStackColl[ind] = sourceHypoColl;

6
moses/SearchNormalBatch.cpp
View File

@ -109,10 +109,10 @@ void SearchNormalBatch::ProcessSentence()
* (base hypothesis score plus future score estimation)
*/
void
void
SearchNormalBatch::
ExpandHypothesis(const Hypothesis &hypothesis,
const TranslationOption &transOpt, float expectedScore)
ExpandHypothesis(const Hypothesis &hypothesis,
const TranslationOption &transOpt, float expectedScore)
{
// Check if the number of partial hypotheses exceeds the batch size.
if (m_partial_hypos.size() >= m_batch_size) {

36
moses/StaticData.cpp
View File

@ -171,8 +171,8 @@ bool StaticData::LoadData(Parameter *parameter)
if (m_parameter->GetParam("n-best-list").size() >= 2) {
m_nBestFilePath = m_parameter->GetParam("n-best-list")[0];
m_nBestSize = Scan<size_t>( m_parameter->GetParam("n-best-list")[1] );
m_onlyDistinctNBest=(m_parameter->GetParam("n-best-list").size()>2
&& m_parameter->GetParam("n-best-list")[2]=="distinct");
m_onlyDistinctNBest=(m_parameter->GetParam("n-best-list").size()>2
&& m_parameter->GetParam("n-best-list")[2]=="distinct");
} else if (m_parameter->GetParam("n-best-list").size() == 1) {
UserMessage::Add(string("wrong format for switch -n-best-list file size"));
return false;
@ -879,8 +879,8 @@ void StaticData::CleanUpAfterSentenceProcessing(const InputType& source) const
void StaticData::LoadFeatureFunctions()
{
const std::vector<FeatureFunction*> &ffs
= FeatureFunction::GetFeatureFunctions();
const std::vector<FeatureFunction*> &ffs
= FeatureFunction::GetFeatureFunctions();
std::vector<FeatureFunction*>::const_iterator iter;
for (iter = ffs.begin(); iter != ffs.end(); ++iter) {
FeatureFunction *ff = *iter;
@ -889,15 +889,15 @@ void StaticData::LoadFeatureFunctions()
if (PhraseDictionary *ffCast = dynamic_cast<PhraseDictionary*>(ff)) {
m_phraseDictionary.push_back(ffCast);
doLoad = false;
} else if (const GenerationDictionary *ffCast
= dynamic_cast<const GenerationDictionary*>(ff)) {
} else if (const GenerationDictionary *ffCast
= dynamic_cast<const GenerationDictionary*>(ff)) {
m_generationDictionary.push_back(ffCast);
} else if (WordPenaltyProducer *ffCast
= dynamic_cast<WordPenaltyProducer*>(ff)) {
} else if (WordPenaltyProducer *ffCast
= dynamic_cast<WordPenaltyProducer*>(ff)) {
CHECK(m_wpProducer == NULL); // max 1 feature;
m_wpProducer = ffCast;
} else if (UnknownWordPenaltyProducer *ffCast
= dynamic_cast<UnknownWordPenaltyProducer*>(ff)) {
} else if (UnknownWordPenaltyProducer *ffCast
= dynamic_cast<UnknownWordPenaltyProducer*>(ff)) {
CHECK(m_unknownWordPenaltyProducer == NULL); // max 1 feature;
m_unknownWordPenaltyProducer = ffCast;
} else if (const InputFeature *ffCast = dynamic_cast<const InputFeature*>(ff)) {
@ -994,12 +994,12 @@ bool StaticData::LoadAlternateWeightSettings()
vector<string> featureFunctionName = Tokenize(args[1], ",");
for(size_t k=0; k<featureFunctionName.size(); k++) {
// check if a valid nane
map<string,FeatureFunction*>::iterator ffLookUp
= nameToFF.find(featureFunctionName[k]);
map<string,FeatureFunction*>::iterator ffLookUp
= nameToFF.find(featureFunctionName[k]);
if (ffLookUp == nameToFF.end()) {
cerr << "ERROR: alternate weight setting " << currentId
<< " specifies to ignore feature function " << featureFunctionName[k]
<< " but there is no such feature function" << endl;
cerr << "ERROR: alternate weight setting " << currentId
<< " specifies to ignore feature function " << featureFunctionName[k]
<< " but there is no such feature function" << endl;
hasErrors = true;
} else {
m_weightSettingIgnoreFF[ currentId ].insert( featureFunctionName[k] );
@ -1027,9 +1027,9 @@ bool StaticData::LoadAlternateWeightSettings()
// check if a valid nane
map<string,FeatureFunction*>::iterator ffLookUp = nameToFF.find(name);
if (ffLookUp == nameToFF.end()) {
cerr << "ERROR: alternate weight setting " << currentId
<< " specifies weight(s) for " << name
<< " but there is no such feature function" << endl;
cerr << "ERROR: alternate weight setting " << currentId
<< " specifies weight(s) for " << name
<< " but there is no such feature function" << endl;
hasErrors = true;
} else {
m_weightSetting[ currentId ]->Assign( nameToFF[name], weights);

4
moses/StaticData.h
View File

@ -354,8 +354,8 @@ public:
}
bool IsDetailedAllTranslationReportingEnabled() const {
return !m_detailedAllTranslationReportingFilePath.empty();
}
return !m_detailedAllTranslationReportingFilePath.empty();
}
const std::string &GetDetailedTranslationReportingFilePath() const {
return m_detailedTranslationReportingFilePath;

4
moses/SyntacticLanguageModel.cpp
View File

@ -1,6 +1,6 @@
// This file should be compiled only when the HAVE_SYNLM flag is enabled.
// This file should be compiled only when the HAVE_SYNLM flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when HAVE_SYNLM is disabled.
//
#ifdef HAVE_SYNLM

33
moses/TargetPhrase.cpp
View File

@ -90,10 +90,9 @@ TargetPhrase::TargetPhrase(const TargetPhrase &copy)
}
if (copy.m_ruleSource) {
m_ruleSource = new Phrase(*copy.m_ruleSource);
}
else {
m_ruleSource = NULL;
m_ruleSource = new Phrase(*copy.m_ruleSource);
} else {
m_ruleSource = NULL;
}
}
@ -213,24 +212,24 @@ void TargetPhrase::Merge(const TargetPhrase &copy, const std::vector<FactorType>
void TargetPhrase::SetProperties(const StringPiece &str)
{
if (str.size() == 0) {
return;
}
if (str.size() == 0) {
return;
}
vector<string> toks;
TokenizeMultiCharSeparator(toks, str.as_string(), "{{");
for (size_t i = 0; i < toks.size(); ++i) {
string &tok = toks[i];
if (tok.empty()) {
continue;
}
size_t endPos = tok.rfind("}");
string &tok = toks[i];
if (tok.empty()) {
continue;
}
size_t endPos = tok.rfind("}");
tok = tok.substr(0, endPos - 1);
tok = tok.substr(0, endPos - 1);
vector<string> keyValue = TokenizeFirstOnly(tok, " ");
CHECK(keyValue.size() == 2);
SetProperty(keyValue[0], keyValue[1]);
vector<string> keyValue = TokenizeFirstOnly(tok, " ");
CHECK(keyValue.size() == 2);
SetProperty(keyValue[0], keyValue[1]);
}
}
@ -249,7 +248,7 @@ void TargetPhrase::GetProperty(const std::string &key, std::string &value, bool
void TargetPhrase::SetRuleSource(const Phrase &ruleSource) const
{
if (m_ruleSource == NULL) {
m_ruleSource = new Phrase(ruleSource);
m_ruleSource = new Phrase(ruleSource);
}
}

5
moses/TargetPhrase.h
View File

@ -124,8 +124,9 @@ public:
return *m_alignNonTerm;
}
const Phrase *GetRuleSource() const
{ return m_ruleSource; }
const Phrase *GetRuleSource() const {
return m_ruleSource;
}
// To be set by the FF that needs it, by default the rule source = NULL
// make a copy of the source side of the rule

8
moses/TargetPhraseCollection.cpp
View File

@ -29,14 +29,14 @@ namespace Moses
{
// helper for sort
bool
CompareTargetPhrase::
operator() (const TargetPhrase *a, const TargetPhrase *b) const
CompareTargetPhrase::
operator() (const TargetPhrase *a, const TargetPhrase *b) const
{
return a->GetFutureScore() > b->GetFutureScore();
}
bool
CompareTargetPhrase::
bool
CompareTargetPhrase::
operator() (const TargetPhrase &a, const TargetPhrase &b) const
{
return a.GetFutureScore() > b.GetFutureScore();

3
moses/TargetPhraseCollection.h
View File

@ -118,8 +118,7 @@ public:
void Add(TargetPhrase *targetPhrase, const Phrase &sourcePhrase);
};
struct CompareTargetPhrase
{
struct CompareTargetPhrase {
bool operator() (const TargetPhrase *a, const TargetPhrase *b) const;
bool operator() (const TargetPhrase &a, const TargetPhrase &b) const;
};

2
moses/TranslationModel/BilingualDynSuffixArray.h
View File

@ -80,7 +80,7 @@ class ScoresComp
{
public:
ScoresComp(const vector<float>& weights)
{}
{}
bool operator()(const Scores& s1, const Scores& s2) const {
return s1[0] < s2[0]; // just p(e|f) as approximation
// float score1(0), score2(0);

51
moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp
View File

@ -18,7 +18,6 @@
***********************************************************************/
#include <iostream>
#include <sstream>
#include "ChartRuleLookupManagerSkeleton.h"
#include "DotChartInMemory.h"
@ -40,8 +39,8 @@ ChartRuleLookupManagerSkeleton::ChartRuleLookupManagerSkeleton(
const ChartParser &parser,
const ChartCellCollectionBase &cellColl,
const SkeletonPT &skeletonPt)
: ChartRuleLookupManager(parser, cellColl)
, m_skeletonPT(skeletonPt)
: ChartRuleLookupManager(parser, cellColl)
, m_skeletonPT(skeletonPt)
{
}
@ -53,33 +52,43 @@ void ChartRuleLookupManagerSkeleton::GetChartRuleCollection(
const WordsRange &range,
ChartParserCallback &outColl)
{
const ChartCellLabel &sourceWordLabel = GetSourceAt(range.GetStartPos());
const Word &sourceWord = sourceWordLabel.GetLabel();
const ChartCellLabel &sourceWordLabel = GetSourceAt(range.GetStartPos());
const Word &sourceWord = sourceWordLabel.GetLabel();
TargetPhrase *tp = CreateTargetPhrase(sourceWord);
TargetPhraseCollection *tpColl = new TargetPhraseCollection();
tpColl->Add(tp);
// almost the same as for SkeletonPT::GetTargetPhraseCollectionBatch()
TargetPhraseCollection *tpColl = new TargetPhraseCollection();
outColl.Add(*tpColl, m_stackVec, range);
if (range.GetStartPos() > 0) {
TargetPhrase *tp = CreateTargetPhrase(sourceWord);
tpColl->Add(tp);
}
outColl.Add(*tpColl, m_stackVec, range);
}
TargetPhrase *ChartRuleLookupManagerSkeleton::CreateTargetPhrase(const Word &sourceWord) const
{
// create a target phrase from the 1st word of the source, prefix with 'ChartManagerSkeleton:'
string str = sourceWord.GetFactor(0)->GetString().as_string();
str = "ChartManagerSkeleton:" + str;
// create a target phrase from the 1st word of the source, prefix with 'ChartManagerSkeleton:'
string str = sourceWord.GetFactor(0)->GetString().as_string();
str = "ChartManagerSkeleton:" + str;
cerr << "str=" << str << endl;
TargetPhrase *tp = new TargetPhrase();
Word &word = tp->AddWord();
word.CreateFromString(Output, m_skeletonPT.GetOutput(), str, false);
TargetPhrase *tp = new TargetPhrase();
Word &word = tp->AddWord();
word.CreateFromString(Output, m_skeletonPT.GetOutput(), str, false);
Word *targetLHS = new Word();
targetLHS->CreateFromString(Output, m_skeletonPT.GetOutput(), "[X]", true);
tp->SetTargetLHS(targetLHS);
// score for this phrase table
vector<float> scores(m_skeletonPT.GetNumScoreComponents(), 1.3);
tp->GetScoreBreakdown().PlusEquals(&m_skeletonPT, scores);
tp->SetAlignmentInfo("0-0");
// score of all other ff when this rule is being loaded
//tp->Evaluate(sourcePhrase, m_skeletonPT.GetFeaturesToApply());
// score for this phrase table
vector<float> scores(m_skeletonPT.GetNumScoreComponents(), 1.3);
tp->GetScoreBreakdown().PlusEquals(&m_skeletonPT, scores);
return tp;
// score of all other ff when this rule is being loaded
//tp->Evaluate(sourcePhrase, m_skeletonPT.GetFeaturesToApply());
return tp;
}
} // namespace Moses

4
moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h
View File

@ -35,8 +35,8 @@ class ChartRuleLookupManagerSkeleton : public ChartRuleLookupManager
{
public:
ChartRuleLookupManagerSkeleton(const ChartParser &parser,
const ChartCellCollectionBase &cellColl,
const SkeletonPT &skeletonPt);
const ChartCellCollectionBase &cellColl,
const SkeletonPT &skeletonPt);
~ChartRuleLookupManagerSkeleton();

14
moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp
View File

@ -100,13 +100,13 @@ void PhraseDictionaryCompact::Load()
CHECK(indexSize && coderSize && phraseSize);
}
// now properly declared in TargetPhraseCollection.h
// and defined in TargetPhraseCollection.cpp
// struct CompareTargetPhrase {
// bool operator() (const TargetPhrase &a, const TargetPhrase &b) {
// return a.GetFutureScore() > b.GetFutureScore();
// }
// };
// now properly declared in TargetPhraseCollection.h
// and defined in TargetPhraseCollection.cpp
// struct CompareTargetPhrase {
// bool operator() (const TargetPhrase &a, const TargetPhrase &b) {
// return a.GetFutureScore() > b.GetFutureScore();
// }
// };
const TargetPhraseCollection*
PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &sourcePhrase) const

8
moses/TranslationModel/PhraseDictionary.cpp
View File

@ -83,14 +83,14 @@ GetTargetPhraseCollectionNonCacheLEGACY(const Phrase& src) const
}
TargetPhraseCollectionWithSourcePhrase const*
TargetPhraseCollectionWithSourcePhrase const*
PhraseDictionary::
GetTargetPhraseCollectionLEGACY(InputType const& src,WordsRange const& range) const
{
UTIL_THROW(util::Exception, "Legacy method not implemented");
}
void
void
PhraseDictionary::
SetParameter(const std::string& key, const std::string& value)
{
@ -105,7 +105,7 @@ SetParameter(const std::string& key, const std::string& value)
}
}
void
void
PhraseDictionary::
SetFeaturesToApply()
{
@ -119,7 +119,7 @@ SetFeaturesToApply()
}
}
void
void
PhraseDictionary::
GetTargetPhraseCollectionBatch(const InputPathList &phraseDictionaryQueue) const
{

8
moses/TranslationModel/PhraseDictionary.h
View File

@ -69,17 +69,17 @@ public:
return m_tableLimit;
}
// LEGACY!
// LEGACY!
// The preferred method is to override GetTargetPhraseCollectionBatch().
// See class PhraseDictionaryMemory or PhraseDictionaryOnDisk for details
//! find list of translations that can translates src. Only for phrase input
virtual
virtual
TargetPhraseCollection const *
GetTargetPhraseCollectionLEGACY(const Phrase& src) const;
virtual
void
virtual
void
GetTargetPhraseCollectionBatch(const InputPathList &phraseDictionaryQueue) const;
//! Create entry for translation of source to targetPhrase

4
moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp
View File

@ -17,7 +17,7 @@ PhraseDictionaryDynSuffixArray(const std::string &line)
}
void
void
PhraseDictionaryDynSuffixArray::
Load()
{
@ -33,7 +33,7 @@ PhraseDictionaryDynSuffixArray::
delete m_biSA;
}
void
void
PhraseDictionaryDynSuffixArray::
SetParameter(const std::string& key, const std::string& value)
{

2
moses/TranslationModel/PhraseDictionaryMemory.cpp
View File

@ -133,7 +133,7 @@ void PhraseDictionaryMemory::SortAndPrune()
}
}
void
void
PhraseDictionaryMemory::
GetTargetPhraseCollectionBatch(const InputPathList &phraseDictionaryQueue) const
{

32
moses/TranslationModel/SkeletonPT.cpp
View File

@ -14,7 +14,7 @@ SkeletonPT::SkeletonPT(const std::string &line)
void SkeletonPT::CleanUpAfterSentenceProcessing(const InputType& source)
{
RemoveAllInColl(m_allTPColl);
RemoveAllInColl(m_allTPColl);
}
void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &phraseDictionaryQueue) const
@ -34,29 +34,29 @@ void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &phraseDicti
TargetPhrase *SkeletonPT::CreateTargetPhrase(const Phrase &sourcePhrase) const
{
// create a target phrase from the 1st word of the source, prefix with 'SkeletonPT:'
CHECK(sourcePhrase.GetSize());
CHECK(m_output.size() == 1);
// create a target phrase from the 1st word of the source, prefix with 'SkeletonPT:'
CHECK(sourcePhrase.GetSize());
CHECK(m_output.size() == 1);
string str = sourcePhrase.GetWord(0).GetFactor(0)->GetString().as_string();
str = "SkeletonPT:" + str;
string str = sourcePhrase.GetWord(0).GetFactor(0)->GetString().as_string();
str = "SkeletonPT:" + str;
TargetPhrase *tp = new TargetPhrase();
Word &word = tp->AddWord();
word.CreateFromString(Output, m_output, str, false);
TargetPhrase *tp = new TargetPhrase();
Word &word = tp->AddWord();
word.CreateFromString(Output, m_output, str, false);
// score for this phrase table
vector<float> scores(m_numScoreComponents, 1.3);
tp->GetScoreBreakdown().PlusEquals(this, scores);
// score for this phrase table
vector<float> scores(m_numScoreComponents, 1.3);
tp->GetScoreBreakdown().PlusEquals(this, scores);
// score of all other ff when this rule is being loaded
tp->Evaluate(sourcePhrase, GetFeaturesToApply());
// score of all other ff when this rule is being loaded
tp->Evaluate(sourcePhrase, GetFeaturesToApply());
return tp;
return tp;
}
ChartRuleLookupManager* SkeletonPT::CreateRuleLookupManager(const ChartParser &parser,
const ChartCellCollectionBase &cellCollection)
const ChartCellCollectionBase &cellCollection)
{
return new ChartRuleLookupManagerSkeleton(parser, cellCollection, *this);
}

6
moses/TranslationOptionCollection.cpp
View File

@ -202,9 +202,9 @@ void TranslationOptionCollection::ProcessUnknownWord()
* \param inputScores a set of scores associated with unknown word (input scores from latties/CNs)
*/
void TranslationOptionCollection::ProcessOneUnknownWord(const InputPath &inputPath,
size_t sourcePos,
size_t length,
const ScorePair *inputScores)
size_t sourcePos,
size_t length,
const ScorePair *inputScores)
{
const StaticData &staticData = StaticData::Instance();
const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = staticData.GetUnknownWordPenaltyProducer();

2
moses/WordLattice.cpp
View File

@ -9,7 +9,7 @@
namespace Moses
{
WordLattice::WordLattice()
WordLattice::WordLattice()
{
CHECK(StaticData::Instance().GetInputFeature());
}

2
phrase-extract/PhraseExtractionOptions.h
View File

@ -109,7 +109,7 @@ public:
void initInstanceWeightsFile(const char* initInstanceWeightsFile) {
instanceWeightsFile = std::string(initInstanceWeightsFile);
}
void initFlexScoreFlag(const bool initflexScoreFlag){
void initFlexScoreFlag(const bool initflexScoreFlag) {
flexScoreFlag=initflexScoreFlag;
}

16
phrase-extract/SentenceAlignment.cpp
View File

@ -124,15 +124,15 @@ bool SentenceAlignment::create( char targetString[], char sourceString[], char a
void SentenceAlignment::invertAlignment()
{
alignedToS.resize(source.size());
for (size_t targetPos = 0; targetPos < alignedToT.size(); ++targetPos) {
const std::vector<int> &vec = alignedToT[targetPos];
for (size_t i = 0; i < vec.size(); ++i) {
int sourcePos = vec[i];
alignedToS[sourcePos].push_back(targetPos);
}
alignedToS.resize(source.size());
for (size_t targetPos = 0; targetPos < alignedToT.size(); ++targetPos) {
const std::vector<int> &vec = alignedToT[targetPos];
for (size_t i = 0; i < vec.size(); ++i) {
int sourcePos = vec[i];
alignedToS[sourcePos].push_back(targetPos);
}
}
}
}
}

33
phrase-extract/extract-ghkm/Alignment.cpp
View File

@ -29,47 +29,36 @@ namespace Moses
namespace GHKM
{
Alignment ReadAlignment(const std::string &s)
void ReadAlignment(const std::string &s, Alignment &a)
{
Alignment a;
const std::string digits = "0123456789";
std::string::size_type begin = s.find_first_of(digits);
if (begin == std::string::npos) {
// Empty word alignments are allowed
return a;
}
a.clear();
std::string::size_type begin = 0;
while (true) {
std::string::size_type end = s.find("-", begin);
if (end == std::string::npos) {
throw Exception("Alignment separator '-' missing");
return;
}
int src = std::atoi(s.substr(begin, end-begin).c_str());
begin = s.find_first_of(digits, end);
if (begin == std::string::npos) {
if (end+1 == s.size()) {
throw Exception("Target index missing");
}
end = s.find(" ", begin);
begin = end+1;
end = s.find_first_not_of(digits, begin+1);
int tgt;
if (end == std::string::npos) {
tgt = std::atoi(s.substr(begin).c_str());
a.push_back(std::make_pair(src, tgt));
return;
} else {
tgt = std::atoi(s.substr(begin, end-begin).c_str());
a.push_back(std::make_pair(src, tgt));
}
a.push_back(std::make_pair(src, tgt));
if (end == std::string::npos) {
break;
}
begin = s.find_first_of(digits, end);
begin = end+1;
}
return a;
}
} // namespace GHKM

2
phrase-extract/extract-ghkm/Alignment.h
View File

@ -32,7 +32,7 @@ namespace GHKM
typedef std::vector<std::pair<int, int> > Alignment;
Alignment ReadAlignment(const std::string &);
void ReadAlignment(const std::string &, Alignment &);
} // namespace GHKM
} // namespace Moses

4
phrase-extract/extract-ghkm/ExtractGHKM.cpp
View File

@ -90,6 +90,7 @@ int ExtractGHKM::Main(int argc, char *argv[])
std::string targetLine;
std::string sourceLine;
std::string alignmentLine;
Alignment alignment;
XmlTreeParser xmlTreeParser(labelSet, topLabelSet);
ScfgRuleWriter writer(fwdExtractStream, invExtractStream, options);
size_t lineNum = options.sentenceOffset;
@ -130,9 +131,8 @@ int ExtractGHKM::Main(int argc, char *argv[])
std::vector<std::string> sourceTokens(ReadTokens(sourceLine));
// Read word alignments.
Alignment alignment;
try {
alignment = ReadAlignment(alignmentLine);
ReadAlignment(alignmentLine, alignment);
} catch (const Exception &e) {
std::ostringstream s;
s << "Failed to read alignment at line " << lineNum << ": ";

38
phrase-extract/extract-main.cpp
View File

@ -300,7 +300,7 @@ int main(int argc, char* argv[])
}
if (sentence.create( englishString, foreignString, alignmentString, weightString, i, false)) {
if (options.placeholders.size()) {
sentence.invertAlignment();
sentence.invertAlignment();
}
ExtractTask *task = new ExtractTask(i-1, sentence, options, extractFile , extractFileInv, extractFileOrientation, extractFileContext, extractFileContextInv);
task->Run();
@ -863,10 +863,10 @@ void ExtractTask::writePhrasesToFile()
for(vector<string>::const_iterator phrase=m_extractedPhrasesOri.begin(); phrase!=m_extractedPhrasesOri.end(); phrase++) {
outextractFileOrientation<<phrase->data();
}
for(vector<string>::const_iterator phrase=m_extractedPhrasesContext.begin();phrase!=m_extractedPhrasesContext.end();phrase++){
for(vector<string>::const_iterator phrase=m_extractedPhrasesContext.begin(); phrase!=m_extractedPhrasesContext.end(); phrase++) {
outextractFileContext<<phrase->data();
}
for(vector<string>::const_iterator phrase=m_extractedPhrasesContextInv.begin();phrase!=m_extractedPhrasesContextInv.end();phrase++){
for(vector<string>::const_iterator phrase=m_extractedPhrasesContextInv.begin(); phrase!=m_extractedPhrasesContextInv.end(); phrase++) {
outextractFileContextInv<<phrase->data();
}
@ -921,15 +921,14 @@ bool ExtractTask::checkPlaceholders (const SentenceAlignment &sentence, int star
const string &sourceWord = sentence.source[pos];
if (isPlaceholder(sourceWord)) {
if (sentence.alignedToS.at(pos).size() != 1) {
return false;
}
else {
// check it actually lines up to another placeholder
int targetPos = sentence.alignedToS.at(pos).at(0);
const string &otherWord = sentence.target[targetPos];
if (!isPlaceholder(otherWord)) {
return false;
}
return false;
} else {
// check it actually lines up to another placeholder
int targetPos = sentence.alignedToS.at(pos).at(0);
const string &otherWord = sentence.target[targetPos];
if (!isPlaceholder(otherWord)) {
return false;
}
}
}
}
@ -939,14 +938,13 @@ bool ExtractTask::checkPlaceholders (const SentenceAlignment &sentence, int star
if (isPlaceholder(targetWord)) {
if (sentence.alignedToT.at(pos).size() != 1) {
return false;
}
else {
// check it actually lines up to another placeholder
int sourcePos = sentence.alignedToT.at(pos).at(0);
const string &otherWord = sentence.source[sourcePos];
if (!isPlaceholder(otherWord)) {
return false;
}
} else {
// check it actually lines up to another placeholder
int sourcePos = sentence.alignedToT.at(pos).at(0);
const string &otherWord = sentence.source[sourcePos];
if (!isPlaceholder(otherWord)) {
return false;
}
}
}
}

125
phrase-extract/extract-rules-main.cpp
View File

@ -315,8 +315,8 @@ int main(int argc, char* argv[])
string fileNameExtractContext = fileNameExtract + ".context" + (options.gzOutput?".gz":"");
extractFileContext.Open(fileNameExtractContext.c_str());
if (!options.onlyDirectFlag) {
string fileNameExtractContextInv = fileNameExtract + ".context.inv" + (options.gzOutput?".gz":"");
extractFileContextInv.Open(fileNameExtractContextInv.c_str());
string fileNameExtractContextInv = fileNameExtract + ".context.inv" + (options.gzOutput?".gz":"");
extractFileContextInv.Open(fileNameExtractContextInv.c_str());
}
}
@ -716,44 +716,43 @@ void ExtractTask::saveHieroPhrase( int startT, int endT, int startS, int endS
// context (words to left and right)
if (m_options.flexScoreFlag) {
rule.sourceContextLeft = startS == 0 ? "<s>" : m_sentence.source[startS-1];
rule.sourceContextRight = endS+1 == m_sentence.source.size() ? "<s>" : m_sentence.source[endS+1];
rule.targetContextLeft = startT == 0 ? "<s>" : m_sentence.target[startT-1];
rule.targetContextRight = endT+1 == m_sentence.target.size() ? "<s>" : m_sentence.target[endT+1];
rule.sourceHoleString = "";
rule.targetHoleString = "";
rule.sourceContextLeft = startS == 0 ? "<s>" : m_sentence.source[startS-1];
rule.sourceContextRight = endS+1 == m_sentence.source.size() ? "<s>" : m_sentence.source[endS+1];
rule.targetContextLeft = startT == 0 ? "<s>" : m_sentence.target[startT-1];
rule.targetContextRight = endT+1 == m_sentence.target.size() ? "<s>" : m_sentence.target[endT+1];
rule.sourceHoleString = "";
rule.targetHoleString = "";
HoleList::const_iterator iterHole;
for (iterHole = holeColl.GetHoles().begin(); iterHole != holeColl.GetHoles().end(); ++iterHole) {
const Hole &hole = *iterHole;
rule.sourceHoleString += hole.GetLabel(0) + ": ";
HoleList::const_iterator iterHole;
for (iterHole = holeColl.GetHoles().begin(); iterHole != holeColl.GetHoles().end(); ++iterHole) {
const Hole &hole = *iterHole;
rule.sourceHoleString += hole.GetLabel(0) + ": ";
// rule starts with nonterminal; end of NT is considered left context
if (hole.GetStart(0) == startS) {
rule.sourceContextLeft = m_sentence.source[hole.GetEnd(0)];
}
// rule ends with nonterminal; start of NT is considered right context
else if (hole.GetEnd(0) == endS) {
rule.sourceContextRight = m_sentence.source[hole.GetStart(0)];
}
if (hole.GetStart(1) == startT) {
rule.targetContextLeft = m_sentence.target[hole.GetEnd(1)];
}
else if (hole.GetEnd(1) == endT) {
rule.targetContextRight = m_sentence.target[hole.GetStart(1)];
}
for (int i = hole.GetStart(0); i <= hole.GetEnd(0); ++i) {
rule.sourceHoleString += m_sentence.source[i] + " ";
}
rule.targetHoleString += hole.GetLabel(1) + ": ";
for (int i = hole.GetStart(1); i <= hole.GetEnd(1); ++i) {
rule.targetHoleString += m_sentence.target[i] + " ";
}
// rule starts with nonterminal; end of NT is considered left context
if (hole.GetStart(0) == startS) {
rule.sourceContextLeft = m_sentence.source[hole.GetEnd(0)];
}
// rule ends with nonterminal; start of NT is considered right context
else if (hole.GetEnd(0) == endS) {
rule.sourceContextRight = m_sentence.source[hole.GetStart(0)];
}
if (hole.GetStart(1) == startT) {
rule.targetContextLeft = m_sentence.target[hole.GetEnd(1)];
} else if (hole.GetEnd(1) == endT) {
rule.targetContextRight = m_sentence.target[hole.GetStart(1)];
}
for (int i = hole.GetStart(0); i <= hole.GetEnd(0); ++i) {
rule.sourceHoleString += m_sentence.source[i] + " ";
}
rule.targetHoleString += hole.GetLabel(1) + ": ";
for (int i = hole.GetStart(1); i <= hole.GetEnd(1); ++i) {
rule.targetHoleString += m_sentence.target[i] + " ";
}
}
}
addRuleToCollection( rule );
}
@ -996,10 +995,10 @@ void ExtractTask::addRule( int startT, int endT, int startS, int endS, int count
// context (words to left and right)
if (m_options.flexScoreFlag) {
rule.sourceContextLeft = startS == 0 ? "<s>" : m_sentence.source[startS-1];
rule.sourceContextRight = endS+1 == m_sentence.source.size() ? "<s>" : m_sentence.source[endS+1];
rule.targetContextLeft = startT == 0 ? "<s>" : m_sentence.target[startT-1];
rule.targetContextRight = endT+1 == m_sentence.target.size() ? "<s>" : m_sentence.target[endT+1];
rule.sourceContextLeft = startS == 0 ? "<s>" : m_sentence.source[startS-1];
rule.sourceContextRight = endS+1 == m_sentence.source.size() ? "<s>" : m_sentence.source[endS+1];
rule.targetContextLeft = startT == 0 ? "<s>" : m_sentence.target[startT-1];
rule.targetContextRight = endT+1 == m_sentence.target.size() ? "<s>" : m_sentence.target[endT+1];
}
rule.alignment.erase(rule.alignment.size()-1);
@ -1084,33 +1083,33 @@ void ExtractTask::writeRulesToFile()
}
if (m_options.flexScoreFlag) {
for(int iContext=0;iContext<2;iContext++){
outContext << rule->source << " ||| "
<< rule->target << " ||| "
<< rule->alignment << " ||| ";
iContext ? outContext << "< " << rule->sourceContextLeft << "\n" : outContext << "> " << rule->sourceContextRight << "\n";
for(int iContext=0; iContext<2; iContext++) {
outContext << rule->source << " ||| "
<< rule->target << " ||| "
<< rule->alignment << " ||| ";
iContext ? outContext << "< " << rule->sourceContextLeft << "\n" : outContext << "> " << rule->sourceContextRight << "\n";
if (!m_options.onlyDirectFlag) {
outContextInv << rule->target << " ||| "
<< rule->source << " ||| "
<< rule->alignmentInv << " ||| ";
iContext ? outContextInv << "< " << rule->targetContextLeft << "\n" : outContextInv << "> " << rule->targetContextRight << "\n";
}
if (!m_options.onlyDirectFlag) {
outContextInv << rule->target << " ||| "
<< rule->source << " ||| "
<< rule->alignmentInv << " ||| ";
iContext ? outContextInv << "< " << rule->targetContextLeft << "\n" : outContextInv << "> " << rule->targetContextRight << "\n";
}
}
if (rule->sourceHoleString != "") {
outContext << rule->source << " ||| "
<< rule->target << " ||| "
<< rule->alignment << " ||| v "
<< rule->sourceHoleString << "\n";
}
if (rule->sourceHoleString != "") {
outContext << rule->source << " ||| "
<< rule->target << " ||| "
<< rule->alignment << " ||| v "
<< rule->sourceHoleString << "\n";
}
if (!m_options.onlyDirectFlag and rule->targetHoleString != "") {
outContextInv << rule->target << " ||| "
<< rule->source << " ||| "
<< rule->alignmentInv << " ||| v "
<< rule->targetHoleString << "\n";
}
if (!m_options.onlyDirectFlag and rule->targetHoleString != "") {
outContextInv << rule->target << " ||| "
<< rule->source << " ||| "
<< rule->alignmentInv << " ||| v "
<< rule->targetHoleString << "\n";
}
}
}
m_extractFile << out.str();

3
scripts/other/beautify.perl
View File

@ -12,7 +12,7 @@ chomp($version);
print STDERR "version=$version";
if ($version ne "Artistic Style Version 2.01") {
print STDERR "Must be astyle version 2.01. Quitting\n";
print STDERR "\nMust be astyle version 2.01. Quitting\n";
exit(1);
}
@ -39,6 +39,7 @@ sub Beautify($)
next if ($name eq "randlm");
next if ($name eq "srilm");
next if ($name eq "irstlm");
next if ($name eq "UG");
$name = $path ."/" .$name;
if (-d $name) {

6
scripts/training/train-model.perl
View File

@ -2011,11 +2011,11 @@ sub create_ini {
my ($factor_f,$factor_e) = split(/\-/,$factor_val);
if($count == 0){
$feature_spec .= "OpSequenceModel num-features=5 path=". $_OSM . $factor_val . "/operationLM.bin" . " sFactor=". $factor_f . " tFactor=". $factor_e . " numFeatures=5 \n";
$feature_spec .= "OpSequenceModel name=OpSequenceModel$count num-features=5 path=". $_OSM . $factor_val . "/operationLM.bin" . " input-factor=". $factor_f . " output-factor=". $factor_e . " support-features=yes \n";
$weight_spec .= "OpSequenceModel$count= 0.08 -0.02 0.02 -0.001 0.03\n";
}
else{
$feature_spec .= "OpSequenceModel num-features=1 path=". $_OSM . $factor_val . "/operationLM.bin" . " sFactor=". $factor_f . " tFactor=". $factor_e . " numFeatures=1 \n";
$feature_spec .= "OpSequenceModel name=OpSequenceModel$count num-features=1 path=". $_OSM . $factor_val . "/operationLM.bin" . " input-factor=". $factor_f . " output-factor=". $factor_e . " support-features=no \n";
$weight_spec .= "OpSequenceModel$count= 0.08 \n";
}
@ -2024,7 +2024,7 @@ sub create_ini {
}
else
{
$feature_spec .= "OpSequenceModel num-features=5 path=". $_OSM . " \n";
$feature_spec .= "OpSequenceModel name=OpSequenceModel0 num-features=5 path=". $_OSM . " \n";
$weight_spec .= "OpSequenceModel0= 0.08 -0.02 0.02 -0.001 0.03\n";
}
}