moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into GHKMStruct

Browse Source 
Conflicts:
	moses-chart-cmd/IOWrapper.cpp
	moses-chart-cmd/IOWrapper.h
	moses/FF/Factory.cpp
	moses/Parameter.cpp
	moses/StaticData.h
	phrase-extract/extract-ghkm/ScfgRuleWriter.cpp
	phrase-extract/score-main.cpp
This commit is contained in:
Phil Williams 2013-09-29 15:27:09 +01:00
commit 2a28d1a73e
326 changed files with 31287 additions and 1046 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
*.d
*.pyc
*.lo
*.o

23
Jamroot
View File

@ -135,7 +135,28 @@ project : requirements
#Add directories here if you want their incidental targets too (i.e. tests).
build-projects lm util phrase-extract search moses moses/LM mert moses-cmd moses-chart-cmd mira scripts regression-testing ;
alias programs : lm//programs moses-chart-cmd//moses_chart moses-cmd//programs OnDiskPt//CreateOnDiskPt OnDiskPt//queryOnDiskPt mert//programs misc//programs symal phrase-extract phrase-extract//lexical-reordering phrase-extract//extract-ghkm phrase-extract//pcfg-extract phrase-extract//pcfg-score biconcor mira//mira contrib/server//mosesserver ;
alias programs :
lm//programs
moses-chart-cmd//moses_chart
moses-cmd//programs
OnDiskPt//CreateOnDiskPt
OnDiskPt//queryOnDiskPt
mert//programs
misc//programs
symal
phrase-extract
phrase-extract//lexical-reordering
phrase-extract//extract-ghkm
phrase-extract//pcfg-extract
phrase-extract//pcfg-score
biconcor
mira//mira
contrib/server//mosesserver
#moses/mm//mtt-build
#moses/mm//mtt-dump
#moses/mm//symal2mam
#moses/mm//custom-pt
;
install-bin-libs programs ;
install-headers headers-base : [ path.glob-tree biconcor contrib lm mert misc moses-chart-cmd moses-cmd OnDiskPt phrase-extract symal util : *.hh *.h ] : . ;

2
OnDiskPt/TargetPhrase.cpp
View File

@ -278,7 +278,7 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
// scores
ret->GetScoreBreakdown().Assign(&phraseDict, m_scores);
ret->Evaluate(mosesSP);
ret->Evaluate(mosesSP, phraseDict.GetFeaturesToApply());
return ret;
}

9
contrib/m4m/README
View File

@ -14,4 +14,11 @@ You'll need the following additional tools (in your PATH):
* BEAR WITH ME WHILE BUGS ARE BEING FIXED AND FUNCTIONALITY IS ADDED. *
******************************************************************************
- Ulrich Germann
KNOWN ISSUES:
- giza phrase table creation sometimes fails when make is run in
multi-threaded mode. Running make without the -j switch seems to work.
Ulrich Germann

2
contrib/m4m/modules/m4m.m4m
View File

@ -10,7 +10,7 @@ m4mdir := $(patsubst %modules/,%,\
M4M_MODULES := aux init
#M4M_MODULES += directory-structure
M4M_MODULES += tools moses-parameters prepare-corpus
M4M_MODULES += mgiza fastalign phrase-table moses-ini
M4M_MODULES += mgiza fastalign mmbitext phrase-table moses-ini
M4M_MODULES += tune-moses eval-system kenlm
NUMCORES = $(shell parallel --number-of-cores)

35
contrib/m4m/modules/mmbitext.m4m Normal file
View File

@ -0,0 +1,35 @@
# -*- makefile -*-
# Makefile for building a Moses system from a word-aligned corpus
# (c) 2011 - 2012 Ulrich Germann
define mmap_ttrack
.INTERMEDIATE += $(strip $1).txt.gz
$2/$(notdir $1).mct: | $2/$(notdir $1).sfa
$2/$(notdir $1).tdx: | $2/$(notdir $1).sfa
$2/$(notdir $1).sfa: | $(strip $1).txt.gz
$$(lock)
zcat -f $$< | ${MOSES_BIN}/mtt-build -i -o $$@.lock/$$(basename $${@F})
mv $$@.lock/$$(basename $${@F}).tdx $${@D}
mv $$@.lock/$$(basename $${@F}).sfa $${@D}
mv $$@.lock/$$(basename $${@F}).mct $${@D}
$$(unlock)
endef
# $1: base name of word-aligned corpus in text format
# $2: directory for mmapped bitext
define mmap_bitext
$(call mmap_ttrack,$1${L1},$2)
$(call mmap_ttrack,$1${L2},$2)
$2/$(notdir $1)${L1}-${L2}.mam: | $(strip $1)${L1}-${L2}.symal.gz
$$(lock)
zcat -f $$< | ${MOSES_BIN}/symal2mam $$@_ && mv $$@_ $$@
$$(unlock)
.INTERMEDIATE += $(strip $1)${L1}-${L2}.symal.gz
endef

17
contrib/m4m/modules/phrase-table.m4m
View File

@ -270,3 +270,20 @@ PTABLES += $(strip $4) $(strip $5) $(strip $6)
endef
#################################################################################
define add_mmsapt
$(call mmap_bitext,$(strip $5),$(strip $4))
ffname := PT$(words ${PTABLE_ENTRIES})
MY_ENTRY := Mmsapt
MY_ENTRY += name=$$(ffname)
MY_ETNRY += input-factor=$(strip $1)
MY_ENTRY += output-factor=$(strip $2)
MY_ENTRY += num-features=$(strip $3)
MY_ENTRY += base=$(abspath $4)/ L1=${L1} L2=${L2}
PTABLE_ENTRIES += $$(subst $$(space),;,$${MY_ENTRY})
MOSES_INI_PREREQ += $(addprefix $(strip $4),${L1}.mct ${L1}.tdx ${L1}.sfa)
MOSES_INI_PREREQ += $(addprefix $(strip $4),${L2}.mct ${L2}.tdx ${L2}.sfa)
MOSES_INI_PREREQ += $(strip $4)${L1}-${L2}.mam
endef

5
contrib/other-builds/extract-rules/.project
View File

@ -25,11 +25,6 @@
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
</natures>
<linkedResources>
<link>
<name>ExtractedRule.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/ExtractedRule.cpp</locationURI>
</link>
<link>
<name>ExtractedRule.h</name>
<type>1</type>

40
contrib/other-builds/moses/.project
View File

@ -1036,6 +1036,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/BleuScoreFeature.h</locationURI>
</link>
<link>
<name>FF/ConstrainedDecoding.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ConstrainedDecoding.cpp</locationURI>
</link>
<link>
<name>FF/ConstrainedDecoding.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ConstrainedDecoding.h</locationURI>
</link>
<link>
<name>FF/ControlRecombination.cpp</name>
<type>1</type>
@ -1481,6 +1491,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/SingleFactor.h</locationURI>
</link>
<link>
<name>LM/SkeletonLM.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/SkeletonLM.cpp</locationURI>
</link>
<link>
<name>LM/SkeletonLM.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/SkeletonLM.h</locationURI>
</link>
<link>
<name>LM/backward.arpa</name>
<type>1</type>
@ -1611,6 +1631,16 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/SkeletonPT.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/SkeletonPT.cpp</locationURI>
</link>
<link>
<name>TranslationModel/SkeletonPT.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/SkeletonPT.h</locationURI>
</link>
<link>
<name>TranslationModel/WordCoocTable.cpp</name>
<type>1</type>
@ -1736,6 +1766,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/DotChart.h</name>
<type>1</type>

0
contrib/server/client_multimodel.py Normal file → Executable file
View File

2
contrib/server/mosesserver.cpp
View File

@ -325,7 +325,7 @@ public:
void outputChartHypo(ostream& out, const ChartHypothesis* hypo) {
Phrase outPhrase(20);
hypo->CreateOutputPhrase(outPhrase);
hypo->GetOutputPhrase(outPhrase);
// delete 1st & last
assert(outPhrase.GetSize() >= 2);

4
jam-files/sanity.jam
View File

@ -290,6 +290,10 @@ rule failure-message ( ok ? ) {
echo "The build failed. If you need support, run:" ;
echo " $(args) --debug-configuration -d2 >build.log" ;
echo "then attach build.log to your e-mail." ;
echo "You MUST do 3 things before sending to the mailing list:" ;
echo " 1. Subscribe to the mailing list at http://mailman.mit.edu/mailman/listinfo/moses-support" ;
echo " 2. Zip up your build.log file before attaching it to the email" ;
echo " 3. Say what is the EXACT command you executed when you got the error" ;
}
echo "ERROR" ;
} else {

47
scripts/OSM/generateSequences.cpp → misc/GenerateSequences.cpp
View File

@ -6,6 +6,7 @@
#include <set>
#include <cstdlib>
using namespace std;
@ -22,16 +23,11 @@ void loadInput(const char * fileName, vector <string> & input)
{
ifstream sr (fileName);
char* tmp;
string line;
if(sr.is_open()) {
while(! sr.eof() ) {
tmp= new char[5000];
sr.getline (tmp,5000);
input.push_back(tmp);
//cout<<tmp<<input.size()<<endl;
delete [] tmp;
while(getline(sr , line )) {
input.push_back(line);
}
sr.close();
@ -42,37 +38,20 @@ void loadInput(const char * fileName, vector <string> & input)
}
void getWords(string inp, vector <string> & currInput)
void getWords(string s, vector <string> & currInput)
{
istringstream iss(s);
currInput.clear();
do {
string sub;
iss >> sub;
currInput.push_back(sub);
int a=0;
a = inp.find(' ', inp.length()-1);
if( a == -1)
inp.append(" ");
a=0;
int b=0;
for (int j=0; j<inp.length(); j++) {
a=inp.find(' ',b);
if(a != -1) {
currInput.push_back(inp.substr(b,a-b));
b=a+1;
j=b;
} else {
j=inp.length();
}
}
} while (iss);
currInput.pop_back();
}
string getTranslation(int index, vector < pair <string , vector <int> > > & gCepts , vector <string> & currF , map <string,int> & singletons)
{
@ -504,7 +483,7 @@ int main(int argc, char * argv[])
loadInput(argv[3],a);
for (int i=0; i<a.size()-1; i++) {
for (int i=0; i<a.size(); i++) {
getWords(e[i],currE);

6
misc/Jamfile
View File

@ -4,7 +4,9 @@ exe processLexicalTable : processLexicalTable.cpp ../moses//moses ;
exe queryPhraseTable : queryPhraseTable.cpp ../moses//moses ;
exe queryLexicalTable : queryLexicalTable.cpp ../moses//moses ;
exe queryLexicalTable : queryLexicalTable.cpp ../moses//moses ;
exe generateSequences : GenerateSequences.cpp ../moses//moses ;
local with-cmph = [ option.get "with-cmph" ] ;
if $(with-cmph) {
@ -19,4 +21,4 @@ else {
alias programsMin ;
}
alias programs : processPhraseTable processLexicalTable queryPhraseTable queryLexicalTable programsMin ;
alias programs : generateSequences processPhraseTable processLexicalTable queryPhraseTable queryLexicalTable programsMin ;

35
moses-chart-cmd/IOWrapper.cpp
View File

@ -388,6 +388,39 @@ void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
m_detailTreeFragmentsOutputCollector->Write(translationId, out.str());
}
//DIMw
void IOWrapper::OutputDetailedAllTranslationReport(
const ChartTrellisPathList &nBestList,
const ChartManager &manager,
const Sentence &sentence,
long translationId)
{
std::ostringstream out;
ApplicationContext applicationContext;
const ChartCellCollection& cells = manager.GetChartCellCollection();
size_t size = manager.GetSource().GetSize();
for (size_t width = 1; width <= size; ++width) {
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
size_t endPos = startPos + width - 1;
WordsRange range(startPos, endPos);
const ChartCell& cell = cells.Get(range);
const HypoList* hyps = cell.GetAllSortedHypotheses();
out << "Chart Cell [" << startPos << ".." << endPos << "]" << endl;
HypoList::const_iterator iter;
size_t c = 1;
for (iter = hyps->begin(); iter != hyps->end(); ++iter) {
out << "----------------Item " << c++ << " ---------------------"
<< endl;
OutputTranslationOptions(out, applicationContext, *iter,
sentence, translationId);
}
}
}
CHECK(m_detailAllOutputCollector);
m_detailAllOutputCollector->Write(translationId, out.str());
}
void IOWrapper::OutputBestHypo(const ChartHypothesis *hypo, long translationId)
{
if (!m_singleBestOutputCollector)
@ -408,7 +441,7 @@ void IOWrapper::OutputBestHypo(const ChartHypothesis *hypo, long translationId)
out << "||| ";
}
Phrase outPhrase(ARRAY_SIZE_INCR);
hypo->CreateOutputPhrase(outPhrase);
hypo->GetOutputPhrase(outPhrase);
// delete 1st & last
CHECK(outPhrase.GetSize() >= 2);

6
moses-chart-cmd/IOWrapper.h
View File

@ -45,6 +45,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "moses/ChartHypothesis.h"
#include "moses/ChartTrellisPath.h"
#include "search/applied.hh"
#include "moses/ChartManager.h"
namespace Moses
{
@ -71,11 +72,15 @@ protected:
std::ostream *m_outputSearchGraphStream;
std::ostream *m_detailedTranslationReportingStream;
std::ostream *m_detailedTreeFragmentsTranslationReportingStream;
//DIMw
std::ostream *m_detailedAllTranslationReportingStream;
std::ostream *m_alignmentInfoStream;
std::string m_inputFilePath;
std::istream *m_inputStream;
Moses::OutputCollector *m_detailOutputCollector;
Moses::OutputCollector *m_detailTreeFragmentsOutputCollector;
//DIMw
Moses::OutputCollector *m_detailAllOutputCollector;
Moses::OutputCollector *m_nBestOutputCollector;
Moses::OutputCollector *m_searchGraphOutputCollector;
Moses::OutputCollector *m_singleBestOutputCollector;
@ -119,6 +124,7 @@ public:
void OutputNBestList(const std::vector<search::Applied> &nbest, long translationId);
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
void OutputDetailedTreeFragmentsTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
void OutputDetailedAllTranslationReport(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
void Backtrack(const Moses::ChartHypothesis *hypo);
void ResetTranslationId();

2
moses-chart-cmd/Jamfile
View File

@ -1,2 +1,2 @@
exe moses_chart : Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp ../moses//moses ;
exe moses_chart : Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp ../moses//moses $(TOP)//boost_iostreams ;

9
moses-chart-cmd/Main.cpp
View File

@ -132,6 +132,15 @@ public:
m_ioWrapper.OutputDetailedTreeFragmentsTranslationReport(bestHypo, sentence, translationId);
}
//DIMw
if (staticData.IsDetailedAllTranslationReportingEnabled()) {
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
size_t nBestSize = staticData.GetNBestSize();
ChartTrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList, staticData.GetDistinctNBest());
m_ioWrapper.OutputDetailedAllTranslationReport(nBestList, manager, sentence, translationId);
}
// n-best
size_t nBestSize = staticData.GetNBestSize();
if (nBestSize > 0) {

42
moses-cmd/IOWrapper.cpp
View File

@ -44,6 +44,9 @@ POSSIBILITY OF SUCH DAMAGE.
#include "moses/StaticData.h"
#include "moses/FeatureVector.h"
#include "moses/InputFileStream.h"
#include "moses/FF/StatefulFeatureFunction.h"
#include "moses/FF/StatelessFeatureFunction.h"
#include "IOWrapper.h"
using namespace std;
@ -188,6 +191,25 @@ InputType*IOWrapper::GetInput(InputType* inputType)
}
}
std::map<size_t, const Factor*> GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor)
{
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
const Phrase &inputPhrase = inputPath.GetPhrase();
std::map<size_t, const Factor*> ret;
for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) {
const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
if (factor) {
std::set<size_t> targetPos = hypo.GetTranslationOption().GetTargetPhrase().GetAlignTerm().GetAlignmentsForSource(sourcePos);
CHECK(targetPos.size() == 1);
ret[*targetPos.begin()] = factor;
}
}
return ret;
}
/***
* print surface factor only for the given phrase
*/
@ -195,23 +217,31 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<
char reportSegmentation, bool reportAllFactors)
{
CHECK(outputFactorOrder.size() > 0);
const Phrase& phrase = edge.GetCurrTargetPhrase();
const TargetPhrase& phrase = edge.GetCurrTargetPhrase();
bool markUnknown = StaticData::Instance().GetMarkUnknown();
if (reportAllFactors == true) {
out << phrase;
} else {
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor().second;
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
std::map<size_t, const Factor*> placeholders;
if (placeholderFactor != NOT_FOUND) {
// creates map of target position -> factor for placeholders
placeholders = GetPlaceholders(edge, placeholderFactor);
}
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
if (placeholderFactor != NOT_FOUND) {
const Factor *origFactor = phrase.GetFactor(pos, placeholderFactor);
if (origFactor) {
factor = origFactor;
if (placeholders.size()) {
// do placeholders
std::map<size_t, const Factor*>::const_iterator iter = placeholders.find(pos);
if (iter != placeholders.end()) {
factor = iter->second;
}
}
CHECK(factor);
//preface surface form with UNK if marking unknowns

5
moses-cmd/IOWrapper.h
View File

@ -56,6 +56,8 @@ POSSIBILITY OF SUCH DAMAGE.
namespace Moses
{
class ScoreComponentCollection;
class Hypothesis;
class Factor;
}
namespace MosesCmd
@ -154,6 +156,9 @@ void OutputFeatureScores( std::ostream& out
, const Moses::FeatureFunction *ff
, std::string &lastName );
// creates a map of TARGET positions which should be replaced by word using placeholder
std::map<size_t, const Moses::Factor*> GetPlaceholders(const Moses::Hypothesis &hypo, Moses::FactorType placeholderFactor);
}
#endif

5
moses-cmd/Main.cpp
View File

@ -117,7 +117,7 @@ public:
// shorthand for "global data"
const StaticData &staticData = StaticData::Instance();
// input sentence
Sentence sentence();
Sentence sentence;
// execute the translation
// note: this executes the search, resulting in a search graph
@ -303,7 +303,10 @@ public:
IFVERBOSE(1) {
debug << "BEST TRANSLATION: " << *bestHypo << endl;
}
} else {
VERBOSE(1, "NO BEST TRANSLATION" << endl);
}
out << endl;
}

15
moses/AlignmentInfo.cpp
View File

@ -60,7 +60,7 @@ std::set<size_t> AlignmentInfo::GetAlignmentsForSource(size_t sourcePos) const
std::set<size_t> ret;
CollType::const_iterator iter;
for (iter = begin(); iter != end(); ++iter) {
const std::pair<size_t,size_t> &align = *iter;
// const std::pair<size_t,size_t> &align = *iter;
if (iter->first == sourcePos) {
ret.insert(iter->second);
}
@ -68,6 +68,19 @@ std::set<size_t> AlignmentInfo::GetAlignmentsForSource(size_t sourcePos) const
return ret;
}
std::set<size_t> AlignmentInfo::GetAlignmentsForTarget(size_t targetPos) const
{
std::set<size_t> ret;
CollType::const_iterator iter;
for (iter = begin(); iter != end(); ++iter) {
// const std::pair<size_t,size_t> &align = *iter;
if (iter->second == targetPos) {
ret.insert(iter->first);
}
}
return ret;
}
bool compare_target(const std::pair<size_t,size_t> *a, const std::pair<size_t,size_t> *b)
{

1
moses/AlignmentInfo.h
View File

@ -70,6 +70,7 @@ public:
}
std::set<size_t> GetAlignmentsForSource(size_t sourcePos) const;
std::set<size_t> GetAlignmentsForTarget(size_t targetPos) const;
size_t GetSize() const {
return m_collection.size();

2
moses/BitmapContainer.cpp
View File

@ -197,7 +197,7 @@ BackwardsEdge::Initialize()
Hypothesis *BackwardsEdge::CreateHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
{
// create hypothesis and calculate all its scores
Hypothesis *newHypo = hypothesis.CreateNext(transOpt, NULL); // TODO FIXME This is absolutely broken - don't pass null here
Hypothesis *newHypo = hypothesis.CreateNext(transOpt); // TODO FIXME This is absolutely broken - don't pass null here
newHypo->Evaluate(m_futurescore);
return newHypo;

14
moses/ChartCell.cpp
View File

@ -126,13 +126,13 @@ const ChartHypothesis *ChartCell::GetBestHypothesis() const
MapType::const_iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
const HypoList &sortedList = iter->second.GetSortedHypotheses();
CHECK(sortedList.size() > 0);
const ChartHypothesis *hypo = sortedList[0];
if (hypo->GetTotalScore() > bestScore) {
bestScore = hypo->GetTotalScore();
ret = hypo;
};
if (sortedList.size() > 0) {
const ChartHypothesis *hypo = sortedList[0];
if (hypo->GetTotalScore() > bestScore) {
bestScore = hypo->GetTotalScore();
ret = hypo;
}
}
}
return ret;

24
moses/ChartHypothesis.cpp
View File

@ -87,8 +87,9 @@ ChartHypothesis::~ChartHypothesis()
/** Create full output phrase that is contained in the hypothesis (and its children)
* \param outPhrase full output phrase as return argument
*/
void ChartHypothesis::CreateOutputPhrase(Phrase &outPhrase) const
void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
{
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
const Word &word = GetCurrTargetPhrase().GetWord(pos);
@ -96,9 +97,26 @@ void ChartHypothesis::CreateOutputPhrase(Phrase &outPhrase) const
// non-term. fill out with prev hypo
size_t nonTermInd = GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap()[pos];
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
prevHypo->CreateOutputPhrase(outPhrase);
prevHypo->GetOutputPhrase(outPhrase);
} else {
outPhrase.AddWord(word);
if (placeholderFactor != NOT_FOUND) {
std::set<size_t> sourcePosSet = GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
CHECK(ruleSourceFromInputPath);
size_t sourcePos = *sourcePosSet.begin();
const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
CHECK(sourceWord);
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor) {
outPhrase.Back()[0] = factor;
}
}
}
}
}
}
@ -107,7 +125,7 @@ void ChartHypothesis::CreateOutputPhrase(Phrase &outPhrase) const
Phrase ChartHypothesis::GetOutputPhrase() const
{
Phrase outPhrase(ARRAY_SIZE_INCR);
CreateOutputPhrase(outPhrase);
GetOutputPhrase(outPhrase);
return outPhrase;
}

2
moses/ChartHypothesis.h
View File

@ -130,7 +130,7 @@ public:
return m_manager;
}
void CreateOutputPhrase(Phrase &outPhrase) const;
void GetOutputPhrase(Phrase &outPhrase) const;
Phrase GetOutputPhrase() const;
int RecombineCompare(const ChartHypothesis &compare) const;

7
moses/ChartHypothesisCollection.cpp
View File

@ -60,6 +60,13 @@ ChartHypothesisCollection::~ChartHypothesisCollection()
*/
bool ChartHypothesisCollection::AddHypothesis(ChartHypothesis *hypo, ChartManager &manager)
{
if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) {
manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, -inf score" << std::endl);
ChartHypothesis::Delete(hypo);
return false;
}
if (hypo->GetTotalScore() < m_bestScore + m_beamWidth) {
// really bad score. don't bother adding hypo into collection
manager.GetSentenceStats().AddDiscarded();

18
moses/ChartManager.cpp
View File

@ -52,12 +52,6 @@ ChartManager::ChartManager(InputType const& source)
,m_parser(source, m_hypoStackColl)
,m_translationOptionList(StaticData::Instance().GetRuleLimit(), source)
{
const StaticData &staticData = StaticData::Instance();
long sentenceID = source.GetTranslationId();
m_constraint = staticData.GetConstrainingPhrase(sentenceID);
if (m_constraint) {
VERBOSE(1, "Search constraint to output: " << *m_constraint<<endl);
}
}
ChartManager::~ChartManager()
@ -135,7 +129,6 @@ void ChartManager::ProcessSentence()
void ChartManager::AddXmlChartOptions()
{
const StaticData &staticData = StaticData::Instance();
const Phrase *constraint = GetConstraint();
const std::vector <ChartTranslationOptions*> xmlChartOptionsList = m_source.GetXmlChartTranslationOptions();
IFVERBOSE(2) {
@ -147,23 +140,12 @@ void ChartManager::AddXmlChartOptions()
i != xmlChartOptionsList.end(); ++i) {
ChartTranslationOptions* opt = *i;
const TargetPhrase &targetPhrase = opt->GetTargetPhrases()[0]->GetPhrase();
const WordsRange &range = opt->GetSourceWordsRange();
RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
if (constraint) {
Phrase hypoPhrase = hypo->GetOutputPhrase();
if (!constraint->Contains(hypoPhrase)) {
delete item;
delete hypo;
continue;
}
}
hypo->Evaluate();
const Word &targetLHS = hypo->GetTargetLHS();
ChartCell &cell = m_hypoStackColl.Get(range);
cell.AddHypothesis(hypo);

9
moses/ChartManager.h
View File

@ -59,7 +59,6 @@ private:
std::auto_ptr<SentenceStats> m_sentenceStats;
clock_t m_start; /**< starting time, used for logging */
unsigned m_hypothesisId; /* For handing out hypothesis ids to ChartHypothesis */
const Phrase *m_constraint;
ChartParser m_parser;
@ -86,6 +85,11 @@ public:
return *m_sentenceStats;
}
//DIMw
const ChartCellCollection& GetChartCellCollection() const {
return m_hypoStackColl;
}
/***
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
* currently an empty function
@ -101,9 +105,6 @@ public:
unsigned GetNextHypoId() {
return m_hypothesisId++;
}
const Phrase *GetConstraint() const
{ return m_constraint; }
};
}

11
moses/ChartParser.cpp
View File

@ -137,16 +137,21 @@ ChartParser::ChartParser(InputType const &source, ChartCellCollectionBase &cells
const StaticData &staticData = StaticData::Instance();
staticData.InitializeForInput(source);
CreateInputPaths(m_source);
const std::vector<PhraseDictionary*> &dictionaries = staticData.GetPhraseDictionaries();
m_ruleLookupManagers.reserve(dictionaries.size());
for (std::vector<PhraseDictionary*>::const_iterator p = dictionaries.begin();
p != dictionaries.end(); ++p) {
const PhraseDictionary *dict = *p;
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(*this, cells));
ChartRuleLookupManager *lookupMgr = nonConstDict->CreateRuleLookupManager(*this, cells);
m_ruleLookupManagers.push_back(lookupMgr);
}
CreateInputPaths(m_source);
}
ChartParser::~ChartParser()
@ -224,7 +229,7 @@ void ChartParser::CreateInputPaths(const InputType &input)
const InputPath &ChartParser::GetInputPath(WordsRange &range) const
{
return GetInputPath(range.GetStartPos(), range.GetEndPos());
return GetInputPath(range.GetStartPos(), range.GetEndPos());
}
const InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos) const

8
moses/ChartRuleLookupManager.cpp
View File

@ -3,11 +3,7 @@
namespace Moses
{
/*
const Sentence &ChartRuleLookupManager::GetSentence() const
{
return m_parser.GetSentence();
}
*/
ChartRuleLookupManager::~ChartRuleLookupManager()
{}
} // namespace Moses

2
moses/ChartRuleLookupManager.h
View File

@ -45,7 +45,7 @@ public:
: m_parser(parser)
, m_cellCollection(cellColl) {}
virtual ~ChartRuleLookupManager() {}
virtual ~ChartRuleLookupManager();
const ChartCellLabelSet &GetTargetLabelSet(size_t begin, size_t end) const {
return m_cellCollection.GetBase(WordsRange(begin, end)).GetTargetLabelSet();

10
moses/ChartTranslationOption.cpp
View File

@ -16,8 +16,16 @@ void ChartTranslationOption::Evaluate(const InputType &input, const InputPath &i
for (size_t i = 0; i < ffs.size(); ++i) {
const FeatureFunction &ff = *ffs[i];
ff.Evaluate(input, inputPath, m_scoreBreakdown);
ff.Evaluate(input, inputPath, m_targetPhrase, m_scoreBreakdown);
}
}
std::ostream& operator<<(std::ostream &out, const ChartTranslationOption &transOpt)
{
out << transOpt.m_targetPhrase << " " << transOpt.m_scoreBreakdown;
return out;
}
}

20
moses/ChartTranslationOption.h
View File

@ -10,10 +10,13 @@ class InputType;
class ChartTranslationOption
{
friend std::ostream& operator<<(std::ostream&, const ChartTranslationOption&);
protected:
const TargetPhrase &m_targetPhrase;
ScoreComponentCollection m_scoreBreakdown;
const InputPath *m_inputPath;
const std::vector<const Word*> *m_ruleSourceFromInputPath; // used by placeholders
public:
ChartTranslationOption(const TargetPhrase &targetPhrase);
@ -22,10 +25,19 @@ public:
return m_targetPhrase;
}
void SetInputPath(const InputPath *inputPath)
{ m_inputPath = inputPath; }
const InputPath *GetInputPath() const
{ return m_inputPath; }
const InputPath *GetInputPath() const {
return m_inputPath;
}
void SetInputPath(const InputPath *inputPath) {
m_inputPath = inputPath;
}
const std::vector<const Word*> *GetSourceRuleFromInputPath() const {
return m_ruleSourceFromInputPath;
}
void SetSourceRuleFromInputPath(const std::vector<const Word*> *obj) {
m_ruleSourceFromInputPath = obj;
}
const ScoreComponentCollection &GetScores() const {
return m_scoreBreakdown;

11
moses/ChartTranslationOptionList.cpp
View File

@ -67,6 +67,14 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &tpc,
return;
}
for (size_t i = 0; i < stackVec.size(); ++i) {
const ChartCellLabel &chartCellLabel = *stackVec[i];
size_t numHypos = chartCellLabel.GetStack().cube->size();
if (numHypos == 0) {
return; // empty stack. These rules can't be used
}
}
float score = ChartTranslationOptions::CalcEstimateOfBestScore(tpc, stackVec);
// If the rule limit has already been reached then don't add the option
@ -149,8 +157,9 @@ void ChartTranslationOptionList::ApplyThreshold()
void ChartTranslationOptionList::Evaluate(const InputType &input, const InputPath &inputPath)
{
// NEVER iterate over ALL of the collection. Just over the first m_size
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
for (iter = m_collection.begin(); iter != m_collection.begin() + m_size; ++iter) {
ChartTranslationOptions &transOpts = **iter;
transOpts.Evaluate(input, inputPath);
}

60
moses/ChartTranslationOptions.cpp
View File

@ -21,6 +21,10 @@
#include "ChartHypothesis.h"
#include "ChartCellLabel.h"
#include "ChartTranslationOption.h"
#include "InputPath.h"
#include "StaticData.h"
using namespace std;
namespace Moses
{
@ -66,6 +70,11 @@ float ChartTranslationOptions::CalcEstimateOfBestScore(
void ChartTranslationOptions::Evaluate(const InputType &input, const InputPath &inputPath)
{
SetInputPath(&inputPath);
if (StaticData::Instance().GetPlaceholderFactor() != NOT_FOUND) {
CreateSourceRuleFromInputPath();
}
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
ChartTranslationOption &transOpt = **iter;
@ -75,4 +84,55 @@ void ChartTranslationOptions::Evaluate(const InputType &input, const InputPath &
}
void ChartTranslationOptions::SetInputPath(const InputPath *inputPath)
{
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
ChartTranslationOption &transOpt = **iter;
transOpt.SetInputPath(inputPath);
}
}
void ChartTranslationOptions::CreateSourceRuleFromInputPath()
{
if (m_collection.size() == 0) {
return;
}
const InputPath *inputPath = m_collection.front()->GetInputPath();
CHECK(inputPath);
std::vector<const Word*> &ruleSourceFromInputPath = inputPath->AddRuleSourceFromInputPath();
size_t chartCellIndex = 0;
const ChartCellLabel *chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
size_t ind = 0;
for (size_t sourcePos = m_wordsRange->GetStartPos(); sourcePos <= m_wordsRange->GetEndPos(); ++sourcePos, ++ind) {
if (chartCellLabel) {
if (sourcePos == chartCellLabel->GetCoverage().GetEndPos()) {
// end of child range. push an empty word to denote non-term
ruleSourceFromInputPath.push_back(NULL);
++chartCellIndex;
chartCellLabel = (chartCellIndex < m_stackVec.size()) ? m_stackVec[chartCellIndex] : NULL;
} else if (sourcePos >= chartCellLabel->GetCoverage().GetStartPos()) {
// in the range of child hypo. do nothing
} else {
// not yet reached child range. add word
ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
}
} else {
// no child in sight. add word
ruleSourceFromInputPath.push_back(&inputPath->GetPhrase().GetWord(ind));
}
}
// save it to each trans opt
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
ChartTranslationOption &transOpt = **iter;
transOpt.SetSourceRuleFromInputPath(&ruleSourceFromInputPath);
}
}
}

4
moses/ChartTranslationOptions.h
View File

@ -83,6 +83,10 @@ public:
void Evaluate(const InputType &input, const InputPath &inputPath);
void SetInputPath(const InputPath *inputPath);
void CreateSourceRuleFromInputPath();
private:
StackVec m_stackVec; //! vector of hypothesis list!

18
moses/ChartTrellisNode.cpp
View File

@ -73,6 +73,8 @@ ChartTrellisNode::~ChartTrellisNode()
Phrase ChartTrellisNode::GetOutputPhrase() const
{
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
// exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
Phrase ret(ARRAY_SIZE_INCR);
@ -89,6 +91,22 @@ Phrase ChartTrellisNode::GetOutputPhrase() const
ret.Append(childPhrase);
} else {
ret.AddWord(word);
if (placeholderFactor != NOT_FOUND) {
std::set<size_t> sourcePosSet = m_hypo.GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = m_hypo.GetTranslationOption().GetSourceRuleFromInputPath();
CHECK(ruleSourceFromInputPath);
size_t sourcePos = *sourcePosSet.begin();
const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
CHECK(sourceWord);
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor) {
ret.Back()[0] = factor;
}
}
}
}
}

6
moses/ConfusionNet.cpp
View File

@ -11,6 +11,7 @@
#include "Sentence.h"
#include "UserMessage.h"
#include "moses/FF/InputFeature.h"
#include "util/check.hh"
namespace Moses
{
@ -69,6 +70,7 @@ ConfusionNet::ConfusionNet()
if (staticData.IsChart()) {
m_defaultLabelSet.insert(StaticData::Instance().GetInputDefaultNonTerminal());
}
CHECK(StaticData::Instance().GetInputFeature());
}
ConfusionNet::~ConfusionNet()
{
@ -79,8 +81,8 @@ ConfusionNet::ConfusionNet(Sentence const& s)
{
data.resize(s.GetSize());
for(size_t i=0; i<s.GetSize(); ++i) {
ScorePair scorePair;
std::pair<Word, ScorePair > temp = std::make_pair(s.GetWord(i), scorePair);
ScorePair scorePair;
std::pair<Word, ScorePair > temp = std::make_pair(s.GetWord(i), scorePair);
data[i].push_back(temp);
}
}

6
moses/DecodeStepTranslation.cpp
View File

@ -72,7 +72,7 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
for (iterTargetPhrase = phraseColl->begin(); iterTargetPhrase != iterEnd; ++iterTargetPhrase) {
const TargetPhrase& targetPhrase = **iterTargetPhrase;
const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
// const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
// skip if the
if (targetPhrase.GetSize() != currSize) continue;
@ -197,7 +197,7 @@ const InputPath &DecodeStepTranslation::GetInputPathLEGACY(
const Phrase &phraseFromIP = inputPath.GetPhrase();
const Word &wordIP = phraseFromIP.GetWord(0);
const WordsRange &range = inputPath.GetWordsRange();
// const WordsRange &range = inputPath.GetWordsRange();
if (wordFromPt == wordIP) {
return inputPath;
@ -238,7 +238,7 @@ void DecodeStepTranslation::ProcessLEGACY(const TranslationOption &inputPartialT
for (iterTargetPhrase = phraseColl->begin(); iterTargetPhrase != iterEnd; ++iterTargetPhrase) {
const TargetPhrase& targetPhrase = **iterTargetPhrase;
const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
// const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
// skip if the
if (targetPhrase.GetSize() != currSize) continue;

2
moses/FF/BleuScoreFeature.cpp
View File

@ -2,6 +2,8 @@
#include "moses/StaticData.h"
#include "moses/UserMessage.h"
#include "moses/Hypothesis.h"
#include "moses/FactorCollection.h"
using namespace std;

11
moses/FF/BleuScoreFeature.h
View File

@ -119,13 +119,14 @@ public:
int featureID,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
bool Enabled() const {

156
moses/FF/ConstrainedDecoding.cpp Normal file
View File

@ -0,0 +1,156 @@
#include "ConstrainedDecoding.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"
#include "moses/StaticData.h"
#include "moses/InputFileStream.h"
#include "moses/Util.h"
#include "util/exception.hh"
using namespace std;
namespace Moses
{
ConstrainedDecodingState::ConstrainedDecodingState(const Hypothesis &hypo)
{
hypo.GetOutputPhrase(m_outputPhrase);
}
ConstrainedDecodingState::ConstrainedDecodingState(const ChartHypothesis &hypo)
{
hypo.GetOutputPhrase(m_outputPhrase);
}
int ConstrainedDecodingState::Compare(const FFState& other) const
{
const ConstrainedDecodingState &otherFF = static_cast<const ConstrainedDecodingState&>(other);
bool ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
return ret;
}
//////////////////////////////////////////////////////////////////
void ConstrainedDecoding::Load()
{
const StaticData &staticData = StaticData::Instance();
bool addBeginEndWord = (staticData.GetSearchAlgorithm() == ChartDecoding) || (staticData.GetSearchAlgorithm() == ChartIncremental);
InputFileStream constraintFile(m_path);
std::string line;
long sentenceID = staticData.GetStartTranslationId() - 1;
while (getline(constraintFile, line)) {
vector<string> vecStr = Tokenize(line, "\t");
Phrase phrase(0);
if (vecStr.size() == 1) {
sentenceID++;
phrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), vecStr[0], staticData.GetFactorDelimiter(), NULL);
} else if (vecStr.size() == 2) {
sentenceID = Scan<long>(vecStr[0]);
phrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), vecStr[1], staticData.GetFactorDelimiter(), NULL);
} else {
CHECK(false);
}
if (addBeginEndWord) {
phrase.InitStartEndWord();
}
m_constraints.insert(make_pair(sentenceID,phrase));
}
}
std::vector<float> ConstrainedDecoding::DefaultWeights() const
{
CHECK(m_numScoreComponents == 1);
vector<float> ret(1, 1);
return ret;
}
template <class H, class M>
const Phrase *GetConstraint(const std::map<long,Phrase> &constraints, const H &hypo)
{
const M &mgr = hypo.GetManager();
const InputType &input = mgr.GetSource();
long id = input.GetTranslationId();
map<long,Phrase>::const_iterator iter;
iter = constraints.find(id);
if (iter == constraints.end()) {
return NULL;
} else {
return &iter->second;
}
}
FFState* ConstrainedDecoding::Evaluate(
const Hypothesis& hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
const Phrase *ref = GetConstraint<Hypothesis, Manager>(m_constraints, hypo);
CHECK(ref);
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
const Phrase &outputPhrase = ret->GetPhrase();
size_t searchPos = ref->Find(outputPhrase, m_maxUnknowns);
float score;
if (hypo.IsSourceCompleted()) {
// translated entire sentence.
score = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize())
? 0 : - std::numeric_limits<float>::infinity();
} else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
}
accumulator->PlusEquals(this, score);
return ret;
}
FFState* ConstrainedDecoding::EvaluateChart(
const ChartHypothesis &hypo,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
const Phrase *ref = GetConstraint<ChartHypothesis, ChartManager>(m_constraints, hypo);
CHECK(ref);
const ChartManager &mgr = hypo.GetManager();
const Sentence &source = static_cast<const Sentence&>(mgr.GetSource());
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
const Phrase &outputPhrase = ret->GetPhrase();
size_t searchPos = ref->Find(outputPhrase, m_maxUnknowns);
float score;
if (hypo.GetCurrSourceRange().GetStartPos() == 0 &&
hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) {
// translated entire sentence.
score = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize())
? 0 : - std::numeric_limits<float>::infinity();
} else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
}
accumulator->PlusEquals(this, score);
return ret;
}
void ConstrainedDecoding::SetParameter(const std::string& key, const std::string& value)
{
if (key == "path") {
m_path = value;
} else if (key == "max-unknowns") {
m_maxUnknowns = Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
}

86
moses/FF/ConstrainedDecoding.h Normal file
View File

@ -0,0 +1,86 @@
#pragma once
#include <string>
#include <map>
#include "StatefulFeatureFunction.h"
#include "FFState.h"
#include "moses/Phrase.h"
namespace Moses
{
class ConstrainedDecodingState : public FFState
{
public:
ConstrainedDecodingState()
{}
ConstrainedDecodingState(const Hypothesis &hypo);
ConstrainedDecodingState(const ChartHypothesis &hypo);
int Compare(const FFState& other) const;
const Phrase &GetPhrase() const {
return m_outputPhrase;
}
protected:
Phrase m_outputPhrase;
};
//////////////////////////////////////////////////////////////////
// only allow hypotheses which match reference
class ConstrainedDecoding : public StatefulFeatureFunction
{
public:
ConstrainedDecoding(const std::string &line)
:StatefulFeatureFunction("ConstrainedDecoding", 1, line)
,m_maxUnknowns(0) {
m_tuneable = false;
ReadParameters();
}
void Load();
bool IsUseable(const FactorMask &mask) const {
return true;
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new ConstrainedDecodingState();
}
std::vector<float> DefaultWeights() const;
void SetParameter(const std::string& key, const std::string& value);
protected:
std::string m_path;
std::map<long,Phrase> m_constraints;
int m_maxUnknowns;
};
}

111
moses/FF/ControlRecombination.cpp
View File

@ -1,70 +1,83 @@
#include "ControlRecombination.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"
#include "moses/StaticData.h"
#include "moses/InputFileStream.h"
#include "moses/Util.h"
#include "util/exception.hh"
using namespace std;
namespace Moses
{
ControlRecombination::ControlRecombination(const std::string &line)
:StatefulFeatureFunction("ControlRecombination", 0, line)
,m_type(Output)
ControlRecombinationState::ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff)
:m_ff(ff)
{
if (ff.GetType() == SameOutput) {
UTIL_THROW(util::Exception, "Implemented not yet completed for phrase-based model. Need to take into account the coverage");
hypo.GetOutputPhrase(m_outputPhrase);
} else {
m_hypo = &hypo;
}
}
ControlRecombinationState::ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff)
:m_ff(ff)
{
if (ff.GetType() == SameOutput) {
hypo.GetOutputPhrase(m_outputPhrase);
} else {
m_hypo = &hypo;
}
}
int ControlRecombinationState::Compare(const FFState& other) const
{
const ControlRecombinationState &otherFF = static_cast<const ControlRecombinationState&>(other);
if (m_ff.GetType() == SameOutput) {
bool ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
return ret;
} else {
// compare hypo address. Won't be equal unless they're actually the same hypo
if (m_hypo == otherFF.m_hypo)
return 0;
return (m_hypo < otherFF.m_hypo) ? -1 : +1;
}
}
std::vector<float> ControlRecombination::DefaultWeights() const
{
CHECK(m_numScoreComponents == 0);
vector<float> ret(0);
return ret;
}
FFState* ControlRecombination::Evaluate(
const Hypothesis& hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
return new ControlRecombinationState(hypo, *this);
}
FFState* ControlRecombination::EvaluateChart(
const ChartHypothesis &hypo,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new ControlRecombinationState(hypo, *this);
}
void ControlRecombination::SetParameter(const std::string& key, const std::string& value)
{
if (key == "type") {
m_type = (Type) Scan<size_t>(value);
m_type = (ControlRecombinationType) Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
FFState* ControlRecombination::Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
ControlRecombinationState *state = new ControlRecombinationState(&cur_hypo);
return state;
}
FFState* ControlRecombination::EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
UTIL_THROW(util::Exception, "Not implemented yet");
}
const FFState* ControlRecombination::EmptyHypothesisState(const InputType &input) const
{
ControlRecombinationState *state = new ControlRecombinationState();
}
ControlRecombinationState::ControlRecombinationState()
:m_hypo(NULL)
{
}
ControlRecombinationState::ControlRecombinationState(const Hypothesis *hypo)
:m_hypo(hypo)
{
}
int ControlRecombinationState::Compare(const FFState& other) const
{
const ControlRecombinationState &other2 = static_cast<const ControlRecombinationState&>(other);
const Hypothesis *otherHypo = other2.m_hypo;
Phrase thisOutputPhrase, otherOutputPhrase;
m_hypo->GetOutputPhrase(thisOutputPhrase);
otherHypo->GetOutputPhrase(otherOutputPhrase);
int ret = thisOutputPhrase.Compare(otherOutputPhrase);
return ret;
}
}

97
moses/FF/ControlRecombination.h
View File

@ -1,68 +1,97 @@
#pragma once
#include <string>
#include <map>
#include "StatefulFeatureFunction.h"
#include "moses/FF/FFState.h"
#include "FFState.h"
#include "moses/Phrase.h"
namespace Moses
{
enum ControlRecombinationType {
// when to recombine
SameOutput = 1,
Never = 2
};
class ControlRecombinationState;
class ControlRecombination;
// force hypotheses NOT to recombine. For forced decoding
class ControlRecombinationState : public FFState
{
public:
ControlRecombinationState(const ControlRecombination &ff)
:m_ff(ff)
{}
ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
int Compare(const FFState& other) const;
const Phrase &GetPhrase() const {
return m_outputPhrase;
}
protected:
Phrase m_outputPhrase;
const ControlRecombination &m_ff;
const void *m_hypo;
};
//////////////////////////////////////////////////////////////////
// only allow recombination for the same output
class ControlRecombination : public StatefulFeatureFunction
{
public:
enum Type {
None,
Output,
Segmentation
};
ControlRecombination(const std::string &line)
:StatefulFeatureFunction("ControlRecombination", 0, line)
,m_type(SameOutput)
ControlRecombination(const std::string &line);
{
m_tuneable = false;
ReadParameters();
}
bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual FFState* Evaluate(
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart(
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new ControlRecombinationState(*this);
}
//! return the state associated with the empty hypothesis for a given sentence
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
std::vector<float> DefaultWeights() const;
void SetParameter(const std::string& key, const std::string& value);
ControlRecombinationType GetType() const {
return m_type;
}
protected:
Type m_type;
ControlRecombinationType m_type;
};
class ControlRecombinationState : public FFState
{
protected:
const Hypothesis *m_hypo;
public:
ControlRecombinationState();
ControlRecombinationState(const Hypothesis *hypo);
int Compare(const FFState& other) const;
}
};
} // namespace

15
moses/FF/DecodeFeature.h
View File

@ -64,19 +64,20 @@ public:
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
protected:

1
moses/FF/DistortionScoreProducer.cpp
View File

@ -3,6 +3,7 @@
#include "FFState.h"
#include "moses/WordsRange.h"
#include "moses/StaticData.h"
#include "moses/Hypothesis.h"
using namespace std;

11
moses/FF/DistortionScoreProducer.h
View File

@ -42,13 +42,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

34
moses/FF/ExternalFeature.cpp
View File

@ -7,9 +7,9 @@ namespace Moses
{
ExternalFeatureState::ExternalFeatureState(int stateSize, void *data)
{
m_stateSize = stateSize;
m_data = malloc(stateSize);
memcpy(m_data, data, stateSize);
m_stateSize = stateSize;
m_data = malloc(stateSize);
memcpy(m_data, data, stateSize);
}
void ExternalFeature::Load()
@ -17,27 +17,28 @@ void ExternalFeature::Load()
string nparam = "testing";
if (m_path.size() < 1) {
cerr << "External requires a path to a dynamic library!\n";
abort();
cerr << "External requires a path to a dynamic library!\n";
abort();
}
lib_handle = dlopen(m_path.c_str(), RTLD_LAZY);
if (!lib_handle) {
cerr << "dlopen reports: " << dlerror() << endl;
cerr << "Did you provide a full path to the dynamic library?\n";
abort();
cerr << "dlopen reports: " << dlerror() << endl;
cerr << "Did you provide a full path to the dynamic library?\n";
abort();
}
CdecFF* (*fn)(const string&) =
(CdecFF* (*)(const string&))(dlsym(lib_handle, "create_ff"));
(CdecFF* (*)(const string&))(dlsym(lib_handle, "create_ff"));
if (!fn) {
cerr << "dlsym reports: " << dlerror() << endl;
abort();
cerr << "dlsym reports: " << dlerror() << endl;
abort();
}
ff_ext = (*fn)(nparam);
m_stateSize = ff_ext->StateSize();
}
ExternalFeature::~ExternalFeature() {
ExternalFeature::~ExternalFeature()
{
delete ff_ext;
dlclose(lib_handle);
}
@ -45,9 +46,8 @@ ExternalFeature::~ExternalFeature() {
void ExternalFeature::SetParameter(const std::string& key, const std::string& value)
{
if (key == "path") {
m_path = value;
}
else {
m_path = value;
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
@ -57,7 +57,7 @@ FFState* ExternalFeature::Evaluate(
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
return new ExternalFeatureState(m_stateSize);
return new ExternalFeatureState(m_stateSize);
}
FFState* ExternalFeature::EvaluateChart(
@ -65,7 +65,7 @@ FFState* ExternalFeature::EvaluateChart(
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new ExternalFeatureState(m_stateSize);
return new ExternalFeatureState(m_stateSize);
}

104
moses/FF/ExternalFeature.h
View File

@ -12,81 +12,79 @@ class CdecFF;
class ExternalFeatureState : public FFState
{
protected:
int m_stateSize;
void *m_data;
int m_stateSize;
void *m_data;
public:
ExternalFeatureState(int stateSize)
:m_stateSize(stateSize)
,m_data(NULL)
{}
ExternalFeatureState(int stateSize, void *data);
ExternalFeatureState(int stateSize)
:m_stateSize(stateSize)
,m_data(NULL)
{}
ExternalFeatureState(int stateSize, void *data);
~ExternalFeatureState()
{
free(m_data);
}
~ExternalFeatureState() {
free(m_data);
}
int Compare(const FFState& other) const
{
const ExternalFeatureState &otherFF = static_cast<const ExternalFeatureState&>(other);
int ret = memcmp(m_data, otherFF.m_data, m_stateSize);
return ret;
}
int Compare(const FFState& other) const {
const ExternalFeatureState &otherFF = static_cast<const ExternalFeatureState&>(other);
int ret = memcmp(m_data, otherFF.m_data, m_stateSize);
return ret;
}
};
// copied from cdec
class ExternalFeature : public StatefulFeatureFunction
{
public:
ExternalFeature(const std::string &line)
:StatefulFeatureFunction("ExternalFeature", line)
{
ReadParameters();
}
~ExternalFeature();
ExternalFeature(const std::string &line)
:StatefulFeatureFunction("ExternalFeature", line) {
ReadParameters();
}
~ExternalFeature();
void Load();
void Load();
bool IsUseable(const FactorMask &mask) const
{ return true; }
bool IsUseable(const FactorMask &mask) const {
return true;
}
void SetParameter(const std::string& key, const std::string& value);
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const
{
return new ExternalFeatureState(m_stateSize);
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new ExternalFeatureState(m_stateSize);
}
protected:
std::string m_path;
void* lib_handle;
CdecFF *ff_ext;
int m_stateSize;
std::string m_path;
void* lib_handle;
CdecFF *ff_ext;
int m_stateSize;
};
class CdecFF
{
public:
virtual int StateSize() const = 0;
virtual int StateSize() const = 0;
};
}

15
moses/FF/Factory.cpp
View File

@ -32,9 +32,16 @@
#include "moses/FF/ControlRecombination.h"
#include "moses/FF/ExternalFeature.h"
#include "moses/FF/InternalStructStatelessFF.h"
#include "moses/FF/ConstrainedDecoding.h"
#include "moses/FF/SkeletonStatelessFF.h"
#include "moses/FF/SkeletonStatefulFF.h"
#include "moses/LM/SkeletonLM.h"
#include "moses/TranslationModel/SkeletonPT.h"
#ifdef PT_UG
#include "moses/TranslationModel/mmsapt.h"
#endif
#include "moses/LM/Ken.h"
#ifdef LM_IRST
@ -143,11 +150,19 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(PhrasePenalty);
MOSES_FNAME2("UnknownWordPenalty", UnknownWordPenaltyProducer);
MOSES_FNAME(ControlRecombination);
MOSES_FNAME(ConstrainedDecoding);
MOSES_FNAME(ExternalFeature);
MOSES_FNAME(SkeletonStatelessFF);
MOSES_FNAME(SkeletonStatefulFF);
MOSES_FNAME(ExternalFeature);
MOSES_FNAME(InternalStructStatelessFF);
MOSES_FNAME(SkeletonLM);
MOSES_FNAME(SkeletonPT);
#ifdef PT_UG
MOSES_FNAME(Mmsapt);
#endif
#ifdef HAVE_SYNLM
MOSES_FNAME(SyntacticLanguageModel);
#endif

19
moses/FF/FeatureFunction.cpp
View File

@ -16,8 +16,6 @@ namespace Moses
multiset<string> FeatureFunction::description_counts;
std::vector<FeatureFunction*> FeatureFunction::m_producers;
std::vector<const StatelessFeatureFunction*> StatelessFeatureFunction::m_statelessFFs;
std::vector<const StatefulFeatureFunction*> StatefulFeatureFunction::m_statefulFFs;
FeatureFunction &FeatureFunction::FindFeatureFunction(const std::string& name)
{
@ -31,21 +29,28 @@ FeatureFunction &FeatureFunction::FindFeatureFunction(const std::string& name)
throw "Unknown feature " + name;
}
FeatureFunction::FeatureFunction(const std::string& description, const std::string &line)
FeatureFunction::
FeatureFunction(const std::string& description,
const std::string& line)
: m_tuneable(true)
, m_numScoreComponents(1)
{
Initialize(description, line);
}
FeatureFunction::FeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
: m_numScoreComponents(numScoreComponents)
, m_tuneable(true)
FeatureFunction::
FeatureFunction(const std::string& description,
size_t numScoreComponents,
const std::string& line)
: m_tuneable(true)
, m_numScoreComponents(numScoreComponents)
{
Initialize(description, line);
}
void FeatureFunction::Initialize(const std::string& description, const std::string &line)
void
FeatureFunction::
Initialize(const std::string& description, const std::string &line)
{
ParseLine(description, line);

1
moses/FF/FeatureFunction.h
View File

@ -107,6 +107,7 @@ public:
// Currently not used by any FF. Not called by moses_chart
virtual void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const = 0;
virtual void SetParameter(const std::string& key, const std::string& value);

1
moses/FF/GlobalLexicalModel.cpp
View File

@ -4,6 +4,7 @@
#include "moses/InputFileStream.h"
#include "moses/TranslationOption.h"
#include "moses/UserMessage.h"
#include "moses/FactorCollection.h"
using namespace std;

11
moses/FF/GlobalLexicalModel.h
View File

@ -81,13 +81,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}

8
moses/FF/GlobalLexicalModelUnlimited.cpp
View File

@ -3,6 +3,7 @@
#include "moses/StaticData.h"
#include "moses/InputFileStream.h"
#include "moses/UserMessage.h"
#include "moses/Hypothesis.h"
#include "util/string_piece_hash.hh"
using namespace std;
@ -111,7 +112,7 @@ void GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, ScoreComp
const Sentence& input = *(m_local->input);
const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
for(int targetIndex = 0; targetIndex < targetPhrase.GetSize(); targetIndex++ ) {
for(size_t targetIndex = 0; targetIndex < targetPhrase.GetSize(); targetIndex++ ) {
StringPiece targetString = targetPhrase.GetWord(targetIndex).GetString(0); // TODO: change for other factors
if (m_ignorePunctuation) {
@ -132,8 +133,9 @@ void GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, ScoreComp
}
boost::unordered_set<uint64_t> alreadyScored;
for(int sourceIndex = 0; sourceIndex < input.GetSize(); sourceIndex++ ) {
const StringPiece sourceString = input.GetWord(sourceIndex).GetString(0); // TODO: change for other factors
for(size_t sourceIndex = 0; sourceIndex < input.GetSize(); sourceIndex++ ) {
const StringPiece sourceString = input.GetWord(sourceIndex).GetString(0);
// TODO: change for other factors
if (m_ignorePunctuation) {
// check if first char is punctuation

11
moses/FF/GlobalLexicalModelUnlimited.h
View File

@ -91,13 +91,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void AddFeature(ScoreComponentCollection* accumulator,

22
moses/FF/InputFeature.cpp
View File

@ -19,11 +19,11 @@ InputFeature::InputFeature(const std::string &line)
void InputFeature::Load()
{
const StaticData &staticData = StaticData::Instance();
const PhraseDictionary *pt = staticData.GetTranslationScoreProducer(0);
const PhraseDictionaryTreeAdaptor *ptBin = dynamic_cast<const PhraseDictionaryTreeAdaptor*>(pt);
const StaticData &staticData = StaticData::Instance();
const PhraseDictionary *pt = staticData.GetTranslationScoreProducer(0);
const PhraseDictionaryTreeAdaptor *ptBin = dynamic_cast<const PhraseDictionaryTreeAdaptor*>(pt);
m_legacy = (ptBin != NULL);
m_legacy = (ptBin != NULL);
}
void InputFeature::SetParameter(const std::string& key, const std::string& value)
@ -40,17 +40,19 @@ void InputFeature::SetParameter(const std::string& key, const std::string& value
void InputFeature::Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{
if (m_legacy) {
//binary phrase-table does input feature itself
return;
}
if (m_legacy) {
//binary phrase-table does input feature itself
return;
}
/*
const ScorePair *scores = inputPath.GetInputScore();
if (scores) {
scoreBreakdown.PlusEquals(this, *scores);
}
*/
}
} // namespace

15
moses/FF/InputFeature.h
View File

@ -33,19 +33,20 @@ public:
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const;
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}

11
moses/FF/LexicalReordering/LexicalReordering.h
View File

@ -52,13 +52,14 @@ public:
UTIL_THROW(util::Exception, "LexicalReordering is not valid for chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
private:

13
moses/FF/OSM-Feature/OpSequenceModel.cpp
View File

@ -235,13 +235,14 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va
if (key == "path") {
m_lmPath = value;
} else if (key == "numFeatures") {
numFeatures = Scan<int>(value);
} else if (key == "order") {
lmOrder = Scan<int>(value);
} else if (key == "sFactor") {
} else if (key == "support-features") {
if(value == "no")
numFeatures = 1;
else
numFeatures = 5;
} else if (key == "input-factor") {
sFactor = Scan<int>(value);
} else if (key == "tFactor") {
} else if (key == "output-factor") {
tFactor = Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);

7
moses/FF/OSM-Feature/OpSequenceModel.h
View File

@ -18,8 +18,6 @@ public:
lm::ngram::Model * OSM;
int lmOrder;
float unkOpProb;
int sFactor; // Source Factor ...
int tFactor; // Target Factor ...
@ -41,8 +39,9 @@ public:
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase

11
moses/FF/PhraseBoundaryFeature.h
View File

@ -54,13 +54,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);

7
moses/FF/PhraseLengthFeature.h
View File

@ -25,7 +25,7 @@ public:
}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis& hypo,
@ -34,8 +34,9 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
virtual void Evaluate(const Phrase &source

11
moses/FF/PhrasePairFeature.h
View File

@ -46,13 +46,14 @@ public:
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Load();

9
moses/FF/PhrasePenalty.h
View File

@ -20,15 +20,16 @@ public:
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
};

52
moses/FF/SkeletonStatefulFF.cpp
View File

@ -1,7 +1,59 @@
#include <vector>
#include "SkeletonStatefulFF.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/Hypothesis.h"
using namespace std;
namespace Moses
{
int SkeletonState::Compare(const FFState& other) const
{
const SkeletonState &otherState = static_cast<const SkeletonState&>(other);
if (m_targetLen == otherState.m_targetLen)
return 0;
return (m_targetLen < otherState.m_targetLen) ? -1 : +1;
}
void SkeletonStatefulFF::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SkeletonStatefulFF::Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* SkeletonStatefulFF::Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
// dense scores
vector<float> newScores(m_numScoreComponents);
newScores[0] = 1.5;
newScores[1] = 0.3;
newScores[2] = 0.4;
accumulator->PlusEquals(this, newScores);
// sparse scores
accumulator->PlusEquals(this, "sparse-name", 2.4);
int targetLen = cur_hypo.GetCurrTargetPhrase().GetSize();
return new SkeletonState(0);
}
FFState* SkeletonStatefulFF::EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState(0);
}
}

70
moses/FF/SkeletonStatefulFF.h
View File

@ -9,53 +9,45 @@ namespace Moses
class SkeletonState : public FFState
{
int m_targetLen;
public:
int Compare(const FFState& other) const
{
return 0;
}
SkeletonState(int targetLen)
:m_targetLen(targetLen)
{}
int Compare(const FFState& other) const;
};
class SkeletonStatefulFF : public StatefulFeatureFunction
{
public:
SkeletonStatefulFF(const std::string &line)
:StatefulFeatureFunction("StatefulFeatureFunction", line)
{}
SkeletonStatefulFF(const std::string &line)
:StatefulFeatureFunction("SkeletonStatefulFF", 3, line)
{}
bool IsUseable(const FactorMask &mask) const
{ return true; }
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState();
}
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState();
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const
{
return new SkeletonState();
}
bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new SkeletonState(0);
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
};

33
moses/FF/SkeletonStatelessFF.cpp
View File

@ -1,7 +1,40 @@
#include "SkeletonStatelessFF.h"
#include "moses/ScoreComponentCollection.h"
#include <vector>
using namespace std;
namespace Moses
{
void SkeletonStatelessFF::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// dense scores
vector<float> newScores(m_numScoreComponents);
newScores[0] = 1.5;
newScores[1] = 0.3;
scoreBreakdown.PlusEquals(this, newScores);
// sparse scores
scoreBreakdown.PlusEquals(this, "sparse-name", 2.4);
}
void SkeletonStatelessFF::Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void SkeletonStatelessFF::Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void SkeletonStatelessFF::EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
}

38
moses/FF/SkeletonStatelessFF.h
View File

@ -9,28 +9,26 @@ namespace Moses
class SkeletonStatelessFF : public StatelessFeatureFunction
{
public:
SkeletonStatelessFF(const std::string &line)
:StatelessFeatureFunction("SkeletonStatelessFF", line)
{}
SkeletonStatelessFF(const std::string &line)
:StatelessFeatureFunction("SkeletonStatelessFF", 2, line)
{}
bool IsUseable(const FactorMask &mask) const
{ return true; }
bool IsUseable(const FactorMask &mask) const {
return true;
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
virtual void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const;
};

15
moses/FF/SourceWordDeletionFeature.h
View File

@ -29,18 +29,19 @@ public:
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,

2
moses/FF/StatefulFeatureFunction.cpp
View File

@ -3,6 +3,8 @@
namespace Moses
{
std::vector<const StatefulFeatureFunction*> StatefulFeatureFunction::m_statefulFFs;
StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, const std::string &line)
: FeatureFunction(description, line)
{

2
moses/FF/StatelessFeatureFunction.cpp
View File

@ -3,6 +3,8 @@
namespace Moses
{
std::vector<const StatelessFeatureFunction*> StatelessFeatureFunction::m_statelessFFs;
StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, const std::string &line)
:FeatureFunction(description, line)
{

11
moses/FF/TargetBigramFeature.h
View File

@ -48,13 +48,14 @@ public:
throw std::logic_error("TargetBigramFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);

11
moses/FF/TargetNgramFeature.h
View File

@ -193,13 +193,14 @@ public:
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);

9
moses/FF/TargetWordInsertionFeature.h
View File

@ -33,14 +33,15 @@ public:
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,

15
moses/FF/UnknownWordPenaltyProducer.h
View File

@ -23,19 +23,20 @@ public:
std::vector<float> DefaultWeights() const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

9
moses/FF/WordPenaltyProducer.h
View File

@ -23,14 +23,15 @@ public:
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
};

11
moses/FF/WordTranslationFeature.h
View File

@ -54,13 +54,14 @@ public:
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

3
moses/FeatureVector.h
View File

@ -377,9 +377,6 @@ public:
return (m_fv->m_features[m_name] -= lhs);
}
private:
FValue m_tmp;
private:
FVector* m_fv;
const FName& m_name;

1
moses/FeatureVectorTest.cpp
View File

@ -53,7 +53,6 @@ BOOST_AUTO_TEST_CASE(vector_sum_diff)
BOOST_CHECK_CLOSE((FValue)diff[n3], -5.7, TOL);
BOOST_CHECK_CLOSE((FValue)diff[n4], -0.6, TOL);
f1 -= f3;
cerr << f1 << endl << f3 << endl ;
BOOST_CHECK_CLOSE((FValue)f1[n1],0,TOL);
}

53
moses/Hypothesis.cpp
View File

@ -147,64 +147,23 @@ void Hypothesis::AddArc(Hypothesis *loserHypo)
/***
* return the subclass of Hypothesis most appropriate to the given translation option
*/
Hypothesis* Hypothesis::CreateNext(const TranslationOption &transOpt, const Phrase* constraint) const
Hypothesis* Hypothesis::CreateNext(const TranslationOption &transOpt) const
{
return Create(*this, transOpt, constraint);
return Create(*this, transOpt);
}
/***
* return the subclass of Hypothesis most appropriate to the given translation option
*/
Hypothesis* Hypothesis::Create(const Hypothesis &prevHypo, const TranslationOption &transOpt, const Phrase* constrainingPhrase)
Hypothesis* Hypothesis::Create(const Hypothesis &prevHypo, const TranslationOption &transOpt)
{
// This method includes code for constraint decoding
bool createHypothesis = true;
if (constrainingPhrase != NULL) {
size_t constraintSize = constrainingPhrase->GetSize();
size_t start = 1 + prevHypo.GetCurrTargetWordsRange().GetEndPos();
const Phrase &transOptPhrase = transOpt.GetTargetPhrase();
size_t transOptSize = transOptPhrase.GetSize();
size_t endpoint = start + transOptSize - 1;
if (endpoint < constraintSize) {
WordsRange range(start, endpoint);
Phrase relevantConstraint = constrainingPhrase->GetSubString(range);
if ( ! relevantConstraint.IsCompatible(transOptPhrase) ) {
createHypothesis = false;
}
} else {
createHypothesis = false;
}
}
if (createHypothesis) {
#ifdef USE_HYPO_POOL
Hypothesis *ptr = s_objectPool.getPtr();
return new(ptr) Hypothesis(prevHypo, transOpt);
Hypothesis *ptr = s_objectPool.getPtr();
return new(ptr) Hypothesis(prevHypo, transOpt);
#else
return new Hypothesis(prevHypo, transOpt);
return new Hypothesis(prevHypo, transOpt);
#endif
} else {
// If the previous hypothesis plus the proposed translation option
// fail to match the provided constraint,
// return a null hypothesis.
return NULL;
}
}
/***
* return the subclass of Hypothesis most appropriate to the given target phrase

4
moses/Hypothesis.h
View File

@ -99,7 +99,7 @@ public:
~Hypothesis();
/** return the subclass of Hypothesis most appropriate to the given translation option */
static Hypothesis* Create(const Hypothesis &prevHypo, const TranslationOption &transOpt, const Phrase* constraint);
static Hypothesis* Create(const Hypothesis &prevHypo, const TranslationOption &transOpt);
static Hypothesis* Create(Manager& manager, const WordsBitmap &initialCoverage);
@ -107,7 +107,7 @@ public:
static Hypothesis* Create(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt);
/** return the subclass of Hypothesis most appropriate to the given translation option */
Hypothesis* CreateNext(const TranslationOption &transOpt, const Phrase* constraint) const;
Hypothesis* CreateNext(const TranslationOption &transOpt) const;
void PrintHypothesis() const;

7
moses/HypothesisStackCubePruning.cpp
View File

@ -81,6 +81,13 @@ pair<HypothesisStackCubePruning::iterator, bool> HypothesisStackCubePruning::Add
bool HypothesisStackCubePruning::AddPrune(Hypothesis *hypo)
{
if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) {
m_manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, constraint" << std::endl);
FREEHYPO(hypo);
return false;
}
if (hypo->GetTotalScore() < m_worstScore) {
// too bad for stack. don't bother adding hypo into collection
m_manager.GetSentenceStats().AddDiscarded();

7
moses/HypothesisStackNormal.cpp
View File

@ -88,6 +88,13 @@ pair<HypothesisStackNormal::iterator, bool> HypothesisStackNormal::Add(Hypothesi
bool HypothesisStackNormal::AddPrune(Hypothesis *hypo)
{
if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) {
m_manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, constraint" << std::endl);
FREEHYPO(hypo);
return false;
}
// too bad for stack. don't bother adding hypo into collection
if (!StaticData::Instance().GetDisableDiscarding() &&
hypo->GetTotalScore() < m_worstScore

36
moses/InputPath.cpp
View File

@ -10,24 +10,18 @@ using namespace std;
namespace Moses
{
InputPath::InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms, const WordsRange &range, const InputPath *prevNode
,const ScorePair *inputScore)
InputPath::
InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms,
const WordsRange &range, const InputPath *prevNode,
const ScorePair *inputScore)
:m_prevNode(prevNode)
,m_phrase(phrase)
,m_sourceNonTerms(sourceNonTerms)
,m_range(range)
,m_inputScore(inputScore)
,m_sourceNonTerms(sourceNonTerms)
{
//cerr << "phrase=" << phrase << " m_inputScore=" << *m_inputScore << endl;
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor().first;
if (placeholderFactor != NOT_FOUND) {
for (size_t pos = 0; pos < m_phrase.GetSize(); ++pos) {
if (m_phrase.GetFactor(pos, placeholderFactor)) {
m_placeholders.push_back(pos);
}
}
}
}
InputPath::~InputPath()
@ -63,26 +57,6 @@ void InputPath::SetTargetPhrases(const PhraseDictionary &phraseDictionary
m_targetPhrases[&phraseDictionary] = value;
}
bool InputPath::SetPlaceholders(TargetPhrase *targetPhrase) const
{
FactorType sourcePlaceholderFactor = StaticData::Instance().GetPlaceholderFactor().first;
FactorType targetPlaceholderFactor = StaticData::Instance().GetPlaceholderFactor().second;
const AlignmentInfo &alignments = targetPhrase->GetAlignTerm();
for (size_t i = 0; i < m_placeholders.size(); ++i) {
size_t sourcePos = m_placeholders[i];
set<size_t> targetPos = alignments.GetAlignmentsForSource(sourcePos);
if (targetPos.size() == 1) {
const Word &sourceWord = m_phrase.GetWord(sourcePos);
Word &targetWord = targetPhrase->GetWord(*targetPos.begin());
targetWord[targetPlaceholderFactor] = sourceWord[sourcePlaceholderFactor];
} else {
return false;
}
}
return true;
}
const Word &InputPath::GetLastWord() const
{
size_t len = m_phrase.GetSize();

13
moses/InputPath.h
View File

@ -13,9 +13,9 @@ namespace Moses
class PhraseDictionary;
class TargetPhraseCollection;
class ScoreComponentCollection;
class ScorePair;
class TargetPhrase;
class InputPath;
struct ScorePair;
typedef std::list<InputPath*> InputPathList;
@ -34,10 +34,14 @@ protected:
Phrase m_phrase;
WordsRange m_range;
const ScorePair *m_inputScore;
// for phrase-based model only
std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> > m_targetPhrases;
// for syntax model only
mutable std::vector<std::vector<const Word*> > m_ruleSourceFromInputPath;
const NonTerminalSet m_sourceNonTerms;
std::vector<size_t> m_placeholders;
bool SetPlaceholders(TargetPhrase *targetPhrase) const;
public:
@ -77,6 +81,11 @@ public:
return m_inputScore;
}
std::vector<const Word*> &AddRuleSourceFromInputPath() const {
m_ruleSourceFromInputPath.push_back(std::vector<const Word*>());
return m_ruleSourceFromInputPath.back();
}
};
};

9
moses/Jamfile
View File

@ -67,12 +67,15 @@ lib moses :
*Test.cpp Mock*.cpp
FF/Factory.cpp
]
headers FF_Factory.o LM//LM TranslationModel/CompactPT//CompactPT synlm ThreadPool rt
..//search ../util/double-conversion//double-conversion ..//z ../OnDiskPt//OnDiskPt ;
headers FF_Factory.o LM//LM TranslationModel/CompactPT//CompactPT synlm ThreadPool rt
..//search ../util/double-conversion//double-conversion ..//z ../OnDiskPt//OnDiskPt
$(TOP)//boost_iostreams ;
#generic//generic mm//mm
alias headers-to-install : [ glob-tree *.h ] ;
import testing ;
#unit-test moses_test : [ glob *Test.cpp Mock*.cpp ] moses headers ..//z ../OnDiskPt//OnDiskPt ..//boost_unit_test_framework ;
unit-test moses_test : [ glob *Test.cpp Mock*.cpp ] moses headers ..//z ../OnDiskPt//OnDiskPt ..//boost_unit_test_framework ;

5
moses/LM/Base.h
View File

@ -91,8 +91,9 @@ public:
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown) const
{}
};

1
moses/LM/Implementation.h
View File

@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/Factor.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/FF/FeatureFunction.h"
#include "moses/Word.h"
#include "Base.h"

2
moses/LM/Jamfile
View File

@ -75,7 +75,7 @@ obj ORLM.o : ORLM.cpp ..//headers ../TranslationModel/DynSAInclude//dynsa : : :
#Top-level LM library. If you've added a file that doesn't depend on external
#libraries, put it here.
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp Implementation.cpp Joint.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp ORLM.o
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp Implementation.cpp Joint.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp ORLM.o
../../lm//kenlm ..//headers $(dependencies) ;
alias macros : : : : <define>$(lmmacros) ;

7
moses/LM/Ken.cpp
View File

@ -138,7 +138,7 @@ private:
};
} // namespace
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
:LanguageModel("KENLM", line)
,m_factorType(factorType)
@ -351,10 +351,11 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateChart(const Cha
return newState;
}
template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Incremental::Manager &manager) const {
template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Incremental::Manager &manager) const
{
manager.LMCallback(*m_ngram, m_lmIdLookup);
}
template <class Model>
bool LanguageModelKen<Model>::IsUseable(const FactorMask &mask) const
{

60
moses/LM/Ken.h
View File

@ -36,7 +36,7 @@ namespace Moses
{
//class LanguageModel;
class FFState;
class FFState;
LanguageModel *ConstructKenLM(const std::string &line);
@ -49,37 +49,37 @@ LanguageModel *ConstructKenLM(const std::string &line, const std::string &file,
template <class Model> class LanguageModelKen : public LanguageModel
{
public:
LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
virtual FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
virtual FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual bool IsUseable(const FactorMask &mask) const;
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
virtual FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
virtual FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual bool IsUseable(const FactorMask &mask) const;
protected:
boost::shared_ptr<Model> m_ngram;
const Factor *m_beginSentenceFactor;
FactorType m_factorType;
lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
boost::shared_ptr<Model> m_ngram;
const Factor *m_beginSentenceFactor;
FactorType m_factorType;
lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
private:
LanguageModelKen(const LanguageModelKen<Model> &copy_from);
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
LanguageModelKen(const LanguageModelKen<Model> &copy_from);
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
int position = hypo.GetCurrTargetWordsRange().GetEndPos();
@ -93,8 +93,8 @@ private:
}
}
std::vector<lm::WordIndex> m_lmIdLookup;
std::vector<lm::WordIndex> m_lmIdLookup;
};
} // namespace Moses

4
moses/LM/LDHT.cpp
View File

@ -2,9 +2,9 @@
// Oliver Wilson <oliver.wilson@ed.ac.uk>
//
// This file should be compiled only when the LM_RAND flag is enabled.
// This file should be compiled only when the LM_RAND flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when LM_RAND is disabled.
//
#ifdef LM_RAND

4
moses/LM/ParallelBackoff.cpp
View File

@ -36,14 +36,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/Util.h"
// By default, SRILM defines a function called zopen.
//
//
// However, on Mac OS X (and possibly other BSDs),
// <stdio.h> already defines a zopen function.
//
// To resolve this conflict, SRILM checks to see if HAVE_ZOPEN is defined.
// If it is, SRILM will rename its zopen function as my_zopen.
//
// So, before importing any SRILM headers,
// So, before importing any SRILM headers,
// it is important to define HAVE_ZOPEN if we are on an Apple OS:
//
#ifdef __APPLE__

4
moses/LM/Rand.cpp
View File

@ -17,9 +17,9 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
// This file should be compiled only when the LM_RAND flag is enabled.
// This file should be compiled only when the LM_RAND flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when LM_RAND is disabled.
//
#ifdef LM_RAND

4
moses/LM/SRI.cpp
View File

@ -32,14 +32,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/StaticData.h"
// By default, SRILM defines a function called zopen.
//
//
// However, on Mac OS X (and possibly other BSDs),
// <stdio.h> already defines a zopen function.
//
// To resolve this conflict, SRILM checks to see if HAVE_ZOPEN is defined.
// If it is, SRILM will rename its zopen function as my_zopen.
//
// So, before importing any SRILM headers,
// So, before importing any SRILM headers,
// it is important to define HAVE_ZOPEN if we are on an Apple OS:
//
#ifdef __APPLE__

1
moses/LM/SingleFactor.cpp
View File

@ -31,6 +31,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/FactorCollection.h"
#include "moses/Phrase.h"
#include "moses/StaticData.h"
#include "moses/FactorTypeSet.h"
using namespace std;

Some files were not shown because too many files have changed in this diff Show More