diff --git a/contrib/other-builds/moses/.project b/contrib/other-builds/moses/.project
index d8679eb44..cf311ed9c 100644
--- a/contrib/other-builds/moses/.project
+++ b/contrib/other-builds/moses/.project
@@ -166,6 +166,16 @@
1
PARENT-3-PROJECT_LOC/moses/ChartHypothesisCollection.h
+
+ ChartKBestExtractor.cpp
+ 1
+ PARENT-3-PROJECT_LOC/moses/ChartKBestExtractor.cpp
+
+
+ ChartKBestExtractor.h
+ 1
+ PARENT-3-PROJECT_LOC/moses/ChartKBestExtractor.h
+
ChartManager.cpp
1
@@ -1251,6 +1261,26 @@
1
PARENT-3-PROJECT_LOC/moses/FF/PhrasePenalty.h
+
+ FF/ReferenceComparison.cpp
+ 1
+ PARENT-3-PROJECT_LOC/moses/FF/ReferenceComparison.cpp
+
+
+ FF/ReferenceComparison.h
+ 1
+ PARENT-3-PROJECT_LOC/moses/FF/ReferenceComparison.h
+
+
+ FF/RuleAmbiguity.cpp
+ 1
+ PARENT-3-PROJECT_LOC/moses/FF/RuleAmbiguity.cpp
+
+
+ FF/RuleAmbiguity.h
+ 1
+ PARENT-3-PROJECT_LOC/moses/FF/RuleAmbiguity.h
+
FF/SetSourcePhrase.cpp
1
diff --git a/contrib/server/mosesserver.cpp b/contrib/server/mosesserver.cpp
index 56e70f766..9ee8c7c06 100644
--- a/contrib/server/mosesserver.cpp
+++ b/contrib/server/mosesserver.cpp
@@ -26,6 +26,7 @@
#include
using namespace Moses;
+using namespace MosesCmd;
using namespace std;
typedef std::map params_t;
@@ -117,20 +118,17 @@ public:
void breakOutParams(const params_t& params) {
params_t::const_iterator si = params.find("source");
if(si == params.end())
- throw xmlrpc_c::fault("Missing source sentence",
- xmlrpc_c::fault::CODE_PARSE);
+ throw xmlrpc_c::fault("Missing source sentence", xmlrpc_c::fault::CODE_PARSE);
source_ = xmlrpc_c::value_string(si->second);
cerr << "source = " << source_ << endl;
si = params.find("target");
if(si == params.end())
- throw xmlrpc_c::fault("Missing target sentence",
- xmlrpc_c::fault::CODE_PARSE);
+ throw xmlrpc_c::fault("Missing target sentence", xmlrpc_c::fault::CODE_PARSE);
target_ = xmlrpc_c::value_string(si->second);
cerr << "target = " << target_ << endl;
si = params.find("alignment");
if(si == params.end())
- throw xmlrpc_c::fault("Missing alignment",
- xmlrpc_c::fault::CODE_PARSE);
+ throw xmlrpc_c::fault("Missing alignment", xmlrpc_c::fault::CODE_PARSE);
alignment_ = xmlrpc_c::value_string(si->second);
cerr << "alignment = " << alignment_ << endl;
si = params.find("bounded");
@@ -163,8 +161,7 @@ public:
xmlrpc_c::fault::CODE_PARSE);
}
const string model_name = xmlrpc_c::value_string(si->second);
- PhraseDictionaryMultiModel*
- pdmm = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
+ PhraseDictionaryMultiModel* pdmm = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
si = params.find("phrase_pairs");
if (si == params.end()) {
@@ -175,13 +172,10 @@ public:
vector > phrase_pairs;
- xmlrpc_c::value_array phrase_pairs_array
- = xmlrpc_c::value_array(si->second);
- vector phrasePairValueVector
- (phrase_pairs_array.vectorValueValue());
+ xmlrpc_c::value_array phrase_pairs_array = xmlrpc_c::value_array(si->second);
+ vector phrasePairValueVector(phrase_pairs_array.vectorValueValue());
for (size_t i=0;i < phrasePairValueVector.size();i++) {
- xmlrpc_c::value_array phrasePairArray
- = xmlrpc_c::value_array(phrasePairValueVector[i]);
+ xmlrpc_c::value_array phrasePairArray = xmlrpc_c::value_array(phrasePairValueVector[i]);
vector phrasePair(phrasePairArray.vectorValueValue());
string L1 = xmlrpc_c::value_string(phrasePair[0]);
string L2 = xmlrpc_c::value_string(phrasePair[1]);
@@ -197,8 +191,7 @@ public:
}
*retvalP = xmlrpc_c::value_array(weight_vector_ret);
#else
- string errmsg = "Error: Perplexity minimization requires dlib ";
- errmsg += "(compilation option --with-dlib)";
+ string errmsg = "Error: Perplexity minimization requires dlib (compilation option --with-dlib)";
cerr << errmsg << endl;
*retvalP = xmlrpc_c::value_string(errmsg);
#endif
@@ -234,6 +227,8 @@ public:
cerr << "Input: " << source << endl;
si = params.find("align");
bool addAlignInfo = (si != params.end());
+ si = params.find("word-align");
+ bool addWordAlignInfo = (si != params.end());
si = params.find("sg");
bool addGraphInfo = (si != params.end());
si = params.find("topt");
@@ -241,8 +236,7 @@ public:
si = params.find("report-all-factors");
bool reportAllFactors = (si != params.end());
si = params.find("nbest");
- int nbest_size = ((si == params.end()) ? 0
- : int(xmlrpc_c::value_int(si->second)));
+ int nbest_size = (si == params.end()) ? 0 : int(xmlrpc_c::value_int(si->second));
si = params.find("nbest-distinct");
bool nbest_distinct = (si != params.end());
@@ -252,21 +246,17 @@ public:
vector multiModelWeights;
si = params.find("lambda");
if (si != params.end()) {
- xmlrpc_c::value_array multiModelArray
- = xmlrpc_c::value_array(si->second);
- vector multiModelValueVector
- (multiModelArray.vectorValueValue());
+ xmlrpc_c::value_array multiModelArray = xmlrpc_c::value_array(si->second);
+ vector multiModelValueVector(multiModelArray.vectorValueValue());
for (size_t i=0;i < multiModelValueVector.size();i++) {
- multiModelWeights.push_back
- (xmlrpc_c::value_double(multiModelValueVector[i]));
+ multiModelWeights.push_back(xmlrpc_c::value_double(multiModelValueVector[i]));
}
}
si = params.find("model_name");
if (si != params.end() && multiModelWeights.size() > 0) {
const string model_name = xmlrpc_c::value_string(si->second);
- PhraseDictionaryMultiModel* pdmm;
- pdmm = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
+ PhraseDictionaryMultiModel* pdmm = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
pdmm->SetTemporaryMultiModelWeightsVector(multiModelWeights);
}
@@ -303,8 +293,21 @@ public:
vector alignInfo;
outputHypo(out,hypo,addAlignInfo,alignInfo,reportAllFactors);
if (addAlignInfo) {
- retData.insert(pair
- ("align", xmlrpc_c::value_array(alignInfo)));
+ retData.insert(pair("align", xmlrpc_c::value_array(alignInfo)));
+ }
+ if (addWordAlignInfo) {
+ stringstream wordAlignment;
+ OutputAlignment(wordAlignment, hypo);
+ vector alignments;
+ string alignmentPair;
+ while (wordAlignment >> alignmentPair) {
+ int pos = alignmentPair.find('-');
+ map wordAlignInfo;
+ wordAlignInfo["source-word"] = xmlrpc_c::value_int(atoi(alignmentPair.substr(0, pos).c_str()));
+ wordAlignInfo["target-word"] = xmlrpc_c::value_int(atoi(alignmentPair.substr(pos + 1).c_str()));
+ alignments.push_back(xmlrpc_c::value_struct(wordAlignInfo));
+ }
+ retData.insert(pair("word-align", alignments));
}
if(addGraphInfo) {
@@ -326,12 +329,9 @@ public:
*retvalP = xmlrpc_c::value_struct(retData);
}
- void outputHypo(ostream& out, const Hypothesis* hypo, bool addAlignmentInfo,
- vector& alignInfo,
- bool reportAllFactors = false) {
+ void outputHypo(ostream& out, const Hypothesis* hypo, bool addAlignmentInfo, vector& alignInfo, bool reportAllFactors = false) {
if (hypo->GetPrevHypo() != NULL) {
- outputHypo(out,hypo->GetPrevHypo(),addAlignmentInfo, alignInfo,
- reportAllFactors);
+ outputHypo(out,hypo->GetPrevHypo(),addAlignmentInfo, alignInfo, reportAllFactors);
Phrase p = hypo->GetCurrTargetPhrase();
if(reportAllFactors) {
out << p << " ";
@@ -349,16 +349,9 @@ public:
* triples.
**/
map phraseAlignInfo;
-
- phraseAlignInfo["tgt-start"]
- = xmlrpc_c::value_int(hypo->GetCurrTargetWordsRange().GetStartPos());
-
- phraseAlignInfo["src-start"]
- = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetStartPos());
-
- phraseAlignInfo["src-end"]
- = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetEndPos());
-
+ phraseAlignInfo["tgt-start"] = xmlrpc_c::value_int(hypo->GetCurrTargetWordsRange().GetStartPos());
+ phraseAlignInfo["src-start"] = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetStartPos());
+ phraseAlignInfo["src-end"] = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetEndPos());
alignInfo.push_back(xmlrpc_c::value_struct(phraseAlignInfo));
}
}
@@ -456,9 +449,25 @@ public:
}
nBestXMLItem["hyp"] = xmlrpc_c::value_string(out.str());
- if (addAlignmentInfo)
+ if (addAlignmentInfo) {
nBestXMLItem["align"] = xmlrpc_c::value_array(alignInfo);
+ if ((int)edges.size() > 0) {
+ stringstream wordAlignment;
+ OutputAlignment(wordAlignment, edges[0]);
+ vector alignments;
+ string alignmentPair;
+ while (wordAlignment >> alignmentPair) {
+ int pos = alignmentPair.find('-');
+ map wordAlignInfo;
+ wordAlignInfo["source-word"] = xmlrpc_c::value_int(atoi(alignmentPair.substr(0, pos).c_str()));
+ wordAlignInfo["target-word"] = xmlrpc_c::value_int(atoi(alignmentPair.substr(pos + 1).c_str()));
+ alignments.push_back(xmlrpc_c::value_struct(wordAlignInfo));
+ }
+ nBestXMLItem["word-align"] = xmlrpc_c::value_array(alignments);
+ }
+ }
+
if (addScoreBreakdown)
{
// should the score breakdown be reported in a more structured manner?
@@ -619,11 +628,20 @@ int main(int argc, char** argv)
myRegistry.addMethod("updater", updater);
myRegistry.addMethod("optimize", optimizer);
+ xmlrpc_c::serverAbyss myAbyssServer(
+ myRegistry,
+ port, // TCP port on which to listen
+ logfile
+ );
+ /* doesn't work with xmlrpc-c v. 1.16.33 - ie very old lib on Ubuntu 12.04
xmlrpc_c::serverAbyss myAbyssServer(
- myRegistry,
- port, // TCP port on which to listen
- logfile
+ xmlrpc_c::serverAbyss::constrOpt()
+ .registryPtr(&myRegistry)
+ .portNumber(port) // TCP port on which to listen
+ .logFileName(logfile)
+ .allowOrigin("*")
);
+ */
cerr << "Listening on port " << port << endl;
if (isSerial) {
diff --git a/moses-chart-cmd/IOWrapper.cpp b/moses-chart-cmd/IOWrapper.cpp
index 56c166422..81612ed1f 100644
--- a/moses-chart-cmd/IOWrapper.cpp
+++ b/moses-chart-cmd/IOWrapper.cpp
@@ -553,7 +553,7 @@ void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
//DIMw
void IOWrapper::OutputDetailedAllTranslationReport(
- const ChartTrellisPathList &nBestList,
+ const std::vector > &nBestList,
const ChartManager &manager,
const Sentence &sentence,
long translationId)
@@ -793,6 +793,58 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, long tran
m_nBestOutputCollector->Write(translationId, out.str());
}
+void IOWrapper::OutputNBestList(const ChartKBestExtractor::KBestVec &nBestList,
+ long translationId)
+{
+ std::ostringstream out;
+
+ if (m_nBestOutputCollector->OutputIsCout()) {
+ // Set precision only if we're writing the n-best list to cout. This is to
+ // preserve existing behaviour, but should probably be done either way.
+ IOWrapper::FixPrecision(out);
+ }
+
+ bool includeWordAlignment =
+ StaticData::Instance().PrintAlignmentInfoInNbest();
+
+ for (ChartKBestExtractor::KBestVec::const_iterator p = nBestList.begin();
+ p != nBestList.end(); ++p) {
+ const ChartKBestExtractor::Derivation &derivation = **p;
+
+ // get the derivation's target-side yield
+ Phrase outputPhrase = ChartKBestExtractor::GetOutputPhrase(derivation);
+
+ // delete and
+ UTIL_THROW_IF2(outputPhrase.GetSize() < 2,
+ "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
+ outputPhrase.RemoveWord(0);
+ outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
+
+ // print the translation ID, surface factors, and scores
+ out << translationId << " ||| ";
+ OutputSurface(out, outputPhrase, m_outputFactorOrder, false);
+ out << " ||| ";
+ OutputAllFeatureScores(derivation.scoreBreakdown, out);
+ out << " ||| " << derivation.score;
+
+ // optionally, print word alignments
+ if (includeWordAlignment) {
+ out << " ||| ";
+ Alignments align;
+ OutputAlignmentNBest(align, derivation, 0);
+ for (Alignments::const_iterator q = align.begin(); q != align.end();
+ ++q) {
+ out << q->first << "-" << q->second << " ";
+ }
+ }
+
+ out << std::endl;
+ }
+
+ assert(m_nBestOutputCollector);
+ m_nBestOutputCollector->Write(translationId, out.str());
+}
+
void IOWrapper::OutputNBestList(const std::vector &nbest, long translationId)
{
std::ostringstream out;
@@ -927,6 +979,85 @@ size_t IOWrapper::OutputAlignmentNBest(Alignments &retAlign, const Moses::ChartT
return totalTargetSize;
}
+size_t IOWrapper::OutputAlignmentNBest(
+ Alignments &retAlign,
+ const Moses::ChartKBestExtractor::Derivation &derivation,
+ size_t startTarget)
+{
+ const ChartHypothesis &hypo = derivation.edge.head->hypothesis;
+
+ size_t totalTargetSize = 0;
+ size_t startSource = hypo.GetCurrSourceRange().GetStartPos();
+
+ const TargetPhrase &tp = hypo.GetCurrTargetPhrase();
+
+ size_t thisSourceSize = CalcSourceSize(&hypo);
+
+ // position of each terminal word in translation rule, irrespective of alignment
+ // if non-term, number is undefined
+ vector sourceOffsets(thisSourceSize, 0);
+ vector targetOffsets(tp.GetSize(), 0);
+
+ const AlignmentInfo &aiNonTerm = hypo.GetCurrTargetPhrase().GetAlignNonTerm();
+ vector sourceInd2pos = aiNonTerm.GetSourceIndex2PosMap();
+ const AlignmentInfo::NonTermIndexMap &targetPos2SourceInd = aiNonTerm.GetNonTermIndexMap();
+
+ UTIL_THROW_IF2(sourceInd2pos.size() != derivation.subderivations.size(),
+ "Error");
+
+ size_t targetInd = 0;
+ for (size_t targetPos = 0; targetPos < tp.GetSize(); ++targetPos) {
+ if (tp.GetWord(targetPos).IsNonTerminal()) {
+ UTIL_THROW_IF2(targetPos >= targetPos2SourceInd.size(), "Error");
+ size_t sourceInd = targetPos2SourceInd[targetPos];
+ size_t sourcePos = sourceInd2pos[sourceInd];
+
+ const Moses::ChartKBestExtractor::Derivation &subderivation =
+ *derivation.subderivations[sourceInd];
+
+ // calc source size
+ size_t sourceSize = subderivation.edge.head->hypothesis.GetCurrSourceRange().GetNumWordsCovered();
+ sourceOffsets[sourcePos] = sourceSize;
+
+ // calc target size.
+ // Recursively look thru child hypos
+ size_t currStartTarget = startTarget + totalTargetSize;
+ size_t targetSize = OutputAlignmentNBest(retAlign, subderivation,
+ currStartTarget);
+ targetOffsets[targetPos] = targetSize;
+
+ totalTargetSize += targetSize;
+ ++targetInd;
+ } else {
+ ++totalTargetSize;
+ }
+ }
+
+ // convert position within translation rule to absolute position within
+ // source sentence / output sentence
+ ShiftOffsets(sourceOffsets, startSource);
+ ShiftOffsets(targetOffsets, startTarget);
+
+ // get alignments from this hypo
+ const AlignmentInfo &aiTerm = hypo.GetCurrTargetPhrase().GetAlignTerm();
+
+ // add to output arg, offsetting by source & target
+ AlignmentInfo::const_iterator iter;
+ for (iter = aiTerm.begin(); iter != aiTerm.end(); ++iter) {
+ const std::pair &align = *iter;
+ size_t relSource = align.first;
+ size_t relTarget = align.second;
+ size_t absSource = sourceOffsets[relSource];
+ size_t absTarget = targetOffsets[relTarget];
+
+ pair alignPoint(absSource, absTarget);
+ pair ret = retAlign.insert(alignPoint);
+ UTIL_THROW_IF2(!ret.second, "Error");
+ }
+
+ return totalTargetSize;
+}
+
void IOWrapper::OutputAlignment(size_t translationId , const Moses::ChartHypothesis *hypo)
{
ostringstream out;
diff --git a/moses-chart-cmd/IOWrapper.h b/moses-chart-cmd/IOWrapper.h
index 9e09ef00f..bd8264eb6 100644
--- a/moses-chart-cmd/IOWrapper.h
+++ b/moses-chart-cmd/IOWrapper.h
@@ -40,6 +40,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "moses/TypeDef.h"
#include "moses/Sentence.h"
#include "moses/FactorTypeSet.h"
+#include "moses/ChartKBestExtractor.h"
#include "moses/ChartTrellisPathList.h"
#include "moses/OutputCollector.h"
#include "moses/ChartHypothesis.h"
@@ -90,6 +91,7 @@ protected:
typedef std::set< std::pair > Alignments;
size_t OutputAlignmentNBest(Alignments &retAlign, const Moses::ChartTrellisNode &node, size_t startTarget);
+ std::size_t OutputAlignmentNBest(Alignments &retAlign, const Moses::ChartKBestExtractor::Derivation &derivation, std::size_t startTarget);
size_t OutputAlignment(Alignments &retAlign, const Moses::ChartHypothesis *hypo, size_t startTarget);
void OutputAlignment(std::vector< std::set > &retAlignmentsS2T, const Moses::AlignmentInfo &ai);
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
@@ -129,12 +131,13 @@ public:
void OutputBestHypo(const std::vector& mbrBestHypo, long translationId);
void OutputBestNone(long translationId);
void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, long translationId);
+ void OutputNBestList(const std::vector > &nBestList, long translationId);
void OutputNBestList(const std::vector &nbest, long translationId);
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
void OutputDetailedTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
void OutputDetailedTreeFragmentsTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
void OutputDetailedTreeFragmentsTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
- void OutputDetailedAllTranslationReport(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
+ void OutputDetailedAllTranslationReport(const std::vector > &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
void Backtrack(const Moses::ChartHypothesis *hypo);
void ResetTranslationId();
diff --git a/moses-chart-cmd/Main.cpp b/moses-chart-cmd/Main.cpp
index f2baff0fa..fd82b5692 100644
--- a/moses-chart-cmd/Main.cpp
+++ b/moses-chart-cmd/Main.cpp
@@ -151,7 +151,7 @@ public:
if (staticData.IsDetailedAllTranslationReportingEnabled()) {
const Sentence &sentence = dynamic_cast(*m_source);
size_t nBestSize = staticData.GetNBestSize();
- ChartTrellisPathList nBestList;
+ std::vector > nBestList;
manager.CalcNBest(nBestSize, nBestList, staticData.GetDistinctNBest());
m_ioWrapper.OutputDetailedAllTranslationReport(nBestList, manager, sentence, translationId);
}
@@ -160,7 +160,7 @@ public:
size_t nBestSize = staticData.GetNBestSize();
if (nBestSize > 0) {
VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl);
- ChartTrellisPathList nBestList;
+ std::vector > nBestList;
manager.CalcNBest(nBestSize, nBestList,staticData.GetDistinctNBest());
m_ioWrapper.OutputNBestList(nBestList, translationId);
IFVERBOSE(2) {
diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp
index 4d20ba9ef..db7aa69e8 100644
--- a/moses-cmd/Main.cpp
+++ b/moses-cmd/Main.cpp
@@ -180,6 +180,7 @@ public:
} else {
TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl);
}
+ delete file;
}
// Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder
@@ -233,7 +234,7 @@ public:
} else {
stringstream hypergraphDirName;
- hypergraphDirName << boost::filesystem::current_path() << "/hypergraph";
+ hypergraphDirName << boost::filesystem::current_path().string() << "/hypergraph";
hypergraphDir = hypergraphDirName.str();
}
}
@@ -536,9 +537,7 @@ size_t OutputFeatureWeightsForHypergraph(size_t index, const FeatureFunction* ff
}
return index+numScoreComps;
} else {
- cerr << "Sparse features are not yet supported when outputting hypergraph format" << endl;
- assert(false);
- return 0;
+ UTIL_THROW2("Sparse features are not yet supported when outputting hypergraph format");
}
}
@@ -650,7 +649,7 @@ int main(int argc, char** argv)
boost::filesystem::path nbestPath(nbestFile);
weightsFilename << nbestPath.parent_path().filename() << "/weights";
} else {
- weightsFilename << boost::filesystem::current_path() << "/hypergraph/weights";
+ weightsFilename << boost::filesystem::current_path().string() << "/hypergraph/weights";
}
}
boost::filesystem::path weightsFilePath(weightsFilename.str());
diff --git a/moses/ChartCell.cpp b/moses/ChartCell.cpp
index 6603139f6..125efd204 100644
--- a/moses/ChartCell.cpp
+++ b/moses/ChartCell.cpp
@@ -114,8 +114,11 @@ void ChartCell::SortHypotheses()
MapType::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
ChartHypothesisCollection &coll = iter->second;
- coll.SortHypotheses();
- m_targetLabelSet.AddConstituent(iter->first, &coll.GetSortedHypotheses());
+
+ if (coll.GetSize()) {
+ coll.SortHypotheses();
+ m_targetLabelSet.AddConstituent(iter->first, &coll.GetSortedHypotheses());
+ }
}
}
diff --git a/moses/ChartHypothesis.cpp b/moses/ChartHypothesis.cpp
index 01eb49ccc..212a28d23 100644
--- a/moses/ChartHypothesis.cpp
+++ b/moses/ChartHypothesis.cpp
@@ -66,6 +66,22 @@ ChartHypothesis::ChartHypothesis(const ChartTranslationOptions &transOpt,
}
}
+// Intended to be used by ChartKBestExtractor only. This creates a mock
+// ChartHypothesis for use by the extractor's top-level target vertex.
+ChartHypothesis::ChartHypothesis(const ChartHypothesis &pred,
+ const ChartKBestExtractor & /*unused*/)
+ :m_currSourceWordsRange(pred.m_currSourceWordsRange)
+ ,m_scoreBreakdown(pred.m_scoreBreakdown)
+ ,m_totalScore(pred.m_totalScore)
+ ,m_arcList(NULL)
+ ,m_winningHypo(NULL)
+ ,m_manager(pred.m_manager)
+ ,m_id(pred.m_manager.GetNextHypoId())
+{
+ // One predecessor, which is an existing top-level ChartHypothesis.
+ m_prevHypos.push_back(&pred);
+}
+
ChartHypothesis::~ChartHypothesis()
{
// delete feature function states
diff --git a/moses/ChartHypothesis.h b/moses/ChartHypothesis.h
index 6e2facb9c..12050e764 100644
--- a/moses/ChartHypothesis.h
+++ b/moses/ChartHypothesis.h
@@ -31,6 +31,7 @@
namespace Moses
{
+class ChartKBestExtractor;
class ChartHypothesis;
class ChartManager;
class RuleCubeItem;
@@ -44,6 +45,7 @@ typedef std::vector ChartArcList;
class ChartHypothesis
{
friend std::ostream& operator<<(std::ostream&, const ChartHypothesis&);
+ friend class ChartKBestExtractor;
protected:
#ifdef USE_HYPO_POOL
@@ -74,6 +76,9 @@ protected:
//! not implemented
ChartHypothesis(const ChartHypothesis ©);
+ //! only used by ChartKBestExtractor
+ ChartHypothesis(const ChartHypothesis &, const ChartKBestExtractor &);
+
public:
#ifdef USE_HYPO_POOL
void *operator new(size_t /* num_bytes */) {
diff --git a/moses/ChartKBestExtractor.cpp b/moses/ChartKBestExtractor.cpp
new file mode 100644
index 000000000..3a16198fc
--- /dev/null
+++ b/moses/ChartKBestExtractor.cpp
@@ -0,0 +1,279 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2014 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+***********************************************************************/
+
+#include "ChartKBestExtractor.h"
+
+#include "ChartHypothesis.h"
+#include "ScoreComponentCollection.h"
+#include "StaticData.h"
+
+#include
+
+#include
+
+namespace Moses
+{
+
+// Extract the k-best list from the search graph.
+void ChartKBestExtractor::Extract(
+ const std::vector &topLevelHypos, std::size_t k,
+ KBestVec &kBestList)
+{
+ kBestList.clear();
+ if (topLevelHypos.empty()) {
+ return;
+ }
+
+ // Create a new ChartHypothesis object, supremeHypo, that has the best
+ // top-level hypothesis as its predecessor and has the same score.
+ std::vector::const_iterator p = topLevelHypos.begin();
+ const ChartHypothesis &bestTopLevelHypo = **p;
+ boost::scoped_ptr supremeHypo(
+ new ChartHypothesis(bestTopLevelHypo, *this));
+
+ // Do the same for each alternative top-level hypothesis, but add the new
+ // ChartHypothesis objects as arcs from supremeHypo, as if they had been
+ // recombined.
+ for (++p; p != topLevelHypos.end(); ++p) {
+ // Check that the first item in topLevelHypos really was the best.
+ UTIL_THROW_IF2((*p)->GetTotalScore() <= bestTopLevelHypo.GetTotalScore(),
+ "top-level hypotheses are not correctly sorted");
+ // Note: there's no need for a smart pointer here: supremeHypo will take
+ // ownership of altHypo.
+ ChartHypothesis *altHypo = new ChartHypothesis(**p, *this);
+ supremeHypo->AddArc(altHypo);
+ }
+
+ // Create the target vertex then lazily fill its k-best list.
+ boost::shared_ptr targetVertex = FindOrCreateVertex(*supremeHypo);
+ LazyKthBest(*targetVertex, k, k);
+
+ // Copy the k-best list from the target vertex, but drop the top edge from
+ // each derivation.
+ kBestList.reserve(targetVertex->kBestList.size());
+ for (std::vector >::const_iterator
+ q = targetVertex->kBestList.begin();
+ q != targetVertex->kBestList.end(); ++q) {
+ const boost::shared_ptr d(*q);
+ assert(d);
+ assert(d->subderivations.size() == 1);
+ kBestList.push_back(d->subderivations[0]);
+ }
+}
+
+// Generate the target-side yield of the derivation d.
+Phrase ChartKBestExtractor::GetOutputPhrase(const Derivation &d)
+{
+ FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
+
+ Phrase ret(ARRAY_SIZE_INCR);
+
+ const ChartHypothesis &hypo = d.edge.head->hypothesis;
+ const TargetPhrase &phrase = hypo.GetCurrTargetPhrase();
+ const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
+ phrase.GetAlignNonTerm().GetNonTermIndexMap();
+ for (std::size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+ const Word &word = phrase.GetWord(pos);
+ if (word.IsNonTerminal()) {
+ std::size_t nonTermInd = nonTermIndexMap[pos];
+ const Derivation &subderivation = *d.subderivations[nonTermInd];
+ Phrase subPhrase = GetOutputPhrase(subderivation);
+ ret.Append(subPhrase);
+ } else {
+ ret.AddWord(word);
+ if (placeholderFactor == NOT_FOUND) {
+ continue;
+ }
+ std::set sourcePosSet =
+ phrase.GetAlignTerm().GetAlignmentsForTarget(pos);
+ if (sourcePosSet.size() == 1) {
+ const std::vector *ruleSourceFromInputPath =
+ hypo.GetTranslationOption().GetSourceRuleFromInputPath();
+ UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
+ "Source Words in of the rules hasn't been filled out");
+ std::size_t sourcePos = *sourcePosSet.begin();
+ const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
+ UTIL_THROW_IF2(sourceWord == NULL,
+ "Null source word at position " << sourcePos);
+ const Factor *factor = sourceWord->GetFactor(placeholderFactor);
+ if (factor) {
+ ret.Back()[0] = factor;
+ }
+ }
+ }
+ }
+
+ return ret;
+}
+
+// Create an unweighted hyperarc corresponding to the given ChartHypothesis.
+ChartKBestExtractor::UnweightedHyperarc ChartKBestExtractor::CreateEdge(
+ const ChartHypothesis &h)
+{
+ UnweightedHyperarc edge;
+ edge.head = FindOrCreateVertex(h);
+ const std::vector &prevHypos = h.GetPrevHypos();
+ edge.tail.resize(prevHypos.size());
+ for (std::size_t i = 0; i < prevHypos.size(); ++i) {
+ const ChartHypothesis *prevHypo = prevHypos[i];
+ edge.tail[i] = FindOrCreateVertex(*prevHypo);
+ }
+ return edge;
+}
+
+// Look for the vertex corresponding to a given ChartHypothesis, creating
+// a new one if necessary.
+boost::shared_ptr
+ChartKBestExtractor::FindOrCreateVertex(const ChartHypothesis &h)
+{
+ VertexMap::value_type element(&h, boost::shared_ptr());
+ std::pair p = m_vertexMap.insert(element);
+ boost::shared_ptr &sp = p.first->second;
+ if (!p.second) {
+ return sp; // Vertex was already in m_vertexMap.
+ }
+ sp.reset(new Vertex(h));
+ // Create the 1-best derivation and add it to the vertex's kBestList.
+ UnweightedHyperarc bestEdge;
+ bestEdge.head = sp;
+ const std::vector &prevHypos = h.GetPrevHypos();
+ bestEdge.tail.resize(prevHypos.size());
+ for (std::size_t i = 0; i < prevHypos.size(); ++i) {
+ const ChartHypothesis *prevHypo = prevHypos[i];
+ bestEdge.tail[i] = FindOrCreateVertex(*prevHypo);
+ }
+ boost::shared_ptr bestDerivation(new Derivation(bestEdge));
+ std::pair q =
+ m_derivations.insert(bestDerivation);
+ assert(q.second);
+ sp->kBestList.push_back(bestDerivation);
+ return sp;
+}
+
+// Create the 1-best derivation for each edge in BS(v) (except the best one)
+// and add it to v's candidate queue.
+void ChartKBestExtractor::GetCandidates(Vertex &v, std::size_t k)
+{
+ // Create derivations for all of v's incoming edges except the best. This
+ // means everything in v.hypothesis.GetArcList() and not the edge defined
+ // by v.hypothesis itself. The 1-best derivation for that edge will already
+ // have been created.
+ const ChartArcList *arcList = v.hypothesis.GetArcList();
+ if (arcList) {
+ for (std::size_t i = 0; i < arcList->size(); ++i) {
+ const ChartHypothesis &recombinedHypo = *(*arcList)[i];
+ boost::shared_ptr w = FindOrCreateVertex(recombinedHypo);
+ assert(w->kBestList.size() == 1);
+ v.candidates.push(w->kBestList[0]);
+ }
+ }
+}
+
+// Lazily fill v's k-best list.
+void ChartKBestExtractor::LazyKthBest(Vertex &v, std::size_t k,
+ std::size_t globalK)
+{
+ // If this is the first visit to vertex v then initialize the priority queue.
+ if (v.visited == false) {
+ // The 1-best derivation should already be in v's k-best list.
+ assert(v.kBestList.size() == 1);
+ // Initialize v's priority queue.
+ GetCandidates(v, globalK);
+ v.visited = true;
+ }
+ // Add derivations to the k-best list until it contains k or there are none
+ // left to add.
+ while (v.kBestList.size() < k) {
+ assert(!v.kBestList.empty());
+ // Update the priority queue by adding the successors of the last
+ // derivation (unless they've been seen before).
+ boost::shared_ptr d(v.kBestList.back());
+ LazyNext(v, *d, globalK);
+ // Check if there are any derivations left in the queue.
+ if (v.candidates.empty()) {
+ break;
+ }
+ // Get the next best derivation and delete it from the queue.
+ boost::weak_ptr next = v.candidates.top();
+ v.candidates.pop();
+ // Add it to the k-best list.
+ v.kBestList.push_back(next);
+ }
+}
+
+// Create the neighbours of Derivation d and add them to v's candidate queue.
+void ChartKBestExtractor::LazyNext(Vertex &v, const Derivation &d,
+ std::size_t globalK)
+{
+ for (std::size_t i = 0; i < d.edge.tail.size(); ++i) {
+ Vertex &pred = *d.edge.tail[i];
+ // Ensure that pred's k-best list contains enough derivations.
+ std::size_t k = d.backPointers[i] + 2;
+ LazyKthBest(pred, k, globalK);
+ if (pred.kBestList.size() < k) {
+ // pred's derivations have been exhausted.
+ continue;
+ }
+ // Create the neighbour.
+ boost::shared_ptr next(new Derivation(d, i));
+ // Check if it has been created before.
+ std::pair p = m_derivations.insert(next);
+ if (p.second) {
+ v.candidates.push(next); // Haven't previously seen it.
+ }
+ }
+}
+
+// Construct the 1-best Derivation that ends at edge e.
+ChartKBestExtractor::Derivation::Derivation(const UnweightedHyperarc &e)
+{
+ edge = e;
+ std::size_t arity = edge.tail.size();
+ backPointers.resize(arity, 0);
+ subderivations.reserve(arity);
+ for (std::size_t i = 0; i < arity; ++i) {
+ const Vertex &pred = *edge.tail[i];
+ assert(pred.kBestList.size() >= 1);
+ boost::shared_ptr sub(pred.kBestList[0]);
+ subderivations.push_back(sub);
+ }
+ scoreBreakdown = edge.head->hypothesis.GetScoreBreakdown();
+ score = edge.head->hypothesis.GetTotalScore();
+}
+
+// Construct a Derivation that neighbours an existing Derivation.
+ChartKBestExtractor::Derivation::Derivation(const Derivation &d, std::size_t i)
+{
+ edge.head = d.edge.head;
+ edge.tail = d.edge.tail;
+ backPointers = d.backPointers;
+ subderivations = d.subderivations;
+ std::size_t j = ++backPointers[i];
+ scoreBreakdown = d.scoreBreakdown;
+ // Deduct the score of the old subderivation.
+ scoreBreakdown.MinusEquals(subderivations[i]->scoreBreakdown);
+ // Update the subderivation pointer.
+ boost::shared_ptr newSub(edge.tail[i]->kBestList[j]);
+ subderivations[i] = newSub;
+ // Add the score of the new subderivation.
+ scoreBreakdown.PlusEquals(subderivations[i]->scoreBreakdown);
+ score = scoreBreakdown.GetWeightedScore();
+}
+
+} // namespace Moses
diff --git a/moses/ChartKBestExtractor.h b/moses/ChartKBestExtractor.h
new file mode 100644
index 000000000..05b016d50
--- /dev/null
+++ b/moses/ChartKBestExtractor.h
@@ -0,0 +1,129 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2014 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+***********************************************************************/
+
+#pragma once
+
+#include
+#include "ChartHypothesis.h"
+#include "ScoreComponentCollection.h"
+
+#include
+#include
+
+#include
+#include
+
+namespace Moses
+{
+
+// k-best list extractor that implements algorithm 3 from this paper:
+//
+// Liang Huang and David Chiang
+// "Better k-best parsing"
+// In Proceedings of IWPT 2005
+//
+class ChartKBestExtractor
+{
+public:
+ struct Vertex;
+
+ struct UnweightedHyperarc {
+ boost::shared_ptr head;
+ std::vector > tail;
+ };
+
+ struct Derivation {
+ Derivation(const UnweightedHyperarc &);
+ Derivation(const Derivation &, std::size_t);
+
+ UnweightedHyperarc edge;
+ std::vector backPointers;
+ std::vector > subderivations;
+ ScoreComponentCollection scoreBreakdown;
+ float score;
+ };
+
+ struct DerivationOrderer {
+ bool operator()(const boost::weak_ptr &d1,
+ const boost::weak_ptr &d2) const {
+ boost::shared_ptr s1(d1);
+ boost::shared_ptr s2(d2);
+ return s1->score < s2->score;
+ }
+ };
+
+ struct Vertex {
+ typedef std::priority_queue,
+ std::vector >,
+ DerivationOrderer> DerivationQueue;
+
+ Vertex(const ChartHypothesis &h) : hypothesis(h), visited(false) {}
+
+ const ChartHypothesis &hypothesis;
+ std::vector > kBestList;
+ DerivationQueue candidates;
+ bool visited;
+ };
+
+ typedef std::vector > KBestVec;
+
+ // Extract the k-best list from the search hypergraph given the full, sorted
+ // list of top-level vertices.
+ void Extract(const std::vector &topHypos,
+ std::size_t k, KBestVec &);
+
+ static Phrase GetOutputPhrase(const Derivation &);
+
+private:
+ typedef boost::unordered_map > VertexMap;
+
+ struct DerivationHasher {
+ std::size_t operator()(const boost::shared_ptr &d) const {
+ std::size_t seed = 0;
+ boost::hash_combine(seed, d->edge.head);
+ boost::hash_combine(seed, d->edge.tail);
+ boost::hash_combine(seed, d->backPointers);
+ return seed;
+ }
+ };
+
+ struct DerivationEqualityPred {
+ bool operator()(const boost::shared_ptr &d1,
+ const boost::shared_ptr &d2) const {
+ return d1->edge.head == d2->edge.head &&
+ d1->edge.tail == d2->edge.tail &&
+ d1->backPointers == d2->backPointers;
+ }
+ };
+
+ typedef boost::unordered_set, DerivationHasher,
+ DerivationEqualityPred> DerivationSet;
+
+ UnweightedHyperarc CreateEdge(const ChartHypothesis &);
+ boost::shared_ptr FindOrCreateVertex(const ChartHypothesis &);
+ void GetCandidates(Vertex &, std::size_t);
+ void LazyKthBest(Vertex &, std::size_t, std::size_t);
+ void LazyNext(Vertex &, const Derivation &, std::size_t);
+
+ VertexMap m_vertexMap;
+ DerivationSet m_derivations;
+};
+
+} // namespace Moses
diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp
index 0e303390e..623968dfc 100644
--- a/moses/ChartManager.cpp
+++ b/moses/ChartManager.cpp
@@ -23,6 +23,7 @@
#include "ChartManager.h"
#include "ChartCell.h"
#include "ChartHypothesis.h"
+#include "ChartKBestExtractor.h"
#include "ChartTranslationOptions.h"
#include "ChartTrellisDetourQueue.h"
#include "ChartTrellisNode.h"
@@ -261,6 +262,65 @@ void ChartManager::CalcNBest(size_t count, ChartTrellisPathList &ret,bool onlyDi
}
}
+/** Calculate the n-best paths through the output hypergraph.
+ * Return the list of paths with the variable ret
+ * \param n how may paths to return
+ * \param ret return argument
+ * \param onlyDistinct whether to check for distinct output sentence or not (default - don't check, just return top n-paths)
+ */
+void ChartManager::CalcNBest(
+ std::size_t n,
+ std::vector > &nBestList,
+ bool onlyDistinct) const
+{
+ nBestList.clear();
+ if (n == 0 || m_source.GetSize() == 0) {
+ return;
+ }
+
+ // Get the list of top-level hypotheses, sorted by score.
+ WordsRange range(0, m_source.GetSize()-1);
+ const ChartCell &lastCell = m_hypoStackColl.Get(range);
+ boost::scoped_ptr > topLevelHypos(
+ lastCell.GetAllSortedHypotheses());
+ if (!topLevelHypos) {
+ return;
+ }
+
+ ChartKBestExtractor extractor;
+
+ if (!onlyDistinct) {
+ // Return the n-best list as is, including duplicate translations.
+ extractor.Extract(*topLevelHypos, n, nBestList);
+ return;
+ }
+
+ // Determine how many derivations to extract. If the n-best list is
+ // restricted to distinct translations then this limit should be bigger
+ // than n. The n-best factor determines how much bigger the limit should be,
+ // with 0 being 'unlimited.' This actually sets a large-ish limit in case
+ // too many translations are identical.
+ const StaticData &staticData = StaticData::Instance();
+ const std::size_t nBestFactor = staticData.GetNBestFactor();
+ std::size_t numDerivations = (nBestFactor == 0) ? n*1000 : n*nBestFactor;
+
+ // Extract the derivations.
+ ChartKBestExtractor::KBestVec bigList;
+ bigList.reserve(numDerivations);
+ extractor.Extract(*topLevelHypos, numDerivations, bigList);
+
+ // Copy derivations into nBestList, skipping ones with repeated translations.
+ std::set distinct;
+ for (ChartKBestExtractor::KBestVec::const_iterator p = bigList.begin();
+ nBestList.size() < n && p != bigList.end(); ++p) {
+ boost::shared_ptr derivation = *p;
+ Phrase translation = ChartKBestExtractor::GetOutputPhrase(*derivation);
+ if (distinct.insert(translation).second) {
+ nBestList.push_back(derivation);
+ }
+ }
+}
+
void ChartManager::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
{
size_t size = m_source.GetSize();
diff --git a/moses/ChartManager.h b/moses/ChartManager.h
index 6beffc45e..27914e207 100644
--- a/moses/ChartManager.h
+++ b/moses/ChartManager.h
@@ -30,6 +30,7 @@
#include "SentenceStats.h"
#include "ChartTranslationOptionList.h"
#include "ChartParser.h"
+#include "ChartKBestExtractor.h"
#include
@@ -71,6 +72,7 @@ public:
void AddXmlChartOptions();
const ChartHypothesis *GetBestHypothesis() const;
void CalcNBest(size_t count, ChartTrellisPathList &ret, bool onlyDistinct=0) const;
+ void CalcNBest(size_t n, std::vector > &nBestList, bool onlyDistinct=false) const;
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
void FindReachableHypotheses( const ChartHypothesis *hypo, std::map &reachable ) const; /* auxilliary function for GetSearchGraph */
diff --git a/moses/ChartTranslationOptionList.cpp b/moses/ChartTranslationOptionList.cpp
index e83fbac79..89955dcee 100644
--- a/moses/ChartTranslationOptionList.cpp
+++ b/moses/ChartTranslationOptionList.cpp
@@ -164,8 +164,8 @@ float ChartTranslationOptionList::GetBestScore(const ChartCellLabel *chartCell)
const HypoList *stack = chartCell->GetStack().cube;
assert(stack);
assert(!stack->empty());
- const ChartHypothesis &bestHypo = **(stack->begin());
- return bestHypo.GetTotalScore();
+ const ChartHypothesis &bestHypo = **(stack->begin());
+ return bestHypo.GetTotalScore();
}
void ChartTranslationOptionList::Evaluate(const InputType &input, const InputPath &inputPath)
diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp
index c42d737cc..ddab3df72 100644
--- a/moses/FF/Factory.cpp
+++ b/moses/FF/Factory.cpp
@@ -39,6 +39,8 @@
#include "moses/FF/HyperParameterAsWeight.h"
#include "moses/FF/SetSourcePhrase.h"
#include "CountNonTerms.h"
+#include "ReferenceComparison.h"
+#include "RuleAmbiguity.h"
#include "moses/FF/SkeletonStatelessFF.h"
#include "moses/FF/SkeletonStatefulFF.h"
@@ -181,6 +183,8 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(HyperParameterAsWeight);
MOSES_FNAME(SetSourcePhrase);
MOSES_FNAME(CountNonTerms);
+ MOSES_FNAME(ReferenceComparison);
+ MOSES_FNAME(RuleAmbiguity);
MOSES_FNAME(SkeletonStatelessFF);
MOSES_FNAME(SkeletonStatefulFF);
diff --git a/moses/FF/ReferenceComparison.cpp b/moses/FF/ReferenceComparison.cpp
new file mode 100644
index 000000000..b11d133c2
--- /dev/null
+++ b/moses/FF/ReferenceComparison.cpp
@@ -0,0 +1,11 @@
+#include "ReferenceComparison.h"
+
+namespace Moses
+{
+ReferenceComparison::ReferenceComparison(const std::string &line)
+:StatelessFeatureFunction(0, line)
+{
+}
+
+}
+
diff --git a/moses/FF/ReferenceComparison.h b/moses/FF/ReferenceComparison.h
new file mode 100644
index 000000000..aef7be493
--- /dev/null
+++ b/moses/FF/ReferenceComparison.h
@@ -0,0 +1,47 @@
+#pragma once
+#include
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+// Count how many hypotheses are in each stack, compare score with reference hypo
+// NOT threadsafe.
+class ReferenceComparison : public StatelessFeatureFunction
+{
+public:
+ ReferenceComparison(const std::string &line);
+
+ virtual bool IsUseable(const FactorMask &mask) const
+ { return true; }
+
+ virtual void Evaluate(const Phrase &source
+ , const TargetPhrase &targetPhrase
+ , ScoreComponentCollection &scoreBreakdown
+ , ScoreComponentCollection &estimatedFutureScore) const
+ {}
+
+ virtual void Evaluate(const InputType &input
+ , const InputPath &inputPath
+ , const TargetPhrase &targetPhrase
+ , ScoreComponentCollection &scoreBreakdown
+ , ScoreComponentCollection *estimatedFutureScore = NULL) const
+ {}
+
+ virtual void Evaluate(const Hypothesis& hypo,
+ ScoreComponentCollection* accumulator) const
+ {}
+
+ virtual void EvaluateChart(const ChartHypothesis &hypo,
+ ScoreComponentCollection* accumulator) const
+ {}
+
+ std::vector DefaultWeights() const
+ { return std::vector(); }
+
+protected:
+
+};
+
+}
+
diff --git a/moses/FF/RuleAmbiguity.cpp b/moses/FF/RuleAmbiguity.cpp
new file mode 100644
index 000000000..8f8760d28
--- /dev/null
+++ b/moses/FF/RuleAmbiguity.cpp
@@ -0,0 +1,61 @@
+#include "RuleAmbiguity.h"
+#include "moses/StaticData.h"
+#include "moses/Word.h"
+
+namespace Moses
+{
+RuleAmbiguity::RuleAmbiguity(const std::string &line)
+:StatelessFeatureFunction(1, line)
+,m_sourceSyntax(true)
+{
+}
+
+bool IsAmbiguous(const Word &word, bool sourceSyntax)
+{
+ const Word &inputDefaultNonTerminal = StaticData::Instance().GetInputDefaultNonTerminal();
+ return word.IsNonTerminal() && (!sourceSyntax || word == inputDefaultNonTerminal);
+}
+
+void RuleAmbiguity::Evaluate(const Phrase &source
+ , const TargetPhrase &targetPhrase
+ , ScoreComponentCollection &scoreBreakdown
+ , ScoreComponentCollection &estimatedFutureScore) const
+{
+ // source can't be empty, right?
+ float score = 0;
+
+ int count = 0;
+ for (size_t i = 0; i < source.GetSize() - 0; ++i) {
+ const Word &word = source.GetWord(i);
+ bool ambiguous = IsAmbiguous(word, m_sourceSyntax);
+ if (ambiguous) {
+ ++count;
+ }
+ else {
+ if (count > 0) {
+ score += count;
+ }
+ count = -1;
+ }
+ }
+
+ // 1st & last always adjacent to ambiguity
+ ++count;
+ if (count > 0) {
+ score += count;
+ }
+
+ scoreBreakdown.PlusEquals(this, score);
+}
+
+void RuleAmbiguity::SetParameter(const std::string& key, const std::string& value)
+{
+ if (key == "source-syntax") {
+ m_sourceSyntax = Scan(value);
+ } else {
+ StatelessFeatureFunction::SetParameter(key, value);
+ }
+}
+
+}
+
diff --git a/moses/FF/RuleAmbiguity.h b/moses/FF/RuleAmbiguity.h
new file mode 100644
index 000000000..436e2fa58
--- /dev/null
+++ b/moses/FF/RuleAmbiguity.h
@@ -0,0 +1,44 @@
+#pragma once
+#include
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+// similar to Scope, however, adjacent non-term count as 1 ammbiguity, rather than 2
+class RuleAmbiguity : public StatelessFeatureFunction
+{
+public:
+ RuleAmbiguity(const std::string &line);
+
+ virtual bool IsUseable(const FactorMask &mask) const
+ { return true; }
+
+ virtual void Evaluate(const Phrase &source
+ , const TargetPhrase &targetPhrase
+ , ScoreComponentCollection &scoreBreakdown
+ , ScoreComponentCollection &estimatedFutureScore) const;
+
+ virtual void Evaluate(const InputType &input
+ , const InputPath &inputPath
+ , const TargetPhrase &targetPhrase
+ , ScoreComponentCollection &scoreBreakdown
+ , ScoreComponentCollection *estimatedFutureScore = NULL) const
+ {}
+
+ virtual void Evaluate(const Hypothesis& hypo,
+ ScoreComponentCollection* accumulator) const
+ {}
+
+ virtual void EvaluateChart(const ChartHypothesis &hypo,
+ ScoreComponentCollection* accumulator) const
+ {}
+
+ void SetParameter(const std::string& key, const std::string& value);
+
+protected:
+ bool m_sourceSyntax;
+};
+
+}
+
diff --git a/moses/FF/SetSourcePhrase.h b/moses/FF/SetSourcePhrase.h
index bd07ffbd4..6b391baa4 100644
--- a/moses/FF/SetSourcePhrase.h
+++ b/moses/FF/SetSourcePhrase.h
@@ -5,6 +5,7 @@
namespace Moses
{
+// the only thing this FF does is set TargetPhrase::m_ruleSource so that other FF can use it in Evaluate(Search).
class SetSourcePhrase : public StatelessFeatureFunction
{
public:
diff --git a/moses/Parameter.cpp b/moses/Parameter.cpp
index 9f47cd946..6e4b8a65d 100644
--- a/moses/Parameter.cpp
+++ b/moses/Parameter.cpp
@@ -102,8 +102,8 @@ Parameter::Parameter()
AddParam("output-search-graph", "osg", "Output connected hypotheses of search into specified filename");
AddParam("output-search-graph-extended", "osgx", "Output connected hypotheses of search into specified filename, in extended format");
AddParam("unpruned-search-graph", "usg", "When outputting chart search graph, do not exclude dead ends. Note: stack pruning may have eliminated some hypotheses");
- AddParam("output-search-graph-slf", "slf", "Output connected hypotheses of search into specified directory, one file per sentence, in HTK standard lattice format (SLF)");
- AddParam("output-search-graph-hypergraph", "Output connected hypotheses of search into specified directory, one file per sentence, in a hypergraph format (see Kenneth Heafield's lazy hypergraph decoder)");
+ AddParam("output-search-graph-slf", "slf", "Output connected hypotheses of search into specified directory, one file per sentence, in HTK standard lattice format (SLF) - the flag should be followed byy a directory name, which must exist");
+ AddParam("output-search-graph-hypergraph", "Output connected hypotheses of search into specified directory, one file per sentence, in a hypergraph format (see Kenneth Heafield's lazy hypergraph decoder). This flag is followed by 3 values: 'true (gz|txt|bz) directory-name'");
AddParam("include-lhs-in-search-graph", "lhssg", "When outputting chart search graph, include the label of the LHS of the rule (useful when using syntax)");
#ifdef HAVE_PROTOBUF
AddParam("output-search-graph-pb", "pb", "Write phrase lattice to protocol buffer objects in the specified path.");
diff --git a/moses/Phrase.h b/moses/Phrase.h
index 1de00bfdf..55fb2bdf5 100644
--- a/moses/Phrase.h
+++ b/moses/Phrase.h
@@ -121,6 +121,14 @@ public:
return m_words[GetSize() - 1];
}
+ inline const Word &Front() const {
+ return m_words[0];
+ }
+
+ inline const Word &Back() const {
+ return m_words[GetSize() - 1];
+ }
+
//! particular factor at a particular position
inline const Factor *GetFactor(size_t pos, FactorType factorType) const {
const Word &ptr = m_words[pos];
diff --git a/moses/PrefixTreeMap.cpp b/moses/PrefixTreeMap.cpp
index c8edce726..ee7565d8b 100644
--- a/moses/PrefixTreeMap.cpp
+++ b/moses/PrefixTreeMap.cpp
@@ -5,6 +5,8 @@
#include
#endif
+using namespace std;
+
namespace Moses
{
void GenericCandidate::readBin(FILE* f)
@@ -62,6 +64,17 @@ void Candidates::readBin(FILE* f)
const LabelId PrefixTreeMap::MagicWord = std::numeric_limits::max() - 1;
+//////////////////////////////////////////////////////////////////
+PrefixTreeMap::~PrefixTreeMap() {
+ if(m_FileSrc) {
+ fClose(m_FileSrc);
+ }
+ if(m_FileTgt) {
+ fClose(m_FileTgt);
+ }
+ FreeMemory();
+}
+
void PrefixTreeMap::FreeMemory()
{
@@ -75,20 +88,21 @@ void PrefixTreeMap::FreeMemory()
m_PtrPool.reset();
}
-static WordVoc* ReadVoc(const std::string& filename)
+WordVoc &ReadVoc(std::map &vocs, const std::string& filename)
{
- static std::map vocs;
#ifdef WITH_THREADS
boost::mutex mutex;
boost::mutex::scoped_lock lock(mutex);
#endif
- std::map::iterator vi = vocs.find(filename);
+ std::map::iterator vi = vocs.find(filename);
if (vi == vocs.end()) {
- WordVoc* voc = new WordVoc();
- voc->Read(filename);
- vocs[filename] = voc;
+ WordVoc &voc = vocs[filename];
+ voc.Read(filename);
+ return voc;
+ }
+ else {
+ return vi->second;
}
- return vocs[filename];
}
int PrefixTreeMap::Read(const std::string& fileNameStem, int numVocs)
@@ -133,7 +147,7 @@ int PrefixTreeMap::Read(const std::string& fileNameStem, int numVocs)
sprintf(num, "%d", i);
//m_Voc[i] = new WordVoc();
//m_Voc[i]->Read(ifv + num);
- m_Voc[i] = ReadVoc(ifv + num);
+ m_Voc[i] = &ReadVoc(m_vocs, ifv + num);
}
TRACE_ERR("binary file loaded, default OFF_T: "<< PTF::getDefault()<<"\n");
diff --git a/moses/PrefixTreeMap.h b/moses/PrefixTreeMap.h
index 06066878d..d6262ca65 100644
--- a/moses/PrefixTreeMap.h
+++ b/moses/PrefixTreeMap.h
@@ -99,18 +99,11 @@ public:
PrefixTreeMap() : m_FileSrc(0), m_FileTgt(0) {
PTF::setDefault(InvalidOffT);
}
- ~PrefixTreeMap() {
- if(m_FileSrc) {
- fClose(m_FileSrc);
- }
- if(m_FileTgt) {
- fClose(m_FileTgt);
- }
- FreeMemory();
- }
+ ~PrefixTreeMap();
+
public:
static const LabelId MagicWord;
-public:
+
void FreeMemory();
int Read(const std::string& fileNameStem, int numVocs = -1);
@@ -135,6 +128,7 @@ private:
std::vector m_Voc;
ObjectPool m_PtrPool;
+ std::map m_vocs;
};
}
diff --git a/moses/TranslationModel/RuleTable/LoaderStandard.cpp b/moses/TranslationModel/RuleTable/LoaderStandard.cpp
index 9d2e3fa20..47f7378d1 100644
--- a/moses/TranslationModel/RuleTable/LoaderStandard.cpp
+++ b/moses/TranslationModel/RuleTable/LoaderStandard.cpp
@@ -218,7 +218,7 @@ bool RuleTableLoaderStandard::Load(FormatType format
// parse source & find pt node
// constituent labels
- Word *sourceLHS;
+ Word *sourceLHS = NULL;
Word *targetLHS;
// create target phrase obj
@@ -251,6 +251,9 @@ bool RuleTableLoaderStandard::Load(FormatType format
TargetPhraseCollection &phraseColl = GetOrCreateTargetPhraseCollection(ruleTable, sourcePhrase, *targetPhrase, sourceLHS);
phraseColl.Add(targetPhrase);
+ // not implemented correctly in memory pt. just delete it for now
+ delete sourceLHS;
+
count++;
}
diff --git a/scripts/Transliteration/in-decoding-transliteration.pl b/scripts/Transliteration/in-decoding-transliteration.pl
new file mode 100755
index 000000000..237aec587
--- /dev/null
+++ b/scripts/Transliteration/in-decoding-transliteration.pl
@@ -0,0 +1,230 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+use utf8;
+use File::Basename;
+use Getopt::Long "GetOptions";
+use FindBin qw($RealBin);
+use Scalar::Util qw(looks_like_number);
+use IO::Handle;
+binmode(STDIN, ':utf8');
+binmode(STDOUT, ':utf8');
+binmode(STDERR, ':utf8');
+
+my $___FACTOR_DELIMITER = "|";
+my $OUT_FILE = "/tmp/transliteration-phrase-table.$$";
+
+my ($MOSES_SRC_DIR,$TRANSLIT_MODEL,$OOV_FILE, $OOV_FILE_NAME, $EXTERNAL_BIN_DIR, $LM_FILE, $INPUT_EXTENSION, $OUTPUT_EXTENSION);
+die("ERROR: wrong syntax when invoking in-decoding-transliteration.perl")
+ unless &GetOptions('moses-src-dir=s' => \$MOSES_SRC_DIR,
+ 'external-bin-dir=s' => \$EXTERNAL_BIN_DIR,
+ 'transliteration-model-dir=s' => \$TRANSLIT_MODEL,
+ 'input-extension=s' => \$INPUT_EXTENSION,
+ 'output-extension=s' => \$OUTPUT_EXTENSION,
+ 'transliteration-file=s' => \$OOV_FILE,
+ 'out-file=s' => \$OUT_FILE);
+
+# check if the files are in place
+die("ERROR: you need to define --moses-src-dir --external-bin-dir, --transliteration-model-dir, --transliteration-file, --input-extension, and --output-extension")
+ unless (defined($MOSES_SRC_DIR) &&
+ defined($TRANSLIT_MODEL) &&
+ defined($OOV_FILE) &&
+ defined($INPUT_EXTENSION)&&
+ defined($OUTPUT_EXTENSION)&&
+ defined($EXTERNAL_BIN_DIR));
+
+die("ERROR: could not find Transliteration Model '$TRANSLIT_MODEL'")
+ unless -e $TRANSLIT_MODEL;
+die("ERROR: could not find Transliteration file $OOV_FILE'")
+ unless -e $OOV_FILE;
+
+$OOV_FILE_NAME = basename ($OOV_FILE);
+
+`mkdir $TRANSLIT_MODEL/evaluation`;
+`cp $OOV_FILE $TRANSLIT_MODEL/evaluation/`;
+my $translitFile = $TRANSLIT_MODEL . "/evaluation/" . $OOV_FILE_NAME;
+
+print "Preparing for Transliteration\n";
+prepare_for_transliteration ($OOV_FILE, $translitFile);
+print "Run Transliteration\n";
+run_transliteration ($MOSES_SRC_DIR , $EXTERNAL_BIN_DIR , $TRANSLIT_MODEL , $OOV_FILE_NAME);
+print "Pick Best Transliteration\n";
+form_corpus ($translitFile , $translitFile.".op.nBest" , $OUT_FILE);
+
+
+################### Read the UNK word file and prepare for Transliteration ###############################
+
+sub prepare_for_transliteration
+{
+ my @list = @_;
+ my $testFile = $list[0];
+ my $translitFile = $list[1];
+ my %UNK;
+ my @words;
+ my $src;
+ my @tW;
+
+ open MYFILE, "<:encoding(UTF-8)", $testFile or die "Can't open $testFile: $!\n";
+
+ while ()
+ {
+ chomp;
+ #print "$_\n";
+ @words = split(/ /, "$_");
+
+ foreach (@words)
+ {
+
+ @tW = split /\Q$___FACTOR_DELIMITER/;
+
+ if (defined $tW[0])
+ {
+
+ if (! ($tW[0] =~ /[0-9.,]/))
+ {
+ $UNK{$tW[0]} = 1;
+ }
+ else
+ {
+ print "Not transliterating $tW[0] \n";
+ }
+ }
+ }
+ }
+ close (MYFILE);
+
+ open MYFILE, ">:encoding(UTF-8)", $translitFile or die "Can't open $translitFile: $!\n";
+
+ foreach my $key ( keys %UNK )
+ {
+ $src=join(' ', split('',$key));
+ print MYFILE "$src\n";
+ }
+ close (MYFILE);
+}
+
+################### Run Transliteration Module to Obtain Transliterations ###############################
+
+sub run_transliteration
+{
+ my @list = @_;
+ my $MOSES_SRC = $list[0];
+ my $EXTERNAL_BIN_DIR = $list[1];
+ my $TRANSLIT_MODEL = $list[2];
+ my $eval_file = $list[3];
+
+ `touch $TRANSLIT_MODEL/evaluation/$eval_file.moses.table.ini`;
+
+ print "Filter Table\n";
+
+ `$MOSES_SRC/scripts/training/train-model.perl -mgiza -mgiza-cpus 10 -dont-zip -first-step 9 -external-bin-dir $EXTERNAL_BIN_DIR -f $INPUT_EXTENSION -e $OUTPUT_EXTENSION -alignment grow-diag-final-and -parts 5 -score-options '--KneserNey' -phrase-translation-table $TRANSLIT_MODEL/model/phrase-table -config $TRANSLIT_MODEL/evaluation/$eval_file.moses.table.ini -lm 0:3:$TRANSLIT_MODEL/evaluation/$eval_file.moses.table.ini:8`;
+
+ `$MOSES_SRC/scripts/training/filter-model-given-input.pl $TRANSLIT_MODEL/evaluation/$eval_file.filtered $TRANSLIT_MODEL/evaluation/$eval_file.moses.table.ini $TRANSLIT_MODEL/evaluation/$eval_file -Binarizer "$MOSES_SRC/bin/processPhraseTable"`;
+
+ `rm $TRANSLIT_MODEL/evaluation/$eval_file.moses.table.ini`;
+
+ print "Apply Filter\n";
+
+ `$MOSES_SRC/scripts/ems/support/substitute-filtered-tables-and-weights.perl $TRANSLIT_MODEL/evaluation/$eval_file.filtered/moses.ini $TRANSLIT_MODEL/model/moses.ini $TRANSLIT_MODEL/tuning/moses.tuned.ini $TRANSLIT_MODEL/evaluation/$eval_file.filtered.ini`;
+
+ `$MOSES_SRC/bin/moses -search-algorithm 1 -cube-pruning-pop-limit 5000 -s 5000 -threads 16 -drop-unknown -distortion-limit 0 -n-best-list $TRANSLIT_MODEL/evaluation/$eval_file.op.nBest 100 distinct -f $TRANSLIT_MODEL/evaluation/$eval_file.filtered.ini < $TRANSLIT_MODEL/evaluation/$eval_file > $TRANSLIT_MODEL/evaluation/$eval_file.op`;
+
+}
+
+################### Read the output of Transliteration Model and Form Corpus ###############################
+
+
+sub form_corpus
+{
+
+ my @list = @_;
+ my $inp_file = $list[0];
+ my $testFile = $list[1];
+ my @words;
+ my $thisStr;
+ my $features;
+ my $prev = 0;
+ my $sNum;
+ my @UNK;
+ my %vocab;
+
+ my $antLog = exp(0.2);
+ my $phraseTable = $list[2];
+
+ open MYFILE, "<:encoding(UTF-8)", $inp_file or die "Can't open $inp_file: $!\n";
+ open PT, ">:encoding(UTF-8)", $phraseTable or die "Can't open $phraseTable: $!\n";
+
+ while ()
+ {
+ chomp;
+ #print "$_\n";
+ @words = split(/ /, "$_");
+
+ $thisStr = "";
+ foreach (@words)
+ {
+ $thisStr = $thisStr . "$_";
+ }
+
+ push(@UNK, $thisStr);
+ $vocab{$thisStr} = 1;
+ }
+ close (MYFILE);
+
+ open MYFILE, "<:encoding(UTF-8)", $testFile or die "Can't open $testFile: $!\n";
+ my $inpCount = 0;
+
+ while ()
+ {
+ chomp;
+ #print "$_\n";
+ @words = split(/ /, "$_");
+
+ $sNum = $words[0];
+
+ if ($prev != $sNum){
+ $inpCount++;
+ }
+
+ my $i = 2;
+ $thisStr = "";
+ $features = "";
+
+ while ($words[$i] ne "|||")
+ {
+ $thisStr = $thisStr . $words[$i];
+ $i++;
+ }
+
+ $i++;
+
+ while ($words[$i] ne "|||")
+ {
+ if ($words[$i] =~ /Penalty0/ || $words[$i] eq "Distortion0=" || $words[$i] eq "LM0=" ){
+ $i++;
+ }
+ elsif (looks_like_number($words[$i])){
+ $features = $features . " " . exp($words[$i]);
+ }
+
+ $i++;
+ }
+ $i++;
+
+ #$features = $features . " " . $words[$i];
+
+ if ($thisStr ne ""){
+ print PT "$UNK[$inpCount] ||| $thisStr ||| $features ||| 0-0 ||| 0 0 0\n";
+ }
+ $prev = $sNum;
+ }
+ close (MYFILE);
+ close (PT);
+
+
+ `gzip $phraseTable`;
+
+}
+
+
diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta
index e2b21019d..83d597aa0 100644
--- a/scripts/ems/experiment.meta
+++ b/scripts/ems/experiment.meta
@@ -533,6 +533,13 @@ build-transliteration-model
ignore-unless: transliteration-module
rerun-on-change: transliteration-module training-options script giza-settings
default-name: model/Transliteration
+build-translit-table
+ in: transliteration-model
+ out: transliteration-table
+ ignore-unless: in-decoding-transliteration
+ rerun-on-change: in-decoding-transliteration transliteration-module
+ default-name: model/transliteration-phrase-table
+ template: $moses-script-dir/Transliteration/in-decoding-transliteration.pl --moses-src-dir $moses-src-dir --external-bin-dir $external-bin-dir --transliteration-model-dir IN --input-extension $input-extension --output-extension $output-extension --transliteration-file $transliteration-file --out-file OUT
extract-phrases
in: corpus-mml-postfilter=OR=word-alignment scored-corpus
out: extracted-phrases
@@ -601,7 +608,7 @@ build-sparse
default-name: model/sparse-features
template: $moses-script-dir/ems/support/build-sparse-features.perl IN $input-extension $output-extension OUT "$sparse-features"
create-config
- in: sigtest-filter-reordering-table sigtest-filter-phrase-translation-table transliteration-model generation-table sparse corpus-mml-prefilter=OR=corpus-mml-postfilter=OR=domains osm-model INTERPOLATED-LM:binlm LM:binlm
+ in: sigtest-filter-reordering-table sigtest-filter-phrase-translation-table transliteration-table generation-table sparse corpus-mml-prefilter=OR=corpus-mml-postfilter=OR=domains osm-model INTERPOLATED-LM:binlm LM:binlm
out: config
ignore-if: use-hiero
rerun-on-change: decoding-steps alignment-factors translation-factors reordering-factors generation-factors lexicalized-reordering training-options script decoding-graph-backoff score-settings additional-ini
@@ -863,7 +870,7 @@ split-reference-devtest
multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl
template: $output-splitter -model IN1.$output-extension < IN > OUT
filter
- in: input TRAINING:sigtest-filter-phrase-translation-table TRAINING:sigtest-filter-reordering-table TRAINING:corpus-mml-prefilter=OR=TRAINING:corpus-mml-postfilter=OR=TRAINING:domains
+ in: input TRAINING:sigtest-filter-phrase-translation-table TRAINING:sigtest-filter-reordering-table TRAINING:corpus-mml-prefilter=OR=TRAINING:corpus-mml-postfilter=OR=TRAINING:domains TRAINING:transliteration-table
out: filtered-dir
default-name: tuning/filtered
rerun-on-change: filter-settings ttable-binarizer
@@ -989,8 +996,8 @@ split-input
pass-unless: input-splitter
template: $input-splitter -model IN1.$input-extension < IN > OUT
filter
- in: input TRAINING:sigtest-filter-phrase-translation-table TRAINING:sigtest-filter-reordering-table TRAINING:corpus-mml-prefilter=OR=TRAINING:corpus-mml-postfilter=OR=TRAINING:domains
- out: filtered-dir
+ in: input TRAINING:sigtest-filter-phrase-translation-table TRAINING:sigtest-filter-reordering-table TRAINING:corpus-mml-prefilter=OR=TRAINING:corpus-mml-postfilter=OR=TRAINING:domains TRAINING:transliteration-table
+ out: filtered-dir
default-name: evaluation/filtered
rerun-on-change: filter-settings report-precision-by-coverage ttable-binarizer
pass-if: TRAINING:binarize-all
@@ -1027,11 +1034,11 @@ remove-markup
pass-unless: report-segmentation
template: $moses-script-dir/ems/support/remove-segmentation-markup.perl < IN > OUT
post-decoding-transliteration
- in: cleaned-output system-output TRAINING:transliteration-model LM:binlm
+ in: cleaned-output system-output TRAINING:transliteration-model
out: transliterated-output
default-name: evaluation/transliterated
pass-unless: TRAINING:post-decoding-transliteration
- template: $moses-script-dir/Transliteration/post-decoding-transliteration.pl --moses-src-dir $moses-src-dir --external-bin-dir $external-bin-dir --transliteration-model-dir IN2 --input-extension $input-extension --output-extension $output-extension --language-model IN3 --output-file IN0 --oov-file IN1.oov
+ template: $moses-script-dir/Transliteration/post-decoding-transliteration.pl --moses-src-dir $moses-src-dir --external-bin-dir $external-bin-dir --transliteration-model-dir IN2 --input-extension $input-extension --output-extension $output-extension --language-model $TRAINING:language-model-file --output-file IN0 --oov-file IN1.oov
recase-output
in: transliterated-output RECASING:recase-config
out: recased-output
diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl
index 761c7a694..f6a7e4db3 100755
--- a/scripts/ems/experiment.perl
+++ b/scripts/ems/experiment.perl
@@ -2233,11 +2233,15 @@ sub get_config_tables {
sub define_training_create_config {
my ($step_id) = @_;
- my ($config,$reordering_table,$phrase_translation_table,$translit_model,$generation_table,$sparse_lexical_features,$domains,$osm, @LM)
+ my ($config,$reordering_table,$phrase_translation_table,$transliteration_pt,$generation_table,$sparse_lexical_features,$domains,$osm, @LM)
= &get_output_and_input($step_id);
my $cmd = &get_config_tables($config,$reordering_table,$phrase_translation_table,$generation_table,$domains);
+ if($transliteration_pt){
+ $cmd .= "-transliteration-phrase-table $transliteration_pt ";
+ }
+
if($osm){
my $osm_settings = &get("TRAINING:operation-sequence-model-settings");
@@ -2623,7 +2627,7 @@ sub define_tuningevaluation_filter {
my $tuning_flag = !defined($set);
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
- my ($filter_dir,$input,$phrase_translation_table,$reordering_table,$domains) = &get_output_and_input($step_id);
+ my ($filter_dir,$input,$phrase_translation_table,$reordering_table,$domains,$transliteration_table) = &get_output_and_input($step_id);
my $binarizer;
$binarizer = &backoff_and_get("EVALUATION:$set:ttable-binarizer") unless $tuning_flag;
@@ -2683,7 +2687,14 @@ sub define_tuningevaluation_filter {
$cmd .= &get_config_tables($config,$reordering_table,$phrase_translation_table,undef,$domains);
+ if (&get("TRAINING:in-decoding-transliteration")) {
+
+ $cmd .= "-transliteration-phrase-table $dir/model/transliteration-phrase-table.$VERSION ";
+ }
+
+
$cmd .= "-lm 0:3:$config:8\n"; # dummy kenlm 3-gram model on factor 0
+
}
# filter command
diff --git a/scripts/ems/web/analysis.php b/scripts/ems/web/analysis.php
index 726e30fbd..a64d5977f 100644
--- a/scripts/ems/web/analysis.php
+++ b/scripts/ems/web/analysis.php
@@ -436,7 +436,8 @@ function ngram_summary() {
$score_line = "";
for($i=0;$iresult[$set]);
for($i=0;$i'.$title."
\n";
}
-if (array_key_exists("setup",$_POST) || array_key_exists("setup",$_GET)) {
+if (array_key_exists("setStepStatus",$_GET)) { set_step_status($_GET["setStepStatus"]); }
+else if (array_key_exists("setup",$_POST) || array_key_exists("setup",$_GET)) {
load_experiment_info();
load_comment();
diff --git a/scripts/ems/web/overview.php b/scripts/ems/web/overview.php
index c49e74be6..e56ed6f08 100644
--- a/scripts/ems/web/overview.php
+++ b/scripts/ems/web/overview.php
@@ -295,7 +295,8 @@ function output_score($id,$info) {
$each_score = explode(" ; ",$score);
for($i=0;$i0) { print "
"; }
$opened_a_tag = 0;
if ($set != "avg") {
diff --git a/scripts/other/delete-scores.perl b/scripts/other/delete-scores.perl
new file mode 100755
index 000000000..442173026
--- /dev/null
+++ b/scripts/other/delete-scores.perl
@@ -0,0 +1,61 @@
+#!/usr/bin/perl
+
+use strict;
+use Getopt::Long "GetOptions";
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+
+sub trim($);
+sub DeleteScore;
+
+my $keepScoresStr;
+GetOptions(
+ "keep-scores=s" => \$keepScoresStr
+) or exit(1);
+
+my @keepScores = split(/,/, $keepScoresStr);
+
+#MAIN LOOP
+while (my $line = ) {
+ chomp($line);
+ #print STDERR "line=$line\n";
+
+ my @toks = split(/\|/, $line);
+ my @scores = split(/ /, $toks[6]);
+
+ $toks[6] = DeleteScore($toks[6], \@keepScores);
+
+ # output
+ print $toks[0];
+ for (my $i = 1; $i < scalar(@toks); ++$i) {
+ print "|" .$toks[$i];
+ }
+ print "\n";
+}
+
+######################
+# Perl trim function to remove whitespace from the start and end of the string
+sub trim($) {
+ my $string = shift;
+ $string =~ s/^\s+//;
+ $string =~ s/\s+$//;
+ return $string;
+}
+
+sub DeleteScore
+{
+ my $string = $_[0];
+ my @keepScores = @{$_[1]};
+
+ $string = trim($string);
+ my @toks = split(/ /, $string);
+
+ $string = "";
+ for (my $i = 0; $i < scalar(@keepScores); ++$i) {
+ $string .= $toks[ $keepScores[$i] ] ." ";
+ }
+ $string = " " .$string;
+
+ return $string;
+}
diff --git a/scripts/training/train-model.perl b/scripts/training/train-model.perl
index 3764ab0c2..46a7e1fe6 100755
--- a/scripts/training/train-model.perl
+++ b/scripts/training/train-model.perl
@@ -31,7 +31,7 @@ my($_EXTERNAL_BINDIR, $_ROOT_DIR, $_CORPUS_DIR, $_GIZA_E2F, $_GIZA_F2E, $_MODEL_
$_DECODING_GRAPH_BACKOFF,
$_DECODING_STEPS, $_PARALLEL, $_FACTOR_DELIMITER, @_PHRASE_TABLE,
@_REORDERING_TABLE, @_GENERATION_TABLE, @_GENERATION_TYPE, $_GENERATION_CORPUS,
- $_DONT_ZIP, $_MGIZA, $_MGIZA_CPUS, $_SNT2COOC, $_HMM_ALIGN, $_CONFIG, $_OSM, $_OSM_FACTORS, $_POST_DECODING_TRANSLIT,
+ $_DONT_ZIP, $_MGIZA, $_MGIZA_CPUS, $_SNT2COOC, $_HMM_ALIGN, $_CONFIG, $_OSM, $_OSM_FACTORS, $_POST_DECODING_TRANSLIT, $_TRANSLITERATION_PHRASE_TABLE,
$_HIERARCHICAL,$_XML,$_SOURCE_SYNTAX,$_TARGET_SYNTAX,$_GLUE_GRAMMAR,$_GLUE_GRAMMAR_FILE,$_UNKNOWN_WORD_LABEL_FILE,$_GHKM,$_GHKM_TREE_FRAGMENTS,$_PCFG,@_EXTRACT_OPTIONS,@_SCORE_OPTIONS,
$_ALT_DIRECT_RULE_SCORE_1, $_ALT_DIRECT_RULE_SCORE_2, $_UNKNOWN_WORD_SOFT_MATCHES_FILE,
$_OMIT_WORD_ALIGNMENT,$_FORCE_FACTORED_FILENAMES,
@@ -122,7 +122,8 @@ $_HELP = 1
'config=s' => \$_CONFIG,
'osm-model=s' => \$_OSM,
'osm-setting=s' => \$_OSM_FACTORS,
- 'post-decoding-translit=s' => \$_POST_DECODING_TRANSLIT,
+ 'post-decoding-translit=s' => \$_POST_DECODING_TRANSLIT,
+ 'transliteration-phrase-table=s' => \$_TRANSLITERATION_PHRASE_TABLE,
'max-lexical-reordering' => \$_MAX_LEXICAL_REORDERING,
'do-steps=s' => \$_DO_STEPS,
'memscore:s' => \$_MEMSCORE,
@@ -1879,6 +1880,8 @@ sub create_ini {
$path++;
}
print INI "1 T 1\n" if $_GLUE_GRAMMAR;
+
+ print INI "1 T 1\n" if $_TRANSLITERATION_PHRASE_TABLE;
if (defined($_DECODING_GRAPH_BACKOFF)) {
$_DECODING_GRAPH_BACKOFF =~ s/\s+/ /g;
@@ -1962,6 +1965,13 @@ sub create_ini {
exit 1 if $i < $stepsused{"T"}; # fatal to define less
}
+ if ($_TRANSLITERATION_PHRASE_TABLE){
+
+ $feature_spec .= "PhraseDictionaryMemory name=TranslationModel$i table-limit=100 num-features=4 path=$_TRANSLITERATION_PHRASE_TABLE input-factor=0 output-factor=0\n";
+ $weight_spec .= "TranslationModel$i= 0.2 0.2 0.2 0.2\n";
+ $i++;
+ }
+
# glue grammar
if ($_GLUE_GRAMMAR) {
&full_path(\$___GLUE_GRAMMAR_FILE);
@@ -2069,8 +2079,9 @@ sub create_ini {
my $lm_oov_prob = 0.1;
- if ($_POST_DECODING_TRANSLIT){
+ if ($_POST_DECODING_TRANSLIT || $_TRANSLITERATION_PHRASE_TABLE){
$lm_oov_prob = -100.0;
+ $_LMODEL_OOV_FEATURE = "yes";
}
$feature_spec .= "$type_name name=LM$i factor=$f path=$fn order=$o\n";