moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-26 19:37:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

add PhraseDictionaryMemoryPerSentence

Browse Source
This commit is contained in:
Hieu Hoang 2015-12-29 20:02:25 +00:00
parent 2042fda961
commit 11809630ea
4 changed files with 179 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
contrib/other-builds/moses/.project
View File

@ -2205,6 +2205,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemory.h</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryMemoryPerSentence.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryMemoryPerSentence.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryMultiModel.cpp</name>
<type>1</type>
@ -3500,16 +3510,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-describe-features.cc</locationURI>
</link>
<link>
<name>TranslationModel/UG/ptable-lookup-corpus.cc</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-lookup-corpus.cc</locationURI>
</link>
<link>
<name>TranslationModel/UG/ptable-lookup.cc</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-lookup.cc</locationURI>
</link>
<link>
<name>TranslationModel/UG/ptable-lookup-corpus.cc</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-lookup-corpus.cc</locationURI>
</link>
</link>
<link>
<name>TranslationModel/UG/sapt_phrase_scorers.h</name>
<type>1</type>
@ -5585,21 +5595,21 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup</locationURI>
</link>
<link>
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</locationURI>
</link>
<link>
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</locationURI>
</link>
<link>
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup.o</locationURI>
</link>
<link>
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</locationURI>
</link>
<link>
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</locationURI>
</link>
</link>
<link>
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/spe-check-coverage</name>
<type>1</type>

2
moses/FF/Factory.cpp
View File

@ -15,6 +15,7 @@
#include "moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h"
#include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h"
#include "moses/TranslationModel/ProbingPT/ProbingPT.h"
#include "moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h"
#include "moses/FF/LexicalReordering/LexicalReordering.h"
@ -223,6 +224,7 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(PhraseDictionaryDynamicCacheBased);
MOSES_FNAME(PhraseDictionaryFuzzyMatch);
MOSES_FNAME(ProbingPT);
MOSES_FNAME(PhraseDictionaryMemoryPerSentence);
MOSES_FNAME2("RuleTable", Syntax::RuleTableFF);
MOSES_FNAME2("SyntaxInputWeight", Syntax::InputWeightFF);

108
moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp Normal file
View File

@ -0,0 +1,108 @@
// vim:tabstop=2
#include "PhraseDictionaryMemoryPerSentence.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
using namespace std;
namespace Moses
{
PhraseDictionaryMemoryPerSentence::PhraseDictionaryMemoryPerSentence(const std::string &line)
: PhraseDictionary(line, true)
{
ReadParameters();
}
void PhraseDictionaryMemoryPerSentence::Load(AllOptions::ptr const& opts)
{
m_options = opts;
SetFeaturesToApply();
// don't load anything. Load when we have the input
}
void PhraseDictionaryMemoryPerSentence::InitializeForInput(ttasksptr const& ttask)
{
Coll &coll = GetColl();
coll.clear();
string filePath = m_filePath + SPrint(ttask.get()->GetSource()->GetTranslationId()) + ".txt";
InputFileStream strme(filePath);
string line;
while (getline(strme, line)) {
vector<string> toks = TokenizeMultiCharSeparator(line, "|||");
Phrase source;
source.CreateFromString(Input, m_input, toks[0], NULL);
TargetPhrase *target = new TargetPhrase(this);
target->CreateFromString(Output, m_output, toks[1], NULL);
// score for this phrase table
vector<float> scores = Tokenize<float>(toks[2]);
std::transform(scores.begin(), scores.end(), scores.begin(),TransformScore);
std::transform(scores.begin(), scores.end(), scores.begin(),FloorScore);
target->GetScoreBreakdown().PlusEquals(this, scores);
// score of all other ff when this rule is being loaded
target->EvaluateInIsolation(source, GetFeaturesToApply());
// add to coll
TargetPhraseCollection::shared_ptr &tpsPtr = coll[source];
TargetPhraseCollection *tps = tpsPtr.get();
if (tps == NULL) {
tps = new TargetPhraseCollection();
tpsPtr.reset(tps);
}
tps->Add(target);
}
}
void PhraseDictionaryMemoryPerSentence::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
{
InputPathList::const_iterator iter;
for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
InputPath &inputPath = **iter;
const Phrase &source = inputPath.GetPhrase();
Coll &coll = GetColl();
Coll::const_iterator iter = coll.find(source);
if (iter == coll.end()) {
TargetPhraseCollection::shared_ptr tprPtr;
inputPath.SetTargetPhrases(*this, tprPtr, NULL);
}
else {
const TargetPhraseCollection::shared_ptr &tprPtr = iter->second;
inputPath.SetTargetPhrases(*this, tprPtr, NULL);
}
}
}
ChartRuleLookupManager* PhraseDictionaryMemoryPerSentence::CreateRuleLookupManager(const ChartParser &parser,
const ChartCellCollectionBase &cellCollection,
std::size_t /*maxChartSpan*/)
{
abort();
}
PhraseDictionaryMemoryPerSentence::Coll &PhraseDictionaryMemoryPerSentence::GetColl() const
{
Coll *coll;
coll = m_coll.get();
if (coll == NULL) {
coll = new Coll;
m_coll.reset(coll);
}
assert(coll);
return *coll;
}
TO_STRING_BODY(PhraseDictionaryMemoryPerSentence);
// friend
ostream& operator<<(ostream& out, const PhraseDictionaryMemoryPerSentence& phraseDict)
{
return out;
}
}

42
moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#include "PhraseDictionary.h"
#include "moses/TypeDef.h"
#include "moses/TranslationTask.h"
namespace Moses
{
class ChartParser;
class ChartCellCollectionBase;
class ChartRuleLookupManager;
class PhraseDictionaryMemoryPerSentence : public PhraseDictionary
{
friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryMemoryPerSentence&);
public:
PhraseDictionaryMemoryPerSentence(const std::string &line);
void Load(AllOptions::ptr const& opts);
void InitializeForInput(ttasksptr const& ttask);
// for phrase-based model
void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const;
// for syntax/hiero model (CKY+ decoding)
ChartRuleLookupManager* CreateRuleLookupManager(const ChartParser&, const ChartCellCollectionBase&, std::size_t);
TO_STRING();
protected:
typedef boost::unordered_map<Phrase, TargetPhraseCollection::shared_ptr> Coll;
mutable boost::thread_specific_ptr<Coll> m_coll;
Coll &GetColl() const;
};
} // namespace Moses