mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-26 19:37:58 +03:00
add PhraseDictionaryMemoryPerSentence
This commit is contained in:
parent
2042fda961
commit
11809630ea
@ -2205,6 +2205,16 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemory.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/PhraseDictionaryMemoryPerSentence.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/PhraseDictionaryMemoryPerSentence.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/PhraseDictionaryMultiModel.cpp</name>
|
||||
<type>1</type>
|
||||
@ -3500,16 +3510,16 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-describe-features.cc</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/ptable-lookup-corpus.cc</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-lookup-corpus.cc</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/ptable-lookup.cc</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-lookup.cc</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/ptable-lookup-corpus.cc</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-lookup-corpus.cc</locationURI>
|
||||
</link>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/sapt_phrase_scorers.h</name>
|
||||
<type>1</type>
|
||||
@ -5585,21 +5595,21 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup.o</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup.o</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ptable-lookup-corpus.o</locationURI>
|
||||
</link>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/UG/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/spe-check-coverage</name>
|
||||
<type>1</type>
|
||||
|
@ -15,6 +15,7 @@
|
||||
#include "moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h"
|
||||
#include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h"
|
||||
#include "moses/TranslationModel/ProbingPT/ProbingPT.h"
|
||||
#include "moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h"
|
||||
|
||||
#include "moses/FF/LexicalReordering/LexicalReordering.h"
|
||||
|
||||
@ -223,6 +224,7 @@ FeatureRegistry::FeatureRegistry()
|
||||
MOSES_FNAME(PhraseDictionaryDynamicCacheBased);
|
||||
MOSES_FNAME(PhraseDictionaryFuzzyMatch);
|
||||
MOSES_FNAME(ProbingPT);
|
||||
MOSES_FNAME(PhraseDictionaryMemoryPerSentence);
|
||||
MOSES_FNAME2("RuleTable", Syntax::RuleTableFF);
|
||||
MOSES_FNAME2("SyntaxInputWeight", Syntax::InputWeightFF);
|
||||
|
||||
|
108
moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp
Normal file
108
moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp
Normal file
@ -0,0 +1,108 @@
|
||||
// vim:tabstop=2
|
||||
#include "PhraseDictionaryMemoryPerSentence.h"
|
||||
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
PhraseDictionaryMemoryPerSentence::PhraseDictionaryMemoryPerSentence(const std::string &line)
|
||||
: PhraseDictionary(line, true)
|
||||
{
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
void PhraseDictionaryMemoryPerSentence::Load(AllOptions::ptr const& opts)
|
||||
{
|
||||
m_options = opts;
|
||||
SetFeaturesToApply();
|
||||
|
||||
// don't load anything. Load when we have the input
|
||||
}
|
||||
|
||||
void PhraseDictionaryMemoryPerSentence::InitializeForInput(ttasksptr const& ttask)
|
||||
{
|
||||
Coll &coll = GetColl();
|
||||
coll.clear();
|
||||
|
||||
string filePath = m_filePath + SPrint(ttask.get()->GetSource()->GetTranslationId()) + ".txt";
|
||||
InputFileStream strme(filePath);
|
||||
|
||||
string line;
|
||||
while (getline(strme, line)) {
|
||||
vector<string> toks = TokenizeMultiCharSeparator(line, "|||");
|
||||
Phrase source;
|
||||
source.CreateFromString(Input, m_input, toks[0], NULL);
|
||||
|
||||
TargetPhrase *target = new TargetPhrase(this);
|
||||
target->CreateFromString(Output, m_output, toks[1], NULL);
|
||||
|
||||
// score for this phrase table
|
||||
vector<float> scores = Tokenize<float>(toks[2]);
|
||||
std::transform(scores.begin(), scores.end(), scores.begin(),TransformScore);
|
||||
std::transform(scores.begin(), scores.end(), scores.begin(),FloorScore);
|
||||
target->GetScoreBreakdown().PlusEquals(this, scores);
|
||||
|
||||
// score of all other ff when this rule is being loaded
|
||||
target->EvaluateInIsolation(source, GetFeaturesToApply());
|
||||
|
||||
// add to coll
|
||||
TargetPhraseCollection::shared_ptr &tpsPtr = coll[source];
|
||||
TargetPhraseCollection *tps = tpsPtr.get();
|
||||
if (tps == NULL) {
|
||||
tps = new TargetPhraseCollection();
|
||||
tpsPtr.reset(tps);
|
||||
}
|
||||
tps->Add(target);
|
||||
}
|
||||
}
|
||||
|
||||
void PhraseDictionaryMemoryPerSentence::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
|
||||
{
|
||||
InputPathList::const_iterator iter;
|
||||
for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
|
||||
InputPath &inputPath = **iter;
|
||||
const Phrase &source = inputPath.GetPhrase();
|
||||
|
||||
Coll &coll = GetColl();
|
||||
Coll::const_iterator iter = coll.find(source);
|
||||
if (iter == coll.end()) {
|
||||
TargetPhraseCollection::shared_ptr tprPtr;
|
||||
inputPath.SetTargetPhrases(*this, tprPtr, NULL);
|
||||
}
|
||||
else {
|
||||
const TargetPhraseCollection::shared_ptr &tprPtr = iter->second;
|
||||
inputPath.SetTargetPhrases(*this, tprPtr, NULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ChartRuleLookupManager* PhraseDictionaryMemoryPerSentence::CreateRuleLookupManager(const ChartParser &parser,
|
||||
const ChartCellCollectionBase &cellCollection,
|
||||
std::size_t /*maxChartSpan*/)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
PhraseDictionaryMemoryPerSentence::Coll &PhraseDictionaryMemoryPerSentence::GetColl() const
|
||||
{
|
||||
Coll *coll;
|
||||
coll = m_coll.get();
|
||||
if (coll == NULL) {
|
||||
coll = new Coll;
|
||||
m_coll.reset(coll);
|
||||
}
|
||||
assert(coll);
|
||||
return *coll;
|
||||
}
|
||||
|
||||
TO_STRING_BODY(PhraseDictionaryMemoryPerSentence);
|
||||
|
||||
// friend
|
||||
ostream& operator<<(ostream& out, const PhraseDictionaryMemoryPerSentence& phraseDict)
|
||||
{
|
||||
return out;
|
||||
}
|
||||
|
||||
}
|
42
moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h
Normal file
42
moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h
Normal file
@ -0,0 +1,42 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "PhraseDictionary.h"
|
||||
#include "moses/TypeDef.h"
|
||||
#include "moses/TranslationTask.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class ChartParser;
|
||||
class ChartCellCollectionBase;
|
||||
class ChartRuleLookupManager;
|
||||
|
||||
class PhraseDictionaryMemoryPerSentence : public PhraseDictionary
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryMemoryPerSentence&);
|
||||
|
||||
public:
|
||||
PhraseDictionaryMemoryPerSentence(const std::string &line);
|
||||
|
||||
void Load(AllOptions::ptr const& opts);
|
||||
|
||||
void InitializeForInput(ttasksptr const& ttask);
|
||||
|
||||
// for phrase-based model
|
||||
void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const;
|
||||
|
||||
// for syntax/hiero model (CKY+ decoding)
|
||||
ChartRuleLookupManager* CreateRuleLookupManager(const ChartParser&, const ChartCellCollectionBase&, std::size_t);
|
||||
|
||||
TO_STRING();
|
||||
|
||||
|
||||
protected:
|
||||
typedef boost::unordered_map<Phrase, TargetPhraseCollection::shared_ptr> Coll;
|
||||
mutable boost::thread_specific_ptr<Coll> m_coll;
|
||||
|
||||
Coll &GetColl() const;
|
||||
|
||||
};
|
||||
|
||||
} // namespace Moses
|
Loading…
Reference in New Issue
Block a user