moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-26 05:14:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

start merging PhraseDictionaryMemory and PhraseDictionarySCFG

Browse Source
This commit is contained in:
Hieu Hoang 2013-03-07 19:12:49 +00:00
parent 1f6592aaa9
commit 66ad6f32ce
6 changed files with 45 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
contrib/other-builds/moses/.project
View File

@ -1366,191 +1366,26 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/lm.log</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/BlockHashIndex.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/BlockHashIndex.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/BlockHashIndex.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/BlockHashIndex.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/CanonicalHuffman.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/CanonicalHuffman.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/CmphStringVectorAdapter.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/CmphStringVectorAdapter.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/CmphStringVectorAdapter.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/ConsistentPhrases.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/ConsistentPhrases.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/Jamfile</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/Jamfile</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/LexicalReorderingTableCompact.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/LexicalReorderingTableCompact.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/LexicalReorderingTableCreator.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/LexicalReorderingTableCreator.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/LexicalReorderingTableCreator.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/LexicalReorderingTableCreator.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/ListCoders.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/ListCoders.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/MmapAllocator.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/MmapAllocator.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/MonotonicVector.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/MonotonicVector.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/MurmurHash3.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/MurmurHash3.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/MurmurHash3.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/MurmurHash3.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/PackedArray.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/PackedArray.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/PhraseDecoder.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/PhraseDecoder.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/PhraseDecoder.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/PhraseDecoder.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/PhraseDictionaryCompact.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/PhraseDictionaryCompact.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/PhraseTableCreator.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/PhraseTableCreator.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/PhraseTableCreator.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/PhraseTableCreator.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/StringVector.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/StringVector.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/TargetPhraseCollectionCache.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/TargetPhraseCollectionCache.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/ThrowingFwrite.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/ThrowingFwrite.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/ThrowingFwrite.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/ThrowingFwrite.h</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.6/release</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/pt.log</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/pt.log</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.6/release/debug-symbols-on</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.6/release/debug-symbols-on/link-static</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/Factory.o</name>
<type>1</type>
@ -1561,55 +1396,5 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ORLM.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BlockHashIndex.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BlockHashIndex.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/CmphStringVectorAdapter.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/CmphStringVectorAdapter.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/LexicalReorderingTableCompact.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/LexicalReorderingTableCompact.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/LexicalReorderingTableCreator.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/LexicalReorderingTableCreator.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/MurmurHash3.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/MurmurHash3.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/PhraseDecoder.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/PhraseDecoder.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryCompact.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryCompact.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/PhraseTableCreator.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/PhraseTableCreator.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPt/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/ThrowingFwrite.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/ThrowingFwrite.o</locationURI>
</link>
</linkedResources>
</projectDescription>

3
moses/Parameter.cpp
View File

@ -477,7 +477,8 @@ void Parameter::ConvertWeightArgsPhraseModel(const string &oldWeightName)
switch (implementation)
{
case Memory:
ptType = "PhraseDictionarySCFG";
ptType = "PhraseDictionaryMemory";
//ptType = "PhraseDictionarySCFG";
break;
case Binary:
ptType = "PhraseDictionaryTreeAdaptor";

26
moses/Phrase.cpp
View File

@ -189,9 +189,26 @@ void Phrase::CreateFromStringNewFormat(FactorDirection direction
// to
// "KOMMA|none" "ART|Def.Z" "NN|Neut.NotGen.Sg" "VVFIN|none"
m_words.reserve(annotatedWordVector.size()-1);
size_t numWords;
const StringPiece &annotatedWord = annotatedWordVector.back();
if (annotatedWord.size() >= 2
&& *annotatedWord.data() == '['
&& annotatedWord.data()[annotatedWord.size() - 1] == ']') {
for (size_t phrasePos = 0 ; phrasePos < annotatedWordVector.size() - 1 ; phrasePos++) {
numWords = annotatedWordVector.size()-1;
// lhs
lhs.CreateFromString(direction, factorOrder, annotatedWord.substr(1, annotatedWord.size() - 2), true);
assert(lhs.IsNonTerminal());
}
else {
numWords = annotatedWordVector.size();
}
// parse each word
m_words.reserve(numWords);
for (size_t phrasePos = 0 ; phrasePos < numWords; phrasePos++) {
StringPiece &annotatedWord = annotatedWordVector[phrasePos];
bool isNonTerminal;
if (annotatedWord.size() >= 2 && *annotatedWord.data() == '[' && annotatedWord.data()[annotatedWord.size() - 1] == ']') {
@ -214,12 +231,7 @@ void Phrase::CreateFromStringNewFormat(FactorDirection direction
}
// lhs
const StringPiece &annotatedWord = annotatedWordVector.back();
CHECK(annotatedWord.size() >= 2 && *annotatedWord.data() == '[' && annotatedWord.data()[annotatedWord.size() - 1] == ']');
lhs.CreateFromString(direction, factorOrder, annotatedWord.substr(1, annotatedWord.size() - 2), true);
assert(lhs.IsNonTerminal());
}
int Phrase::Compare(const Phrase &other) const

8
moses/TranslationModel/RuleTable/LoaderStandard.cpp
View File

@ -186,13 +186,17 @@ bool RuleTableLoaderStandard::Load(FormatType format
line = hiero_after;
}
cerr << "line=" << line << endl;
util::TokenIter<util::MultiCharacter> pipes(line, "|||");
StringPiece sourcePhraseString(*pipes);
StringPiece targetPhraseString(*++pipes);
StringPiece scoreString(*++pipes);
StringPiece alignString(*++pipes);
StringPiece alignString;
if (++pipes) {
StringPiece temp(*pipes);
alignString = temp;
}
// Allow but ignore rule count.
if (++pipes && ++pipes) {
stringstream strme;

17
moses/TranslationModel/RuleTable/Trie.cpp
View File

@ -54,5 +54,22 @@ bool RuleTableTrie::InitDictionary()
return ret;
}
const TargetPhraseCollection *RuleTableTrie::GetTargetPhraseCollection(const Phrase &) const
{
// exactly like CreateTargetPhraseCollection, but don't create
/*
const size_t size = source.GetSize();
const PhraseDictionaryNode *currNode = &m_collection;
for (size_t pos = 0 ; pos < size ; ++pos) {
const Word& word = source.GetWord(pos);
currNode = currNode->GetChild(word);
if (currNode == NULL)
return NULL;
}
return currNode->GetTargetPhraseCollection();
*/
}
} // namespace Moses

6
moses/TranslationModel/RuleTable/Trie.h
View File

@ -51,11 +51,7 @@ class RuleTableTrie : public PhraseDictionary
bool InitDictionary();
// Required by PhraseDictionary.
const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase &) const
{
assert(false);
return NULL;
}
const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase &) const;
private:
friend class RuleTableLoader;