moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 05:55:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rename LBLLM -> OxLM.

Browse Source
This commit is contained in:
Paul Baltescu 2014-09-26 15:18:14 +01:00
parent af28063e3b
commit 6f9d59129f
5 changed files with 13 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
contrib/other-builds/moses/.cproject
View File

@ -74,7 +74,6 @@
</toolChain>
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.511477442" name="Rand.h" rcbsApplicability="disable" resourcePath="LM/Rand.h" toolsToInvoke=""/>
<<<<<<< HEAD
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1211280539" name="DALMWrapper.h" rcbsApplicability="disable" resourcePath="LM/DALMWrapper.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.790052015" name="IRST.h" rcbsApplicability="disable" resourcePath="LM/IRST.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1845526535" name="SRI.h" rcbsApplicability="disable" resourcePath="LM/SRI.h" toolsToInvoke=""/>
@ -143,7 +142,7 @@
</folderInfo>
<sourceEntries>
<entry excluding="LM/IRST.h|LM/IRST.cpp|LM/SRI.h|LM/SRI.cpp|TranslationModel/UG|LM/DALMWrapper.h|LM/DALMWrapper.cpp|TranslationModel/UG/mm/test-dynamic-im-tsa.cc|TranslationModel/UG/mm/symal2mam.cc|TranslationModel/UG/mm/mtt-dump.cc|TranslationModel/UG/mm/mtt-count-words.cc|TranslationModel/UG/mm/mtt-build.cc|TranslationModel/UG/mm/mmlex-lookup.cc|TranslationModel/UG/mm/mmlex-build.cc|TranslationModel/UG/mm/mam_verify.cc|TranslationModel/UG/mm/mam2symal.cc|TranslationModel/UG/mm/custom-pt.cc|TranslationModel/UG/mm/calc-coverage.cc|TranslationModel/UG/mm/mtt.count.cc|TranslationModel/UG/util|LM/oxlm|LM/Rand.h|LM/Rand.cpp|TranslationModel/CompactPT|LM/NeuralLMWrapper.cpp|FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
=======
</sourceEntries>
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1176518033" name="/" resourcePath="LM/bilingual-lm">
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.2110557759" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug" unusedChildren="">
<tool id="cdt.managedbuild.tool.gnu.archiver.base.2066996463" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base.1976472988"/>
@ -158,7 +157,6 @@
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1211280539" name="DALMWrapper.h" rcbsApplicability="disable" resourcePath="LM/DALMWrapper.h" toolsToInvoke=""/>
<sourceEntries>
<entry excluding="LM/bilingual-lm|TranslationModel/UG/mm/test-dynamic-im-tsa.cc|TranslationModel/UG/mm/symal2mam.cc|TranslationModel/UG/mm/mtt-dump.cc|TranslationModel/UG/mm/mtt-count-words.cc|TranslationModel/UG/mm/mtt-build.cc|TranslationModel/UG/mm/mmlex-lookup.cc|TranslationModel/UG/mm/mmlex-build.cc|TranslationModel/UG/mm/mam_verify.cc|TranslationModel/UG/mm/mam2symal.cc|TranslationModel/UG/mm/custom-pt.cc|TranslationModel/UG/mm/calc-coverage.cc|TranslationModel/UG/mm/mtt.count.cc|TranslationModel/UG/util|LM/oxlm|LM/Rand.h|LM/Rand.cpp|TranslationModel/CompactPT|LM/NeuralLMWrapper.cpp|FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
>>>>>>> fix Jamfile
</sourceEntries>
</configuration>
</storageModule>
@ -265,7 +263,7 @@
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_DALM"/>
<listOptionValue builtIn="false" value="LM_NPLM"/>
<listOptionValue builtIn="false" value="LM_LBL"/>
<listOptionValue builtIn="false" value="LM_OXLM"/>
<listOptionValue builtIn="false" value="_FILE_OFFSET_BIT=64"/>
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
</option>

11
contrib/other-builds/moses/.project
View File

@ -2249,6 +2249,7 @@
<locationURI>PARENT-1-ECLIPSE_HOME/github/mosesdecoder/moses/LM/bilingual-lm/BilingualLM.h</locationURI>
</link>
<link>
<<<<<<< HEAD
<name>LM/oxlm/LBLLM.cpp</name>
>>>>>>> add Bilingual LM class
<type>1</type>
@ -2353,6 +2354,16 @@
<name>Syntax/S2T/RuleTrieLoader.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/Syntax/S2T/RuleTrieLoader.cpp</locationURI>
=======
<name>LM/oxlm/OxLM.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/oxlm/OxLM.cpp</locationURI>
</link>
<link>
<name>LM/oxlm/OxLM.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/oxlm/OxLM.h</locationURI>
>>>>>>> Rename LBLLM -> OxLM.
</link>
<link>
<name>Syntax/S2T/RuleTrieLoader.h</name>

8
moses/Jamfile
View File

@ -10,14 +10,6 @@ if $(with-dlib) {
dlib = ;
}
with-oxlm = [ option.get "with-oxlm" ] ;
if $(with-oxlm) {
oxlm2 = <cxxflags>-std=c++0x <define>LM_OXLM <include>$(with-oxlm)/src <include>$(with-oxlm)/third_party/eigen ;
} else {
oxlm2 = ;
}
alias headers : ../util//kenutil : : : $(max-factors) $(dlib) $(oxlm2) ;
alias ThreadPool : ThreadPool.cpp ;
alias Util : Util.cpp Timer.cpp ;

60
moses/LM/oxlm/Mapper.cpp
View File

@ -1,60 +0,0 @@
#include "Mapper.h"
#include "moses/FactorCollection.h"
using namespace std;
namespace Moses {
OXLMMapper::OXLMMapper(const boost::shared_ptr<oxlm::Vocabulary>& vocab)
: vocab(vocab) {
for (int i = 0; i < vocab->size(); ++i) {
const string &str = vocab->convert(i);
FactorCollection &fc = FactorCollection::Instance();
const Moses::Factor *factor = fc.AddFactor(str, false);
moses2lbl[factor] = i;
}
kUNKNOWN = vocab->convert("<unk>");
}
int OXLMMapper::convert(const Moses::Factor* factor) const {
Coll::const_iterator iter;
iter = moses2lbl.find(factor);
if (iter == moses2lbl.end()) {
return kUNKNOWN;
} else {
int ret = iter->second;
return ret;
}
}
std::vector<int> OXLMMapper::convert(const Phrase& phrase) const {
size_t size = phrase.GetSize();
vector<int> ret(size);
for (size_t i = 0; i < size; ++i) {
const Moses::Factor *factor = phrase.GetFactor(i, 0);
int id = convert(factor);
ret[i] = id;
}
return ret;
}
void OXLMMapper::convert(
const std::vector<const Word*>& contextFactor,
std::vector<int>& ids,
int& word) const {
size_t size = contextFactor.size();
ids.resize(size - 1);
for (size_t i = 0; i < size - 1; ++i) {
const Moses::Factor *factor = contextFactor[i]->GetFactor(0);
int id = convert(factor);
ids[i] = id;
}
std::reverse(ids.begin(), ids.end());
const Moses::Factor *factor = contextFactor.back()->GetFactor(0);
word = convert(factor);
}
} // namespace Moses

51
moses/LM/oxlm/Mapper.h
View File

@ -1,51 +0,0 @@
#pragma once
#include <map>
#include "lbl/vocabulary.h"
#include "moses/Factor.h"
#include "moses/Phrase.h"
namespace Moses {
class OXLMMapper {
public:
OXLMMapper(const boost::shared_ptr<oxlm::Vocabulary>& vocab);
int convert(const Moses::Factor *factor) const;
std::vector<int> convert(const Phrase &phrase) const;
void convert(
const std::vector<const Word*> &contextFactor,
std::vector<int> &ids,
int &word) const;
private:
void add(int lbl_id, int cdec_id);
boost::shared_ptr<oxlm::Vocabulary> vocab;
typedef std::map<const Moses::Factor*, int> Coll;
Coll moses2lbl;
int kUNKNOWN;
};
/**
* Wraps the feature values computed from the LBL language model.
*/
struct LBLFeatures {
LBLFeatures() : LMScore(0), OOVScore(0) {}
LBLFeatures(double lm_score, double oov_score)
: LMScore(lm_score), OOVScore(oov_score) {}
LBLFeatures& operator+=(const LBLFeatures& other) {
LMScore += other.LMScore;
OOVScore += other.OOVScore;
return *this;
}
double LMScore;
double OOVScore;
};
} // namespace Moses