moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-07-14 14:50:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge ../mosesdecoder into perf_moses2

Browse Source
This commit is contained in:
Hieu Hoang 2015-12-14 13:18:15 +00:00
commit ffaf24419d
49 changed files with 139 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cruise-control/example.config
View File

@ -1,21 +1,25 @@
# the tmp directory
MCC_TEMPDIR="/tmp/"
MCC_TEMPDIR="/disk4/cruise-control/cruise-control/tmp/"
# where logs are saved
MCC_LOGDIR="/home/cruise/"
# where moses gets checked out, tested etc.
MCC_WORKDIR="/home/cruise/work/"
MCC_LOGDIR="/disk4/cruise-control/cruise-control/"
# repository that will be checked out
MCC_GITREPO="https://github.com/moses-smt/mosesdecoder.git"
# arguments passed to Moses configure
MCC_CONFIGURE_ARGS=" --with-srilm=/path/to/srilm --with-irstlm=/path/to/irstlm "
MCC_CONFIGURE_ARGS=" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
ALTERNATIVE_CONFIGURE_ARGS=(
" --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
" --with-srilm=/disk4/cruise-control/srilm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 "
" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --no-xmlrpc-c -j8 "
)
# list of branches to be checked
MCC_SCAN_BRANCHES="master"
# regression test data archive (optional, this is the default)
MCC_REGTEST_ARCHIVE="http://www.statmt.org/moses/reg-testing/moses-reg-test-data-8.tgz"
MCC_SCAN_BRANCHES="origin/master"
# run full training/eval pipeline using EMS?
MCC_RUN_EMS="yes"

12
cruise-control/test_all_new_commits.sh
View File

@ -116,8 +116,16 @@ function run_single_test () {
#regtest_dir=$PWD/$(basename $regtest_file .tgz)
cd ..
# test build with different configurations
echo "## test build with different configurations" >> $longlog
for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}"
do
echo "building with args: $configArgs" >> $longlog
./bjam clean -a $configArgs >> $longlog 2>&1 || warn "bjam clean failed, suspicious"
done
echo "## ./bjam clean" >> $longlog
./bjam clean $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious"
./bjam clean -a $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious"
echo "## ./bjam $MCC_CONFIGURE_ARGS" >> $longlog
if [ -z "$err" ]; then
@ -169,7 +177,7 @@ function run_single_test () {
echo "## Status: $status" >> $longlog
nicedate=$(date +"%Y%m%d-%H%M%S")
echo "$commit $status $configname $ccversion $nicedate" \
echo "$commit$status$configname$ccversion$nicedate" \
>> "$LOGDIR/brief.log"
if [ -z "$err" ]; then

2
moses/BaseManager.cpp
View File

@ -101,7 +101,7 @@ OutputSurface(std::ostream &out, Phrase const& phrase) const
std::vector<FactorType> const& factor_order = options()->output.factor_order;
bool markUnknown = options()->unk.mark;
std::string const& fd = options()->output.FactorDelimiter;
std::string const& fd = options()->output.factor_delimiter;
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {

7
moses/ChartManager.cpp
View File

@ -34,6 +34,7 @@
#include "moses/OutputCollector.h"
#include "moses/ChartKBestExtractor.h"
#include "moses/HypergraphOutput.h"
#include "moses/TranslationTask.h"
using namespace std;
@ -50,7 +51,7 @@ ChartManager::ChartManager(ttasksptr const& ttask)
, m_start(clock())
, m_hypothesisId(0)
, m_parser(ttask, m_hypoStackColl)
, m_translationOptionList(StaticData::Instance().GetRuleLimit(), m_source)
, m_translationOptionList(ttask->options()->syntax.rule_limit, m_source)
{ }
ChartManager::~ChartManager()
@ -297,8 +298,8 @@ OutputSearchGraphAsHypergraph(std::ostream& out) const
void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const
{
ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream,
m_source.GetTranslationId());
ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream,
m_source.GetTranslationId());
WriteSearchGraph(writer);
}

9
moses/ChartParser.cpp
View File

@ -237,7 +237,10 @@ void ChartParser::CreateInputPaths(const InputType &input)
m_inputPathMatrix.resize(size);
UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType,
"Input must be a sentence or a tree, not lattice or confusion networks");
"Input must be a sentence or a tree, " <<
"not lattice or confusion networks");
TranslationTask const* ttask = m_ttask.lock().get();
for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) {
size_t endPos = startPos + phaseSize -1;
@ -249,11 +252,11 @@ void ChartParser::CreateInputPaths(const InputType &input)
InputPath *node;
if (range.GetNumWordsCovered() == 1) {
node = new InputPath(m_ttask, subphrase, labels, range, NULL, NULL);
node = new InputPath(ttask, subphrase, labels, range, NULL, NULL);
vec.push_back(node);
} else {
const InputPath &prevNode = GetInputPath(startPos, endPos - 1);
node = new InputPath(m_ttask, subphrase, labels, range, &prevNode, NULL);
node = new InputPath(ttask, subphrase, labels, range, &prevNode, NULL);
vec.push_back(node);
}

4
moses/ChartTranslationOptionList.cpp
View File

@ -158,8 +158,8 @@ void ChartTranslationOptionList::ApplyThreshold(float const threshold)
scoreThreshold += threshold; // StaticData::Instance().GetTranslationOptionThreshold();
CollType::iterator bound = std::partition(m_collection.begin(),
m_collection.begin()+m_size,
ScoreThresholdPred(scoreThreshold));
m_collection.begin()+m_size,
ScoreThresholdPred(scoreThreshold));
m_size = std::distance(m_collection.begin(), bound);
}

2
moses/ChartTranslationOptions.cpp
View File

@ -68,7 +68,7 @@ void ChartTranslationOptions::EvaluateWithSourceContext(const InputType &input,
{
SetInputPath(&inputPath);
// if (StaticData::Instance().GetPlaceholderFactor() != NOT_FOUND) {
if (inputPath.ttask.lock()->options()->input.placeholder_factor != NOT_FOUND) {
if (inputPath.ttask->options()->input.placeholder_factor != NOT_FOUND) {
CreateSourceRuleFromInputPath();
}

2
moses/ConfusionNet.cpp
View File

@ -283,7 +283,7 @@ CreateTranslationOptionCollection(ttasksptr const& ttask) const
// float translationOptionThreshold
// = ttask->options()->search.trans_opt_threshold;
TranslationOptionCollection *rv
= new TranslationOptionCollectionConfusionNet(ttask, *this);
= new TranslationOptionCollectionConfusionNet(ttask, *this);
//, maxNoTransOptPerCoverage, translationOptionThreshold);
assert(rv);
return rv;

2
moses/ContextScope.h
View File

@ -36,7 +36,7 @@ protected:
#endif
SPTR<std::map<std::string,float> const> m_context_weights;
public:
typedef boost::shared_ptr<ContextScope> ptr;
template<typename T>
boost::shared_ptr<void> const&
set(void const* const key, boost::shared_ptr<T> const& val) {

2
moses/FF/BleuScoreFeature.h
View File

@ -144,7 +144,7 @@ public:
return m_avg_input_length;
}
void Load(AllOptions::ptr const& opts);
void Load(AllOptions::ptr const& opts);
private:
static std::vector<BleuScoreFeature*> s_staticColl;

2
moses/FF/ConstrainedDecoding.cpp
View File

@ -56,7 +56,7 @@ void ConstrainedDecoding::Load(AllOptions::ptr const& opts)
for(size_t i = 0; i < m_paths.size(); ++i) {
InputFileStream constraintFile(m_paths[i]);
std::string line;
long sentenceID = opts->output.start_translation_id - 1 ;
long sentenceID = opts->output.start_translation_id - 1 ;
while (getline(constraintFile, line)) {
vector<string> vecStr = Tokenize(line, "\t");

6
moses/FF/DistortionScoreProducer.cpp
View File

@ -49,16 +49,16 @@ const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &in
NOT_FOUND);
}
float
float
DistortionScoreProducer::
CalculateDistortionScore(const Hypothesis& hypo,
const Range &prev, const Range &curr, const int FirstGap)
const Range &prev, const Range &curr, const int FirstGap)
{
// if(!StaticData::Instance().UseEarlyDistortionCost()) {
if(!hypo.GetManager().options()->reordering.use_early_distortion_cost) {
return - (float) hypo.GetInput().ComputeDistortionDistance(prev, curr);
} // else {
/* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
Definitions:
S : current source range

14
moses/FF/GlobalLexicalModel.cpp
View File

@ -55,7 +55,9 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts)
{
m_options = opts;
FactorCollection &factorCollection = FactorCollection::Instance();
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
const std::string& oFactorDelimiter = opts->output.factor_delimiter;
const std::string& iFactorDelimiter = opts->input.factor_delimiter;
VERBOSE(2, "Loading global lexical model from file " << m_filePath << endl);
@ -76,21 +78,23 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts)
// create the output word
Word *outWord = new Word();
vector<string> factorString = Tokenize( token[0], factorDelimiter );
vector<string> factorString = Tokenize( token[0], oFactorDelimiter );
for (size_t i=0 ; i < m_outputFactorsVec.size() ; i++) {
const FactorDirection& direction = Output;
const FactorType& factorType = m_outputFactorsVec[i];
const Factor* factor = factorCollection.AddFactor( direction, factorType, factorString[i] );
const Factor* factor
= factorCollection.AddFactor( direction, factorType, factorString[i] );
outWord->SetFactor( factorType, factor );
}
// create the input word
Word *inWord = new Word();
factorString = Tokenize( token[1], factorDelimiter );
factorString = Tokenize( token[1], iFactorDelimiter );
for (size_t i=0 ; i < m_inputFactorsVec.size() ; i++) {
const FactorDirection& direction = Input;
const FactorType& factorType = m_inputFactorsVec[i];
const Factor* factor = factorCollection.AddFactor( direction, factorType, factorString[i] );
const Factor* factor
= factorCollection.AddFactor( direction, factorType, factorString[i] );
inWord->SetFactor( factorType, factor );
}

6
moses/HypergraphOutput.h
View File

@ -73,8 +73,8 @@ public:
class ChartSearchGraphWriterMoses : public virtual ChartSearchGraphWriter
{
public:
ChartSearchGraphWriterMoses(AllOptions::ptr const& opts,
std::ostream* out, size_t lineNumber)
ChartSearchGraphWriterMoses(AllOptions::ptr const& opts,
std::ostream* out, size_t lineNumber)
: ChartSearchGraphWriter(opts), m_out(out), m_lineNumber(lineNumber) {}
virtual void WriteHeader(size_t, size_t) const {
/* do nothing */
@ -91,7 +91,7 @@ private:
class ChartSearchGraphWriterHypergraph : public virtual ChartSearchGraphWriter
{
public:
ChartSearchGraphWriterHypergraph(AllOptions::ptr const& opts, std::ostream* out)
ChartSearchGraphWriterHypergraph(AllOptions::ptr const& opts, std::ostream* out)
: ChartSearchGraphWriter(opts), m_out(out), m_nodeId(0) { }
virtual void WriteHeader(size_t winners, size_t losers) const;
virtual void WriteHypos(const ChartHypothesisCollection& hypos,

2
moses/InputPath.cpp
View File

@ -11,7 +11,7 @@ using namespace std;
namespace Moses
{
InputPath::
InputPath(ttaskwptr const theTask,
InputPath(TranslationTask const* theTask,
Phrase const& phrase,
NonTerminalSet const& sourceNonTerms,
Range const& range, InputPath const *prevNode,

5
moses/InputPath.h
View File

@ -40,7 +40,8 @@ public:
TargetPhrases;
public:
ttaskwptr const ttask;
// ttaskwptr const ttask;
TranslationTask const* ttask;
protected:
const InputPath *m_prevPath;
Phrase m_phrase;
@ -65,7 +66,7 @@ public:
, m_nextNode(NOT_FOUND) {
}
InputPath(ttaskwptr const ttask,
InputPath(TranslationTask const* ttask, // ttaskwptr const ttask,
Phrase const& phrase,
NonTerminalSet const& sourceNonTerms,
Range const& range,

8
moses/LM/BackwardTest.cpp
View File

@ -123,7 +123,6 @@ public:
Input,
outputFactorOrder,
"the",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -150,7 +149,6 @@ public:
Input,
outputFactorOrder,
"the licenses",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 2 );
@ -179,7 +177,6 @@ public:
Input,
outputFactorOrder,
"the licenses for",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 3 );
@ -206,7 +203,6 @@ public:
Input,
outputFactorOrder,
"the licenses for most",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 4 );
@ -252,7 +248,6 @@ public:
Input,
outputFactorOrder,
"the",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -280,7 +275,6 @@ public:
Input,
outputFactorOrder,
"licenses",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -308,7 +302,6 @@ public:
Input,
outputFactorOrder,
"for",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -336,7 +329,6 @@ public:
Input,
outputFactorOrder,
"most",
// StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );

2
moses/Manager.cpp
View File

@ -1699,7 +1699,7 @@ OutputSurface(std::ostream &out, Hypothesis const& edge, bool const recursive) c
}
bool markUnknown = options()->unk.mark;
std::string const& fd = options()->output.FactorDelimiter;
std::string const& fd = options()->output.factor_delimiter;
TargetPhrase const& phrase = edge.GetCurrTargetPhrase();
size_t size = phrase.GetSize();

9
moses/PDTAimp.cpp
View File

@ -116,6 +116,7 @@ PDTAimp::GetTargetPhraseCollection(Phrase const &src) const
// convert into TargetPhrases
std::string fd = m_obj->options()->output.factor_delimiter;
for(size_t i=0; i<cands.size(); ++i) {
TargetPhrase targetPhrase(m_obj);
@ -134,7 +135,8 @@ PDTAimp::GetTargetPhraseCollection(Phrase const &src) const
targetPhrase.GetScoreBreakdown().Assign(m_obj, *cands[i].fnames[j], cands[i].fvalues[j]);
}
CreateTargetPhrase(targetPhrase,factorStrings,scoreVector, Scores(0), &wacands[i], &src);
CreateTargetPhrase(targetPhrase,factorStrings, fd, scoreVector, Scores(0),
&wacands[i], &src);
costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size()));
tCands.push_back(targetPhrase);
@ -375,6 +377,7 @@ void PDTAimp::CacheSource(ConfusionNet const& src)
TargetPhrase targetPhrase(m_obj);
CreateTargetPhrase(targetPhrase
, j ->first
, m_obj->options()->output.factor_delimiter
, scores.transScore
, scores.inputScores
, NULL
@ -403,6 +406,7 @@ void PDTAimp::CacheSource(ConfusionNet const& src)
void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase,
StringTgtCand::Tokens const& factorStrings,
std::string const& factorDelimiter,
Scores const& transVector,
Scores const& inputVector,
const std::string *alignmentString,
@ -411,7 +415,8 @@ void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase,
FactorCollection &factorCollection = FactorCollection::Instance();
for(size_t k=0; k<factorStrings.size(); ++k) {
util::TokenIter<util::MultiCharacter, false> word(*factorStrings[k], StaticData::Instance().GetFactorDelimiter());
util::TokenIter<util::MultiCharacter, false>
word(*factorStrings[k], factorDelimiter);
Word& w=targetPhrase.AddWord();
for(size_t l=0; l<m_output.size(); ++l, ++word) {
w[m_output[l]]= factorCollection.AddFactor(*word);

1
moses/PDTAimp.h
View File

@ -116,6 +116,7 @@ public:
void CreateTargetPhrase(TargetPhrase& targetPhrase,
StringTgtCand::Tokens const& factorStrings,
std::string const& factorDelimiter,
Scores const& transVector,
Scores const& inputVector,
const std::string *alignmentString,

12
moses/Phrase.h
View File

@ -55,18 +55,6 @@ protected:
public:
// /// return shared pointer to ttask
// // only TargetPhrases have non-NULL ttaskptrs!
// virtual ttasksptr GetTtask() const {
// return ttasksptr();
// }
// /// check if this phrase belongs to a valid ttask
// // only TargetPhrases have non-NULL ttaskptrs!
// virtual bool HasTtaskSPtr() const {
// return false;
// }
virtual bool HasScope() const {
return false;
}

2
moses/Sentence.cpp
View File

@ -257,7 +257,7 @@ Sentence::
CreateTranslationOptionCollection(ttasksptr const& ttask) const
{
TranslationOptionCollection *rv
= new TranslationOptionCollectionText(ttask, *this);
= new TranslationOptionCollectionText(ttask, *this);
assert(rv);
return rv;
}

25
moses/StaticData.cpp
View File

@ -53,7 +53,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#ifdef HAVE_CMPH
#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h"
#endif
#if !defined WIN32 || defined __MINGW32__ || defined HAVE_CMPH
#if defined HAVE_CMPH
#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
#endif
@ -127,18 +127,10 @@ StaticData
::ini_output_options()
{
const PARAM_VEC *params;
// verbose level
m_parameter->SetParameter(m_verboseLevel, "verbose", (size_t) 1);
m_parameter->SetParameter<string>(m_outputUnknownsFile,
"output-unknowns", "");
// m_parameter->SetParameter<long>(m_startTranslationId,
// "start-translation-id", 0);
return true;
}
@ -206,16 +198,10 @@ bool StaticData::LoadData(Parameter *parameter)
// threading etc.
if (!ini_performance_options()) return false;
// Compact phrase table and reordering model
// m_parameter->SetParameter(m_minphrMemory, "minphr-memory", false );
// m_parameter->SetParameter(m_minlexrMemory, "minlexr-memory", false );
// S2T decoder
// FEATURE FUNCTION INITIALIZATION HAPPENS HERE ===============================
// set class-specific default parameters
#if !defined WIN32 || defined __MINGW32__ || defined HAVE_CMPH
#if defined HAVE_CMPH
LexicalReorderingTableCompact::SetStaticDefaultParameters(*parameter);
PhraseDictionaryCompact::SetStaticDefaultParameters(*parameter);
#endif
@ -322,8 +308,6 @@ void StaticData::LoadChartDecodingParameters()
// source label overlap
m_parameter->SetParameter(m_sourceLabelOverlap, "source-label-overlap",
SourceLabelOverlapAdd);
m_parameter->SetParameter(m_ruleLimit, "rule-limit",
DEFAULT_MAX_TRANS_OPT_SIZE);
}
@ -438,7 +422,7 @@ LoadDecodeGraphsOld(const vector<string> &mappingVector,
UTIL_THROW_IF2(decodeStep == NULL, "Null decode step");
if (m_decodeGraphs.size() < decodeGraphInd + 1) {
DecodeGraph *decodeGraph;
if (is_syntax(m_options->search.algo)) {
if (is_syntax(m_options->search.algo)) {
size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN;
VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl);
decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan);
@ -506,7 +490,7 @@ void StaticData::LoadDecodeGraphsNew(const std::vector<std::string> &mappingVect
UTIL_THROW_IF2(decodeStep == NULL, "Null decode step");
if (m_decodeGraphs.size() < decodeGraphInd + 1) {
DecodeGraph *decodeGraph;
if (is_syntax(m_options->search.algo)) {
if (is_syntax(m_options->search.algo)) {
size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN;
VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl);
decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan);
@ -619,7 +603,6 @@ void StaticData::LoadFeatureFunctions()
m_requireSortingAfterSourceContext = true;
}
// if (PhraseDictionary *ffCast = dynamic_cast<PhraseDictionary*>(ff)) {
if (dynamic_cast<PhraseDictionary*>(ff)) {
doLoad = false;
}

18
moses/StaticData.h
View File

@ -104,12 +104,6 @@ protected:
std::string m_outputUnknownsFile; //! output unknowns in this file
size_t m_ruleLimit;
// Whether to load compact phrase table and reordering table into memory
bool m_minphrMemory;
bool m_minlexrMemory;
// Initial = 0 = can be used when creating poss trans
// Other = 1 = used to calculate LM score once all steps have been processed
Word m_inputDefaultNonTerminal, m_outputDefaultNonTerminal;
@ -146,7 +140,6 @@ protected:
void NoCache();
bool m_continuePartialTranslation;
std::string m_binPath;
// soft NT lookup for chart models
@ -252,17 +245,10 @@ public:
return m_unknownLHS;
}
size_t GetRuleLimit() const {
return m_ruleLimit;
}
float GetRuleCountThreshold() const {
return 999999; /* TODO wtf! */
}
bool ContinuePartialTranslation() const {
return m_continuePartialTranslation;
}
void ReLoadBleuScoreFeatureParameter(float weight);
Parameter* GetParameter() {
@ -273,10 +259,6 @@ public:
return m_threadCount;
}
// long GetStartTranslationId() const {
// return m_startTranslationId;
// }
void SetExecPath(const std::string &path);
const std::string &GetBinDirectory() const;

6
moses/Syntax/F2S/Manager-inl.h
View File

@ -60,9 +60,9 @@ void Manager<RuleMatcher>::Decode()
const StaticData &staticData = StaticData::Instance();
// Get various pruning-related constants.
const std::size_t popLimit = staticData.options()->cube.pop_limit;
const std::size_t ruleLimit = staticData.GetRuleLimit();
const std::size_t stackLimit = staticData.options()->search.stack_size;
const std::size_t popLimit = options()->cube.pop_limit;
const std::size_t ruleLimit = options()->syntax.rule_limit;
const std::size_t stackLimit = options()->search.stack_size;
// Initialize the stacks.
InitializeStacks();

6
moses/Syntax/S2T/Manager-inl.h
View File

@ -163,9 +163,9 @@ void Manager<Parser>::Decode()
const StaticData &staticData = StaticData::Instance();
// Get various pruning-related constants.
const std::size_t popLimit = staticData.options()->cube.pop_limit;
const std::size_t ruleLimit = staticData.GetRuleLimit();
const std::size_t stackLimit = staticData.options()->search.stack_size;
const std::size_t popLimit = options()->cube.pop_limit;
const std::size_t ruleLimit = options()->syntax.rule_limit;
const std::size_t stackLimit = options()->search.stack_size;
// Initialise the PChart and SChart.
InitializeCharts();

8
moses/Syntax/T2S/Manager-inl.h
View File

@ -94,11 +94,11 @@ void Manager<RuleMatcher>::InitializeStacks()
template<typename RuleMatcher>
void Manager<RuleMatcher>::Decode()
{
const StaticData &staticData = StaticData::Instance();
// const StaticData &staticData = StaticData::Instance();
// Get various pruning-related constants.
const std::size_t popLimit = this->options()->cube.pop_limit;
const std::size_t ruleLimit = staticData.GetRuleLimit();
const std::size_t ruleLimit = this->options()->syntax.rule_limit;
const std::size_t stackLimit = this->options()->search.stack_size;
// Initialize the stacks.
@ -215,8 +215,8 @@ void Manager<RuleMatcher>::ExtractKBest(
// than k. The k-best factor determines how much bigger the limit should be,
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
// too many translations are identical.
const StaticData &staticData = StaticData::Instance();
const std::size_t nBestFactor = staticData.options()->nbest.factor;
// const StaticData &staticData = StaticData::Instance();
const std::size_t nBestFactor = this->options()->nbest.factor;
std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor;
// Extract the derivations.

3
moses/Syntax/T2S/RuleTrieLoader.cpp
View File

@ -44,9 +44,6 @@ bool RuleTrieLoader::Load(Moses::AllOptions const& opts,
{
PrintUserTime(std::string("Start loading text phrase table. Moses format"));
// const StaticData &staticData = StaticData::Instance();
// const std::string &factorDelimiter = staticData.GetFactorDelimiter();
std::size_t count = 0;
std::ostream *progress = NULL;

1
moses/TargetPhrase.cpp
View File

@ -53,7 +53,6 @@ TargetPhrase::TargetPhrase( std::string out_string, const PhraseDictionary *pt)
const StaticData &staticData = StaticData::Instance();
// XXX should this really be InputFactorOrder???
CreateFromString(Output, staticData.options()->input.factor_order, out_string,
// staticData.GetFactorDelimiter(), // eliminated [UG]
NULL);
}

4
moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp
View File

@ -45,8 +45,8 @@ ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory(
{
size_t sourceSize = parser.GetSize();
m_completedRules.resize(sourceSize);
size_t ruleLimit = parser.options()->syntax.rule_limit;
m_completedRules.resize(sourceSize, CompletedRuleCollection(ruleLimit));
m_isSoftMatching = !m_softMatchingMap.empty();
}

4
moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp
View File

@ -45,8 +45,8 @@ ChartRuleLookupManagerMemoryPerSentence::ChartRuleLookupManagerMemoryPerSentence
{
size_t sourceSize = parser.GetSize();
m_completedRules.resize(sourceSize);
size_t ruleLimit = parser.options()->syntax.rule_limit;
m_completedRules.resize(sourceSize, CompletedRuleCollection(ruleLimit));
m_isSoftMatching = !m_softMatchingMap.empty();
}

3
moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp
View File

@ -27,7 +27,8 @@ using namespace std;
namespace Moses
{
CompletedRuleCollection::CompletedRuleCollection() : m_ruleLimit(StaticData::Instance().GetRuleLimit())
CompletedRuleCollection::CompletedRuleCollection(size_t rule_limit)
: m_ruleLimit(rule_limit)
{
m_scoreThreshold = numeric_limits<float>::infinity();
}

2
moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h
View File

@ -74,7 +74,7 @@ public:
struct CompletedRuleCollection {
public:
CompletedRuleCollection();
CompletedRuleCollection(size_t rule_limit);
~CompletedRuleCollection();
CompletedRuleCollection(const CompletedRuleCollection &old)

4
moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h
View File

@ -84,9 +84,9 @@ public:
void
Load(std::string filePath);
static void
static void
SetStaticDefaultParameters(Parameter const& param);
};
}

12
moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp
View File

@ -70,8 +70,8 @@ void PhraseDictionaryCompact::Load(AllOptions::ptr const& opts)
if (!FileExists(tFilePath))
throw runtime_error("Error: File " + tFilePath + " does not exist.");
m_phraseDecoder
= new PhraseDecoder(*this, &m_input, &m_output, m_numScoreComponents);
m_phraseDecoder
= new PhraseDecoder(*this, &m_input, &m_output, m_numScoreComponents);
std::FILE* pFile = std::fopen(tFilePath.c_str() , "r");
@ -155,7 +155,7 @@ PhraseDictionaryCompact::
delete m_phraseDecoder;
}
void
void
PhraseDictionaryCompact::
CacheForCleanup(TargetPhraseCollection::shared_ptr tpc)
{
@ -164,12 +164,12 @@ CacheForCleanup(TargetPhraseCollection::shared_ptr tpc)
m_sentenceCache->push_back(tpc);
}
void
void
PhraseDictionaryCompact::
AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase)
AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase)
{ }
void
void
PhraseDictionaryCompact::
CleanUpAfterSentenceProcessing(const InputType &source)
{

42
moses/TranslationModel/PhraseDictionary.cpp
View File

@ -35,16 +35,6 @@ namespace Moses
{
std::vector<PhraseDictionary*> PhraseDictionary::s_staticColl;
// CacheColl::~CacheColl()
// {
// // not needed any more since the switch to shared pointers
// // for (iterator iter = begin(); iter != end(); ++iter) {
// // std::pair<TargetPhraseCollection::shared_ptr , clock_t> &key = iter->second;
// // TargetPhraseCollection::shared_ptr tps = key.first;
// // delete tps;
// // }
// }
PhraseDictionary::PhraseDictionary(const std::string &line, bool registerNow)
: DecodeFeature(line, registerNow)
, m_tableLimit(20) // default
@ -82,8 +72,7 @@ GetTargetPhraseCollectionLEGACY(const Phrase& src) const
ret.reset(new TargetPhraseCollection(*ret));
}
cache[hash] = entry(ret, clock());
} else {
// in cache. just use it
} else { // in cache. just use it
iter->second.second = clock();
ret = iter->second.first;
}
@ -175,31 +164,6 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
}
}
// persistent cache handling
// saving presistent cache to disk
//void PhraseDictionary::SaveCache() const
//{
// CacheColl &cache = GetCache();
// for( std::map<size_t, std::pair<TargetPhraseCollection::shared_ptr ,clock_t> >::iterator iter,
// iter != cache.end(),
// iter++ ) {
//
// }
//}
// loading persistent cache from disk
//void PhraseDictionary::LoadCache() const
//{
// CacheColl &cache = GetCache();
// std::map<size_t, std::pair<TargetPhraseCollection::shared_ptr ,clock_t> >::iterator iter;
// iter = cache.begin();
// while( iter != cache.end() ) {
// std::map<size_t, std::pair<TargetPhraseCollection::shared_ptr ,clock_t> >::iterator iterRemove = iter++;
// delete iterRemove->second.first;
// cache.erase(iterRemove);
// }
//}
// reduce presistent cache by half of maximum size
void PhraseDictionary::ReduceCache() const
{
@ -233,7 +197,9 @@ void PhraseDictionary::ReduceCache() const
<< reduceCacheTime << " seconds." << std::endl);
}
CacheColl &PhraseDictionary::GetCache() const
CacheColl &
PhraseDictionary::
GetCache() const
{
CacheColl *cache;
cache = m_cache.get();

10
moses/TranslationModel/PhraseDictionary.h
View File

@ -44,6 +44,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/TargetPhraseCollection.h"
#include "moses/InputPath.h"
#include "moses/FF/DecodeFeature.h"
#include "moses/ContextScope.h"
namespace Moses
{
@ -58,15 +59,6 @@ class ChartParser;
// typedef std::pair<TargetPhraseCollection::shared_ptr, clock_t> TPCollLastUse;
typedef std::pair<TargetPhraseCollection::shared_ptr, clock_t> CacheCollEntry;
typedef boost::unordered_map<size_t, CacheCollEntry> CacheColl;
// class CacheColl : public boost::unordered_map<size_t, TPCollLastUse>
// {
// // 1st = hash of source phrase/ address of phrase-table node
// // 2nd = all translations
// // 3rd = time of last access
// public:
// ~CacheColl();
// };
/**
* Abstract base class for phrase dictionaries (tables).

3
moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp
View File

@ -330,7 +330,6 @@ void PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseStr
{
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseString, std::string targetPhraseString)" << std::endl);
const StaticData &staticData = StaticData::Instance();
// const std::string& factorDelimiter = staticData.GetFactorDelimiter();
Phrase sourcePhrase(0);
Phrase targetPhrase(0);
@ -428,7 +427,6 @@ void PhraseDictionaryDynamicCacheBased::ClearSource(std::vector<std::string> ent
{
VERBOSE(3,"entries.size():|" << entries.size() << "|" << std::endl);
const StaticData &staticData = StaticData::Instance();
// const std::string& factorDelimiter = staticData.GetFactorDelimiter();
Phrase sourcePhrase(0);
std::vector<std::string>::iterator it;
@ -517,7 +515,6 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s
{
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString, std::string waString)" << std::endl);
const StaticData &staticData = StaticData::Instance();
// const std::string& factorDelimiter = staticData.GetFactorDelimiter();
Phrase sourcePhrase(0);
TargetPhrase targetPhrase(0);

5
moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp
View File

@ -144,7 +144,10 @@ TargetPhraseCollection::shared_ptr PhraseDictionaryMultiModelCounts::GetTargetPh
}
void PhraseDictionaryMultiModelCounts::CollectSufficientStats(const Phrase& src, vector<float> &fs, map<string,multiModelCountsStats*>* allStats) const
void
PhraseDictionaryMultiModelCounts::
CollectSufficientStats(const Phrase& src, vector<float> &fs,
map<string,multiModelCountsStats*>* allStats) const
//fill fs and allStats with statistics from models
{
for(size_t i = 0; i < m_numModels; ++i) {

2
moses/TranslationModel/ProbingPT/ProbingPT.cpp
View File

@ -77,7 +77,7 @@ void ProbingPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQue
InputPath &inputPath = **iter;
const Phrase &sourcePhrase = inputPath.GetPhrase();
if (sourcePhrase.GetSize() > StaticData::Instance().options()->search.max_phrase_length) {
if (sourcePhrase.GetSize() > m_options->search.max_phrase_length) {
continue;
}

12
moses/TranslationOptionCollectionConfusionNet.cpp
View File

@ -20,11 +20,11 @@ namespace Moses
/** constructor; just initialize the base class */
TranslationOptionCollectionConfusionNet::
TranslationOptionCollectionConfusionNet(ttasksptr const& ttask,
TranslationOptionCollectionConfusionNet(ttasksptr const& ttask,
const ConfusionNet &input)
// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
: TranslationOptionCollection(ttask,input)//
// , maxNoTransOptPerCoverage, translationOptionThreshold)
// , maxNoTransOptPerCoverage, translationOptionThreshold)
{
size_t maxNoTransOptPerCoverage = ttask->options()->search.max_trans_opt_per_cov;
float translationOptionThreshold = ttask->options()->search.trans_opt_threshold;
@ -63,7 +63,8 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask,
const ScorePair &scores = col[i].second;
ScorePair *inputScore = new ScorePair(scores);
InputPath *path = new InputPath(ttask, subphrase, labels, range, NULL, inputScore);
InputPath* path = new InputPath(ttask.get(), subphrase, labels,
range, NULL, inputScore);
list.push_back(path);
m_inputPathQueue.push_back(path);
@ -114,7 +115,8 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask,
ScorePair *inputScore = new ScorePair(*prevInputScore);
inputScore->PlusEquals(scores);
InputPath *path = new InputPath(ttask, subphrase, labels, range, &prevPath, inputScore);
InputPath *path = new InputPath(ttask.get(), subphrase, labels, range,
&prevPath, inputScore);
list.push_back(path);
m_inputPathQueue.push_back(path);

2
moses/TranslationOptionCollectionConfusionNet.h
View File

@ -36,7 +36,7 @@ protected:
public:
TranslationOptionCollectionConfusionNet
(ttasksptr const& ttask, const ConfusionNet &source);
(ttasksptr const& ttask, const ConfusionNet &source);
// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold);
void ProcessUnknownWord(size_t sourcePos);

6
moses/TranslationOptionCollectionLattice.cpp
View File

@ -22,9 +22,9 @@ namespace Moses
TranslationOptionCollectionLattice
::TranslationOptionCollectionLattice
( ttasksptr const& ttask, const WordLattice &input)
// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
: TranslationOptionCollection(ttask, input)//
// , maxNoTransOptPerCoverage, translationOptionThreshold)
// , maxNoTransOptPerCoverage, translationOptionThreshold)
{
UTIL_THROW_IF2(StaticData::Instance().GetUseLegacyPT(),
"Not for models using the legqacy binary phrase table");
@ -65,7 +65,7 @@ TranslationOptionCollectionLattice
ScorePair *inputScore = new ScorePair(scores);
InputPath *path
= new InputPath(ttask, subphrase, labels, range, NULL, inputScore);
= new InputPath(ttask.get(), subphrase, labels, range, NULL, inputScore);
path->SetNextNode(nextNode);
m_inputPathQueue.push_back(path);

12
moses/TranslationOptionCollectionText.cpp
View File

@ -35,14 +35,14 @@ namespace Moses
/** constructor; just initialize the base class */
TranslationOptionCollectionText::
TranslationOptionCollectionText(ttasksptr const& ttask, Sentence const &input)
//, size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
//, size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
: TranslationOptionCollection(ttask,input)
// , maxNoTransOptPerCoverage, translationOptionThreshold)
// , maxNoTransOptPerCoverage, translationOptionThreshold)
{
size_t maxNoTransOptPerCoverage
= ttask->options()->search.max_trans_opt_per_cov;
= ttask->options()->search.max_trans_opt_per_cov;
float translationOptionThreshold
= ttask->options()->search.trans_opt_threshold;
= ttask->options()->search.trans_opt_threshold;
size_t size = input.GetSize();
m_inputPathMatrix.resize(size);
for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
@ -56,11 +56,11 @@ TranslationOptionCollectionText(ttasksptr const& ttask, Sentence const &input)
InputPath *path;
if (range.GetNumWordsCovered() == 1) {
path = new InputPath(ttask, subphrase, labels, range, NULL, NULL);
path = new InputPath(ttask.get(), subphrase, labels, range, NULL, NULL);
vec.push_back(path);
} else {
const InputPath &prevPath = GetInputPath(startPos, endPos - 1);
path = new InputPath(ttask, subphrase, labels, range, &prevPath, NULL);
path = new InputPath(ttask.get(), subphrase, labels, range, &prevPath, NULL);
vec.push_back(path);
}

3
moses/Word.cpp
View File

@ -213,7 +213,8 @@ TO_STRING_BODY(Word);
ostream& operator<<(ostream& out, const Word& word)
{
util::StringStream strme;
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
const std::string& factorDelimiter
= StaticData::Instance().options()->output.factor_delimiter;
bool firstPass = true;
unsigned int stop = max_fax();
for (unsigned int currFactor = 0 ; currFactor < stop; currFactor++) {

2
moses/parameters/InputOptions.cpp
View File

@ -17,6 +17,7 @@ namespace Moses {
xml_brackets.first = "<";
xml_brackets.second = ">";
factor_order.assign(1,0);
factor_delimiter = "|";
}
bool
@ -76,6 +77,7 @@ namespace Moses {
if (factor_order.empty()) factor_order.assign(1,0);
param.SetParameter(placeholder_factor, "placeholder-factor", NOT_FOUND);
param.SetParameter<std::string>(factor_delimiter, "factor-delimiter", "|");
param.SetParameter<std::string>(input_file_path,"input-file","");
return true;

3
moses/parameters/InputOptions.h
View File

@ -14,10 +14,9 @@ namespace Moses
InputTypeEnum input_type;
XmlInputType xml_policy; // pass through, ignore, exclusive, inclusive
std::vector<FactorType> factor_order; // input factor order
std::string factor_delimiter;
FactorType placeholder_factor; // where to store original text for placeholders
std::string input_file_path;
std::pair<std::string,std::string> xml_brackets;
// strings to use as XML tags' opening and closing brackets.
// Default are "<" and ">"

11
moses/parameters/ReportingOptions.cpp
View File

@ -24,6 +24,7 @@ namespace Moses {
, lattice_sample_size(0)
{
factor_order.assign(1,0);
factor_delimiter = "|";
}
bool
@ -94,14 +95,14 @@ namespace Moses {
params= param.GetParam("output-factors");
if (params) factor_order = Scan<FactorType>(*params);
if (factor_order.empty()) factor_order.assign(1,0);
if (ReportAllFactors) {
for (size_t i = 1; i < MAX_NUM_FACTORS; ++i)
factor_order.push_back(i);
}
param.SetParameter(FactorDelimiter, "factor-delimiter", std::string("|"));
param.SetParameter(FactorDelimiter, "output-factor-delimiter", FactorDelimiter);
param.SetParameter(factor_delimiter, "factor-delimiter", std::string("|"));
param.SetParameter(factor_delimiter, "output-factor-delimiter", factor_delimiter);
return true;
}
@ -134,12 +135,12 @@ namespace Moses {
m = param.find("factor-delimiter");
if (m != param.end()) {
FactorDelimiter = Trim(xmlrpc_c::value_string(m->second));
factor_delimiter = Trim(xmlrpc_c::value_string(m->second));
}
m = param.find("output-factor-delimiter");
if (m != param.end()) {
FactorDelimiter = Trim(xmlrpc_c::value_string(m->second));
factor_delimiter = Trim(xmlrpc_c::value_string(m->second));
}
return true;

2
moses/parameters/ReportingOptions.h
View File

@ -13,6 +13,7 @@ namespace Moses
long start_translation_id;
std::vector<FactorType> factor_order;
std::string factor_delimiter;
bool ReportAllFactors; // m_reportAllFactors;
int ReportSegmentation; // 0: no 1: m_reportSegmentation 2: ..._enriched
@ -23,7 +24,6 @@ namespace Moses
WordAlignmentSort WA_SortOrder; // 0: no, 1: target order
std::string AlignmentOutputFile;
std::string FactorDelimiter;
bool WordGraph;