mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
daily automatic beautifier
This commit is contained in:
parent
883c34aee9
commit
21aa5af640
@ -191,11 +191,11 @@ int main(int argc, char* argv[])
|
||||
BOOST_FOREACH(float const& p, pgrid) {
|
||||
lmbr.precision = p;
|
||||
BOOST_FOREACH(float const& r, rgrid) {
|
||||
lmbr.ratio = r;
|
||||
lmbr.ratio = r;
|
||||
BOOST_FOREACH(size_t const prune_i, prune_grid) {
|
||||
lmbr.pruning_factor = prune_i;
|
||||
lmbr.pruning_factor = prune_i;
|
||||
BOOST_FOREACH(float const& scale_i, scale_grid) {
|
||||
mbr.scale = scale_i;
|
||||
mbr.scale = scale_i;
|
||||
size_t lineCount = source->GetTranslationId();
|
||||
cout << lineCount << " ||| " << p << " "
|
||||
<< r << " " << size_t(prune_i) << " " << scale_i
|
||||
|
@ -289,9 +289,9 @@ void ChartHypothesis::CleanupArcList()
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
size_t nBestSize = staticData.options().nbest.nbest_size;
|
||||
bool distinctNBest = (staticData.options().nbest.only_distinct
|
||||
|| staticData.options().mbr.enabled
|
||||
|| staticData.GetOutputSearchGraph()
|
||||
|| staticData.GetOutputSearchGraphHypergraph());
|
||||
|| staticData.options().mbr.enabled
|
||||
|| staticData.GetOutputSearchGraph()
|
||||
|| staticData.GetOutputSearchGraphHypergraph());
|
||||
|
||||
if (!distinctNBest && m_arcList->size() > nBestSize) {
|
||||
// prune arc list only if there too many arcs
|
||||
|
@ -323,7 +323,7 @@ void ChartManager::OutputNBest(OutputCollector *collector) const
|
||||
const size_t translationId = m_source.GetTranslationId();
|
||||
|
||||
VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO "
|
||||
<< staticData.options().nbest.output_file_path << endl);
|
||||
<< staticData.options().nbest.output_file_path << endl);
|
||||
std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
|
||||
CalcNBest(nBestSize, nBestList,staticData.options().nbest.only_distinct);
|
||||
OutputNBestList(collector, nBestList, translationId);
|
||||
|
@ -68,8 +68,8 @@ ConfusionNet() : InputType()
|
||||
|
||||
const StaticData& SD = StaticData::Instance();
|
||||
if (SD.IsSyntax()) {
|
||||
m_defaultLabelSet.insert(SD.GetInputDefaultNonTerminal());
|
||||
}
|
||||
m_defaultLabelSet.insert(SD.GetInputDefaultNonTerminal());
|
||||
}
|
||||
UTIL_THROW_IF2(&InputFeature::Instance() == NULL, "Input feature must be specified");
|
||||
}
|
||||
|
||||
|
@ -190,7 +190,7 @@ batch_run()
|
||||
bool use_context_window = ioWrapper->GetLookAhead() || ioWrapper->GetLookBack();
|
||||
bool use_context = use_context_window || context_string.size();
|
||||
bool use_sliding_context_window = (use_context_window
|
||||
&& ioWrapper->GetLookAhead() != size_t_max);
|
||||
&& ioWrapper->GetLookAhead() != size_t_max);
|
||||
|
||||
boost::shared_ptr<std::vector<std::string> > context_window;
|
||||
boost::shared_ptr<std::vector<std::string> >* cw;
|
||||
@ -216,13 +216,12 @@ batch_run()
|
||||
boost::shared_ptr<TranslationTask> task;
|
||||
task = TranslationTask::create(source, ioWrapper, lscope);
|
||||
|
||||
if (cw)
|
||||
{
|
||||
if (context_string.size())
|
||||
context_window->push_back(context_string);
|
||||
if(!use_sliding_context_window)
|
||||
cw = NULL;
|
||||
}
|
||||
if (cw) {
|
||||
if (context_string.size())
|
||||
context_window->push_back(context_string);
|
||||
if(!use_sliding_context_window)
|
||||
cw = NULL;
|
||||
}
|
||||
if (context_window)
|
||||
task->SetContextWindow(context_window);
|
||||
|
||||
|
@ -44,8 +44,8 @@ void ConstrainedDecoding::Load()
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
bool addBeginEndWord
|
||||
= ((staticData.options().search.algo == CYKPlus)
|
||||
|| (staticData.options().search.algo == ChartIncremental));
|
||||
= ((staticData.options().search.algo == CYKPlus)
|
||||
|| (staticData.options().search.algo == ChartIncremental));
|
||||
|
||||
for(size_t i = 0; i < m_paths.size(); ++i) {
|
||||
InputFileStream constraintFile(m_paths[i]);
|
||||
|
@ -166,7 +166,7 @@ FeatureFactory
|
||||
weights.assign(feature->GetNumScoreComponents(),1.0);
|
||||
} else {
|
||||
VERBOSE(2,"WARNING: No weights specified in config file for FF "
|
||||
<< featureName << ". Using default values supplied by FF.");
|
||||
<< featureName << ". Using default values supplied by FF.");
|
||||
}
|
||||
}
|
||||
UTIL_THROW_IF2(weights.size() != feature->GetNumScoreComponents(),
|
||||
|
@ -79,12 +79,12 @@ namespace Moses
|
||||
|
||||
IOWrapper::IOWrapper()
|
||||
: m_nBestStream(NULL)
|
||||
// , m_outputWordGraphStream(NULL)
|
||||
// , m_outputSearchGraphStream(NULL)
|
||||
// , m_detailedTranslationReportingStream(NULL)
|
||||
// , m_unknownsStream(NULL)
|
||||
// , m_alignmentInfoStream(NULL)
|
||||
// , m_latticeSamplesStream(NULL)
|
||||
// , m_outputWordGraphStream(NULL)
|
||||
// , m_outputSearchGraphStream(NULL)
|
||||
// , m_detailedTranslationReportingStream(NULL)
|
||||
// , m_unknownsStream(NULL)
|
||||
// , m_alignmentInfoStream(NULL)
|
||||
// , m_latticeSamplesStream(NULL)
|
||||
, m_surpressSingleBestOutput(false)
|
||||
, m_look_ahead(0)
|
||||
, m_look_back(0)
|
||||
@ -193,8 +193,8 @@ IOWrapper::~IOWrapper()
|
||||
if (m_inputFile != NULL)
|
||||
delete m_inputFile;
|
||||
// if (m_nBestStream != NULL && !m_surpressSingleBestOutput) {
|
||||
// outputting n-best to file, rather than stdout. need to close file and delete obj
|
||||
// delete m_nBestStream;
|
||||
// outputting n-best to file, rather than stdout. need to close file and delete obj
|
||||
// delete m_nBestStream;
|
||||
// }
|
||||
|
||||
// delete m_detailedTranslationReportingStream;
|
||||
@ -253,13 +253,12 @@ ReadInput(boost::shared_ptr<std::vector<std::string> >* cw)
|
||||
source->SetTranslationId(m_currentLine++);
|
||||
|
||||
// when using a sliding context window, remove obsolete past input from buffer:
|
||||
if (m_past_input.size() && m_look_back != std::numeric_limits<size_t>::max())
|
||||
{
|
||||
list<boost::shared_ptr<InputType> >::iterator m = m_past_input.end();
|
||||
for (size_t cnt = 0; cnt < m_look_back && --m != m_past_input.begin();)
|
||||
cnt += (*m)->GetSize();
|
||||
while (m_past_input.begin() != m) m_past_input.pop_front();
|
||||
}
|
||||
if (m_past_input.size() && m_look_back != std::numeric_limits<size_t>::max()) {
|
||||
list<boost::shared_ptr<InputType> >::iterator m = m_past_input.end();
|
||||
for (size_t cnt = 0; cnt < m_look_back && --m != m_past_input.begin();)
|
||||
cnt += (*m)->GetSize();
|
||||
while (m_past_input.begin() != m) m_past_input.pop_front();
|
||||
}
|
||||
|
||||
if (m_look_back)
|
||||
m_past_input.push_back(source);
|
||||
@ -274,9 +273,9 @@ GetCurrentContextWindow() const
|
||||
{
|
||||
boost::shared_ptr<std::vector<string> > context(new std::vector<string>);
|
||||
BOOST_FOREACH(boost::shared_ptr<InputType> const& i, m_past_input)
|
||||
context->push_back(i->ToString());
|
||||
context->push_back(i->ToString());
|
||||
BOOST_FOREACH(boost::shared_ptr<InputType> const& i, m_future_input)
|
||||
context->push_back(i->ToString());
|
||||
context->push_back(i->ToString());
|
||||
return context;
|
||||
}
|
||||
|
||||
|
@ -500,8 +500,8 @@ ostream& operator<< (ostream& out, const Edge& edge)
|
||||
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b)
|
||||
{
|
||||
return (a->GetWordsBitmap().GetNumWordsCovered()
|
||||
<
|
||||
b->GetWordsBitmap().GetNumWordsCovered());
|
||||
<
|
||||
b->GetWordsBitmap().GetNumWordsCovered());
|
||||
}
|
||||
|
||||
void getLatticeMBRNBest(const Manager& manager, const TrellisPathList& nBestList,
|
||||
@ -515,11 +515,11 @@ void getLatticeMBRNBest(const Manager& manager, const TrellisPathList& nBestList
|
||||
map<const Hypothesis*, vector<Edge> > incomingEdges;
|
||||
vector< float> estimatedScores;
|
||||
manager.GetForwardBackwardSearchGraph(&connected, &connectedList,
|
||||
&outgoingHyps, &estimatedScores);
|
||||
&outgoingHyps, &estimatedScores);
|
||||
LMBR_Options const& lmbr = manager.options().lmbr;
|
||||
MBR_Options const& mbr = manager.options().mbr;
|
||||
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores,
|
||||
manager.GetBestHypothesis(), lmbr.pruning_factor, mbr.scale);
|
||||
manager.GetBestHypothesis(), lmbr.pruning_factor, mbr.scale);
|
||||
calcNgramExpectations(connectedList, incomingEdges, ngramPosteriors,true);
|
||||
|
||||
vector<float> mbrThetas = lmbr.theta;
|
||||
@ -581,7 +581,7 @@ const TrellisPath doConsensusDecoding(const Manager& manager, const TrellisPathL
|
||||
LMBR_Options const& lmbr = manager.options().lmbr;
|
||||
MBR_Options const& mbr = manager.options().mbr;
|
||||
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores,
|
||||
manager.GetBestHypothesis(), lmbr.pruning_factor, mbr.scale);
|
||||
manager.GetBestHypothesis(), lmbr.pruning_factor, mbr.scale);
|
||||
calcNgramExpectations(connectedList, incomingEdges, ngramExpectations,false);
|
||||
|
||||
//expected length is sum of expected unigram counts
|
||||
|
@ -1616,9 +1616,9 @@ void Manager::OutputNBest(OutputCollector *collector) const
|
||||
TrellisPathList nBestList;
|
||||
ostringstream out;
|
||||
CalcNBest(options().nbest.nbest_size, nBestList,
|
||||
options().nbest.only_distinct);
|
||||
options().nbest.only_distinct);
|
||||
OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(),
|
||||
m_source.GetTranslationId(),
|
||||
m_source.GetTranslationId(),
|
||||
staticData.GetReportSegmentation());
|
||||
collector->Write(m_source.GetTranslationId(), out.str());
|
||||
}
|
||||
|
@ -44,7 +44,7 @@ class OutputCollector
|
||||
{
|
||||
public:
|
||||
OutputCollector(std::ostream* outStream= &std::cout,
|
||||
std::ostream* debugStream=&std::cerr)
|
||||
std::ostream* debugStream=&std::cerr)
|
||||
: m_nextOutput(0)
|
||||
, m_outStream(outStream)
|
||||
, m_debugStream(debugStream)
|
||||
@ -52,37 +52,36 @@ public:
|
||||
, m_isHoldingDebugStream(false) {}
|
||||
|
||||
OutputCollector(std::string xout, std::string xerr = "")
|
||||
: m_nextOutput(0)
|
||||
{
|
||||
// TO DO open magic streams instead of regular ofstreams! [UG]
|
||||
: m_nextOutput(0) {
|
||||
// TO DO open magic streams instead of regular ofstreams! [UG]
|
||||
|
||||
if (xout == "/dev/stderr") {
|
||||
m_outStream = &std::cerr;
|
||||
m_isHoldingOutputStream = false;
|
||||
} else if (xout.size() && xout != "/dev/stdout" && xout != "-") {
|
||||
m_outStream = new std::ofstream(xout.c_str());
|
||||
UTIL_THROW_IF2(!m_outStream->good(), "Failed to open output file"
|
||||
<< xout);
|
||||
m_isHoldingOutputStream = true;
|
||||
} else {
|
||||
m_outStream = &std::cout;
|
||||
m_isHoldingOutputStream = false;
|
||||
}
|
||||
|
||||
if (xerr == "/dev/stdout") {
|
||||
m_debugStream = &std::cout;
|
||||
m_isHoldingDebugStream = false;
|
||||
} else if (xerr.size() && xerr != "/dev/stderr") {
|
||||
m_debugStream = new std::ofstream(xerr.c_str());
|
||||
UTIL_THROW_IF2(!m_debugStream->good(), "Failed to open debug stream"
|
||||
<< xerr);
|
||||
m_isHoldingDebugStream = true;
|
||||
} else {
|
||||
m_debugStream = &std::cerr;
|
||||
m_isHoldingDebugStream = false;
|
||||
}
|
||||
if (xout == "/dev/stderr") {
|
||||
m_outStream = &std::cerr;
|
||||
m_isHoldingOutputStream = false;
|
||||
} else if (xout.size() && xout != "/dev/stdout" && xout != "-") {
|
||||
m_outStream = new std::ofstream(xout.c_str());
|
||||
UTIL_THROW_IF2(!m_outStream->good(), "Failed to open output file"
|
||||
<< xout);
|
||||
m_isHoldingOutputStream = true;
|
||||
} else {
|
||||
m_outStream = &std::cout;
|
||||
m_isHoldingOutputStream = false;
|
||||
}
|
||||
|
||||
if (xerr == "/dev/stdout") {
|
||||
m_debugStream = &std::cout;
|
||||
m_isHoldingDebugStream = false;
|
||||
} else if (xerr.size() && xerr != "/dev/stderr") {
|
||||
m_debugStream = new std::ofstream(xerr.c_str());
|
||||
UTIL_THROW_IF2(!m_debugStream->good(), "Failed to open debug stream"
|
||||
<< xerr);
|
||||
m_isHoldingDebugStream = true;
|
||||
} else {
|
||||
m_debugStream = &std::cerr;
|
||||
m_isHoldingDebugStream = false;
|
||||
}
|
||||
}
|
||||
|
||||
~OutputCollector() {
|
||||
if (m_isHoldingOutputStream)
|
||||
delete m_outStream;
|
||||
|
@ -215,9 +215,9 @@ Parameter::Parameter()
|
||||
AddParam(server_opts,"server-port", "Port for moses server");
|
||||
AddParam(server_opts,"server-log", "Log destination for moses server");
|
||||
AddParam(server_opts,"session-timeout",
|
||||
"Timeout for sessions, e.g. '2h30m' or 1d (=24h)");
|
||||
"Timeout for sessions, e.g. '2h30m' or 1d (=24h)");
|
||||
AddParam(server_opts,"session-cache-size", string("Max. number of sessions cached.")
|
||||
+"Least recently used session is dumped first.");
|
||||
+"Least recently used session is dumped first.");
|
||||
AddParam(server_opts,"serial", "Run server in serial mode, processing only one request at a time.");
|
||||
|
||||
po::options_description irstlm_opts("IRSTLM Options");
|
||||
|
@ -20,8 +20,8 @@ Search::Search(Manager& manager)
|
||||
Search *
|
||||
Search::
|
||||
CreateSearch(Manager& manager, const InputType &source,
|
||||
SearchAlgorithm searchAlgorithm,
|
||||
const TranslationOptionCollection &transOptColl)
|
||||
SearchAlgorithm searchAlgorithm,
|
||||
const TranslationOptionCollection &transOptColl)
|
||||
{
|
||||
switch(searchAlgorithm) {
|
||||
case Normal:
|
||||
@ -43,7 +43,7 @@ out_of_time()
|
||||
double elapsed_time = GetUserTime();
|
||||
if (elapsed_time <= timelimit) return false;
|
||||
VERBOSE(1,"Decoding is out of time (" << elapsed_time << ","
|
||||
<< timelimit << ")" << std::endl);
|
||||
<< timelimit << ")" << std::endl);
|
||||
interrupted_flag = 1;
|
||||
return true;
|
||||
}
|
||||
|
@ -93,7 +93,7 @@ void SearchCubePruning::Decode()
|
||||
double _elapsed_time = GetUserTime();
|
||||
if (timelimit && _elapsed_time > timelimit) {
|
||||
VERBOSE(1,"Decoding is out of time (" << _elapsed_time << ","
|
||||
<< timelimit << ")" << std::endl);
|
||||
<< timelimit << ")" << std::endl);
|
||||
return;
|
||||
}
|
||||
HypothesisStackCubePruning &sourceHypoColl = *static_cast<HypothesisStackCubePruning*>(*iterStack);
|
||||
|
@ -17,7 +17,7 @@ namespace Moses
|
||||
*/
|
||||
SearchNormal::
|
||||
SearchNormal(Manager& manager, const InputType &source,
|
||||
const TranslationOptionCollection &transOptColl)
|
||||
const TranslationOptionCollection &transOptColl)
|
||||
: Search(manager)
|
||||
, m_source(source)
|
||||
, m_hypoStackColl(source.GetSize() + 1)
|
||||
@ -39,7 +39,7 @@ SearchNormal(Manager& manager, const InputType &source,
|
||||
for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind) {
|
||||
HypothesisStackNormal *sourceHypoColl = new HypothesisStackNormal(m_manager);
|
||||
sourceHypoColl->SetMaxHypoStackSize(this->m_options.search.stack_size,
|
||||
this->m_options.search.stack_diversity);
|
||||
this->m_options.search.stack_diversity);
|
||||
sourceHypoColl->SetBeamWidth(this->m_options.search.beam_width);
|
||||
m_hypoStackColl[ind] = sourceHypoColl;
|
||||
}
|
||||
@ -58,7 +58,7 @@ ProcessOneStack(HypothesisStack* hstack)
|
||||
if (this->out_of_time()) return false;
|
||||
SentenceStats &stats = m_manager.GetSentenceStats();
|
||||
HypothesisStackNormal &sourceHypoColl
|
||||
= *static_cast<HypothesisStackNormal*>(hstack);
|
||||
= *static_cast<HypothesisStackNormal*>(hstack);
|
||||
|
||||
// the stack is pruned before processing (lazy pruning):
|
||||
VERBOSE(3,"processing hypothesis from next stack");
|
||||
@ -70,7 +70,7 @@ ProcessOneStack(HypothesisStack* hstack)
|
||||
|
||||
// go through each hypothesis on the stack and try to expand it
|
||||
BOOST_FOREACH(Hypothesis* h, sourceHypoColl)
|
||||
ProcessOneHypothesis(*h);
|
||||
ProcessOneHypothesis(*h);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -45,7 +45,7 @@ protected:
|
||||
|
||||
virtual void
|
||||
ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt,
|
||||
float expectedScore);
|
||||
float expectedScore);
|
||||
|
||||
public:
|
||||
SearchNormal(Manager& manager, const InputType &source, const TranslationOptionCollection &transOptColl);
|
||||
|
@ -268,11 +268,11 @@ StaticData
|
||||
|
||||
// additional output
|
||||
m_parameter->SetParameter<string>(m_detailedTranslationReportingFilePath,
|
||||
"translation-details", "");
|
||||
"translation-details", "");
|
||||
m_parameter->SetParameter<string>(m_detailedTreeFragmentsTranslationReportingFilePath,
|
||||
"tree-translation-details", "");
|
||||
"tree-translation-details", "");
|
||||
m_parameter->SetParameter<string>(m_detailedAllTranslationReportingFilePath,
|
||||
"translation-all-details", "");
|
||||
"translation-all-details", "");
|
||||
m_parameter->SetParameter<long>(m_startTranslationId, "start-translation-id", 0);
|
||||
|
||||
//lattice samples
|
||||
|
@ -368,7 +368,7 @@ public:
|
||||
|
||||
bool UseEarlyDiscarding() const {
|
||||
return m_options.search.early_discarding_threshold
|
||||
!= -std::numeric_limits<float>::infinity();
|
||||
!= -std::numeric_limits<float>::infinity();
|
||||
}
|
||||
bool UseEarlyDistortionCost() const {
|
||||
return m_options.reordering.use_early_distortion_cost;
|
||||
|
@ -182,7 +182,8 @@ bool TargetPhrase::HasTtaskSPtr() const
|
||||
return m_ttask_flag;
|
||||
}
|
||||
|
||||
const ttasksptr TargetPhrase::GetTtask() const {
|
||||
const ttasksptr TargetPhrase::GetTtask() const
|
||||
{
|
||||
return m_ttask.lock();
|
||||
}
|
||||
|
||||
|
@ -88,7 +88,7 @@ boost::shared_ptr<TranslationTask>
|
||||
TranslationTask
|
||||
::create(boost::shared_ptr<InputType> const& source,
|
||||
boost::shared_ptr<IOWrapper> const& ioWrapper,
|
||||
boost::shared_ptr<ContextScope> const& scope)
|
||||
boost::shared_ptr<ContextScope> const& scope)
|
||||
{
|
||||
boost::shared_ptr<TranslationTask> ret(new TranslationTask(source, ioWrapper));
|
||||
ret->m_self = ret;
|
||||
|
@ -97,7 +97,7 @@ public:
|
||||
boost::shared_ptr<TranslationTask>
|
||||
create(boost::shared_ptr<Moses::InputType> const& source,
|
||||
boost::shared_ptr<Moses::IOWrapper> const& ioWrapper,
|
||||
boost::shared_ptr<ContextScope> const& scope);
|
||||
boost::shared_ptr<ContextScope> const& scope);
|
||||
|
||||
~TranslationTask();
|
||||
/** Translate one sentence
|
||||
|
Loading…
Reference in New Issue
Block a user