beautify. Add sed for trailing spaces

This commit is contained in:
Hieu Hoang 2015-05-13 11:29:16 +01:00
parent 0859e9a844
commit 5173b9f617
15 changed files with 130 additions and 141 deletions

View File

@ -44,8 +44,8 @@ OutputSearchGraphAsHypergraph(std::string const& fname, size_t const precision)
if (! boost::filesystem::exists(odir))
boost::filesystem::create_directory(odir);
UTIL_THROW_IF2(!boost::filesystem::is_directory(odir),
"Cannot output hypergraphs to " << odir
<< " because that path exists but is not a directory.");
"Cannot output hypergraphs to " << odir
<< " because that path exists but is not a directory.");
// not clear why we need to output the weights every time we dump a search
// graph into a file again, but that's what the old code did.
@ -67,21 +67,18 @@ OutputSearchGraphAsHypergraph(std::string const& fname, size_t const precision)
else if (boost::ends_with(fname, ".bz2"))
file.push( boost::iostreams::bzip2_compressor() );
file.push( boost::iostreams::file_sink(fname, ios_base::out) );
if (file.is_complete() && file.good())
{
file.setf(std::ios::fixed);
file.precision(precision);
this->OutputSearchGraphAsHypergraph(file);
file.flush();
}
else
{
TRACE_ERR("Cannot output hypergraph for line "
<< this->GetSource().GetTranslationId()
<< " because the output file " << fname
<< " is not open or not ready for writing"
<< std::endl);
}
if (file.is_complete() && file.good()) {
file.setf(std::ios::fixed);
file.precision(precision);
this->OutputSearchGraphAsHypergraph(file);
file.flush();
} else {
TRACE_ERR("Cannot output hypergraph for line "
<< this->GetSource().GetTranslationId()
<< " because the output file " << fname
<< " is not open or not ready for writing"
<< std::endl);
}
file.pop();
}

View File

@ -66,13 +66,13 @@ public:
// virtual void OutputSearchGraphHypergraph() const = 0;
virtual void OutputSearchGraphAsHypergraph(std::ostream& out) const;
virtual void OutputSearchGraphAsHypergraph(std::string const& fname,
size_t const precision) const;
virtual void OutputSearchGraphAsHypergraph(std::string const& fname,
size_t const precision) const;
/***
* to be called after processing a sentence
*/
virtual void CalcDecoderStatistics() const = 0;
};
}

View File

@ -50,7 +50,7 @@ template class HypergraphOutput<ChartManager>;
void
ChartSearchGraphWriterMoses::
WriteHypos(const ChartHypothesisCollection& hypos,
const map<unsigned, bool> &reachable) const
const map<unsigned, bool> &reachable) const
{
ChartHypothesisCollection::const_iterator iter;
@ -62,16 +62,14 @@ WriteHypos(const ChartHypothesisCollection& hypos,
}
const ChartArcList *arcList = mainHypo.GetArcList();
if (arcList)
{
ChartArcList::const_iterator iterArc;
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc)
{
const ChartHypothesis &arc = **iterArc;
if (reachable.find(arc.GetId()) != reachable.end())
(*m_out) << m_lineNumber << " " << arc << endl;
}
if (arcList) {
ChartArcList::const_iterator iterArc;
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
const ChartHypothesis &arc = **iterArc;
if (reachable.find(arc.GetId()) != reachable.end())
(*m_out) << m_lineNumber << " " << arc << endl;
}
}
}
}
@ -86,7 +84,7 @@ WriteHeader(size_t winners, size_t losers) const
void
ChartSearchGraphWriterHypergraph::
WriteHypos(const ChartHypothesisCollection& hypos,
const map<unsigned, bool> &reachable) const
const map<unsigned, bool> &reachable) const
{
ChartHypothesisCollection::const_iterator iter;
@ -114,7 +112,7 @@ WriteHypos(const ChartHypothesisCollection& hypos,
}
(*m_out) << edges.size() << endl;
for (vector<const ChartHypothesis*>::const_iterator ei = edges.begin();
ei != edges.end(); ++ei) {
ei != edges.end(); ++ei) {
const ChartHypothesis* hypo = *ei;
const TargetPhrase& target = hypo->GetCurrTargetPhrase();
size_t ntIndex = 0;

View File

@ -313,7 +313,7 @@ struct CompareHypothesisTotalScore {
ObjectPool<Hypothesis> &pool = Hypothesis::GetObjectPool(); \
pool.freeObject(hypo); \
} \
#else
#define FREEHYPO(hypo) delete hypo
#endif

View File

@ -101,7 +101,7 @@ IOWrapper::IOWrapper()
m_inputType = staticData.GetInputType();
UTIL_THROW_IF2((m_look_ahead || m_look_back) && m_inputType != SentenceInput,
"Context-sensitive decoding currently works only with sentence input.");
"Context-sensitive decoding currently works only with sentence input.");
m_currentLine = staticData.GetStartTranslationId();
@ -224,17 +224,15 @@ IOWrapper::IOWrapper()
// first, determine the output directory
if (p && p->size() > 2) fmt = p->at(2);
else if (nBestFilePath.size() && nBestFilePath != "-" &&
! boost::starts_with(nBestFilePath, "/dev/stdout"))
{
fmt = boost::filesystem::path(nBestFilePath).parent_path().string();
if (fmt.empty()) fmt = ".";
}
else fmt = boost::filesystem::current_path().string() + "/hypergraph";
! boost::starts_with(nBestFilePath, "/dev/stdout")) {
fmt = boost::filesystem::path(nBestFilePath).parent_path().string();
if (fmt.empty()) fmt = ".";
} else fmt = boost::filesystem::current_path().string() + "/hypergraph";
if (*fmt.rbegin() != '/') fmt += "/";
std::string extension = (p && p->size() > 1 ? p->at(1) : std::string("txt"));
UTIL_THROW_IF2(extension != "txt" && extension != "gz" && extension != "bz2",
"Unknown compression type '" << extension
<< "' for hypergraph output!");
"Unknown compression type '" << extension
<< "' for hypergraph output!");
fmt += string("%d.") + extension;
if (staticData.GetParameter().GetParam("spe-src")) {
@ -304,11 +302,10 @@ IOWrapper::ReadInput()
boost::lock_guard<boost::mutex> lock(m_lock);
#endif
boost::shared_ptr<InputType> source = GetBufferedInput();
if (source)
{
source->SetTranslationId(m_currentLine++);
this->set_context_for(*source);
}
if (source) {
source->SetTranslationId(m_currentLine++);
this->set_context_for(*source);
}
m_past_input.push_back(source);
return source;
}
@ -320,37 +317,32 @@ set_context_for(InputType& source)
boost::shared_ptr<string> context(new string);
list<boost::shared_ptr<InputType> >::iterator m = m_past_input.end();
// remove obsolete past input from buffer:
if (m_past_input.end() != m_past_input.begin())
{
for (size_t cnt = 0; cnt < m_look_back && --m != m_past_input.begin();
cnt += (*m)->GetSize());
while (m_past_input.begin() != m) m_past_input.pop_front();
}
if (m_past_input.end() != m_past_input.begin()) {
for (size_t cnt = 0; cnt < m_look_back && --m != m_past_input.begin();
cnt += (*m)->GetSize());
while (m_past_input.begin() != m) m_past_input.pop_front();
}
// cerr << string(80,'=') << endl;
if (m_past_input.size())
{
m = m_past_input.begin();
*context += (*m)->ToString();
// cerr << (*m)->ToString() << endl;
for (++m; m != m_past_input.end(); ++m)
{
// cerr << "\n" << (*m)->ToString() << endl;
*context += string(" ") + (*m)->ToString();
}
// cerr << string(80,'-') << endl;
if (m_past_input.size()) {
m = m_past_input.begin();
*context += (*m)->ToString();
// cerr << (*m)->ToString() << endl;
for (++m; m != m_past_input.end(); ++m) {
// cerr << "\n" << (*m)->ToString() << endl;
*context += string(" ") + (*m)->ToString();
}
// cerr << string(80,'-') << endl;
}
// cerr << source.ToString() << endl;
if (m_future_input.size())
{
// cerr << string(80,'-') << endl;
for (m = m_future_input.begin(); m != m_future_input.end(); ++m)
{
// if (m != m_future_input.begin()) cerr << "\n";
// cerr << (*m)->ToString() << endl;
if (context->size()) *context += " ";
*context += (*m)->ToString();
}
if (m_future_input.size()) {
// cerr << string(80,'-') << endl;
for (m = m_future_input.begin(); m != m_future_input.end(); ++m) {
// if (m != m_future_input.begin()) cerr << "\n";
// cerr << (*m)->ToString() << endl;
if (context->size()) *context += " ";
*context += (*m)->ToString();
}
}
// cerr << string(80,'=') << endl;
source.SetContext(context);
}

View File

@ -119,7 +119,7 @@ protected:
size_t m_look_ahead; /// for context-sensitive decoding: # of wrds to look ahead
size_t m_look_back; /// for context-sensitive decoding: # of wrds to look back
size_t m_buffered_ahead; /// number of words buffered ahead
// For context-sensitive decoding:
// For context-sensitive decoding:
// Number of context words ahead and before the current sentence.
std::string m_hypergraph_output_filepattern;
@ -184,14 +184,14 @@ public:
std::ifstream *spe_src, *spe_trg, *spe_aln;
private:
template<class itype>
template<class itype>
boost::shared_ptr<InputType>
BufferInput();
boost::shared_ptr<InputType>
GetBufferedInput();
void
void
set_context_for(InputType& source);
};
@ -202,27 +202,23 @@ BufferInput()
{
boost::shared_ptr<itype> source;
boost::shared_ptr<InputType> ret;
if (m_future_input.size())
{
ret = m_future_input.front();
m_future_input.pop_front();
m_buffered_ahead -= ret->GetSize();
}
else
{
source.reset(new itype);
if (!source->Read(*m_inputStream, *m_inputFactorOrder))
return ret;
ret = source;
}
if (m_future_input.size()) {
ret = m_future_input.front();
m_future_input.pop_front();
m_buffered_ahead -= ret->GetSize();
} else {
source.reset(new itype);
if (!source->Read(*m_inputStream, *m_inputFactorOrder))
return ret;
ret = source;
}
while (m_buffered_ahead < m_look_ahead)
{
source.reset(new itype);
if (!source->Read(*m_inputStream, *m_inputFactorOrder)) break;
m_future_input.push_back(source);
m_buffered_ahead += source->GetSize();
}
while (m_buffered_ahead < m_look_ahead) {
source.reset(new itype);
if (!source->Read(*m_inputStream, *m_inputFactorOrder)) break;
m_future_input.push_back(source);
m_buffered_ahead += source->GetSize();
}
return ret;
}

View File

@ -57,9 +57,9 @@ public:
void OutputSearchGraphSLF() const {
}
void
void
OutputSearchGraphAsHypergraph
( std::string const& fname, size_t const precision ) const
( std::string const& fname, size_t const precision ) const
{ }

View File

@ -173,13 +173,16 @@ public:
//! number of words in this sentence/confusion network
virtual size_t GetSize() const =0;
virtual boost::shared_ptr<std::string> const&
GetContext() const { return m_context; }
virtual void
SetContext(boost::shared_ptr<std::string> const& ctx)
{ m_context = ctx; }
virtual boost::shared_ptr<std::string> const&
GetContext() const {
return m_context;
}
virtual void
SetContext(boost::shared_ptr<std::string> const& ctx) {
m_context = ctx;
}
//! populate this InputType with data from in stream
virtual int Read(std::istream& in,const std::vector<FactorType>& factorOrder) =0;

View File

@ -241,9 +241,9 @@ Parameter::Parameter()
AddParam(misc_opts,"feature", "All the feature functions should be here");
AddParam(misc_opts,"context-string",
"A (tokenized) string containing context words for context-sensitive translation.");
"A (tokenized) string containing context words for context-sensitive translation.");
AddParam(misc_opts,"context-window",
"Context window (in words) for context-sensitive translation: {+|-|+-}<number>.");
"Context window (in words) for context-sensitive translation: {+|-|+-}<number>.");
// Compact phrase table and reordering table.
po::options_description cpt_opts("Options when using compact phrase and reordering tables.");

View File

@ -301,7 +301,9 @@ public:
}
const ContextParameters&
GetContextParameters() const { return m_context_parameters; }
GetContextParameters() const {
return m_context_parameters;
}
const std::vector<FactorType> &GetInputFactorOrder() const {
return m_inputFactorOrder;
@ -449,15 +451,15 @@ public:
}
bool IsNBestEnabled() const {
return (!m_nBestFilePath.empty() || m_mbr || m_useLatticeMBR || m_mira ||
m_outputSearchGraph || m_outputSearchGraphSLF ||
m_outputSearchGraphHypergraph || m_useConsensusDecoding ||
return (!m_nBestFilePath.empty() || m_mbr || m_useLatticeMBR || m_mira ||
m_outputSearchGraph || m_outputSearchGraphSLF ||
m_outputSearchGraphHypergraph || m_useConsensusDecoding ||
#ifdef HAVE_PROTOBUF
m_outputSearchGraphPB ||
m_outputSearchGraphPB ||
#endif
!m_latticeSamplesFilePath.empty());
!m_latticeSamplesFilePath.empty());
}
size_t GetLatticeSamplesSize() const {
return m_latticeSamplesSize;
}

View File

@ -31,9 +31,9 @@ public:
void OutputSearchGraph(OutputCollector *collector) const {}
// void OutputSearchGraphHypergraph() const {}
void
void
OutputSearchGraphAsHypergraph
( std::string const& fname, size_t const precision ) const
( std::string const& fname, size_t const precision ) const
{ }
void OutputSearchGraphSLF() const {}

View File

@ -178,12 +178,11 @@ void TranslationTask::Run()
// Output search graph in hypergraph format for Kenneth Heafield's
// lazy hypergraph decoder; writes to stderr
if (StaticData::Instance().GetOutputSearchGraphHypergraph())
{
size_t transId = manager->GetSource().GetTranslationId();
string fname = io->GetHypergraphOutputFileName(transId);
manager->OutputSearchGraphAsHypergraph(fname, PRECISION);
}
if (StaticData::Instance().GetOutputSearchGraphHypergraph()) {
size_t transId = manager->GetSource().GetTranslationId();
string fname = io->GetHypergraphOutputFileName(transId);
manager->OutputSearchGraphAsHypergraph(fname, PRECISION);
}
additionalReportingTime.stop();

View File

@ -427,7 +427,7 @@ inline float CalcTranslationScore(const std::vector<float> &probVector,
out << *this; \
return out.str(); \
} \
//! delete and remove every element of a collection object such as set, list etc
template<class COLL>
void RemoveAllInColl(COLL &coll)

View File

@ -23,22 +23,20 @@ init(Parameter& params)
size_t p = context_window.find_first_of("0123456789");
if (p == 0)
look_back = look_ahead = atoi(context_window.c_str());
if (p == 1)
{
if (context_window[0] == '-')
look_back = atoi(context_window.substr(1).c_str());
else if (context_window[0] == '+')
look_ahead = atoi(context_window.substr(1).c_str());
else
UTIL_THROW2("Invalid specification of context window.");
}
if (p == 2)
{
if (context_window.substr(0,2) == "+-" ||
context_window.substr(0,2) == "-+")
look_back = look_ahead = atoi(context_window.substr(p).c_str());
else
UTIL_THROW2("Invalid specification of context window.");
}
if (p == 1) {
if (context_window[0] == '-')
look_back = atoi(context_window.substr(1).c_str());
else if (context_window[0] == '+')
look_ahead = atoi(context_window.substr(1).c_str());
else
UTIL_THROW2("Invalid specification of context window.");
}
if (p == 2) {
if (context_window.substr(0,2) == "+-" ||
context_window.substr(0,2) == "-+")
look_back = look_ahead = atoi(context_window.substr(p).c_str());
else
UTIL_THROW2("Invalid specification of context window.");
}
}
}

View File

@ -56,6 +56,10 @@ sub Beautify($)
my $cmd = "astyle --style='k&r' -s2 -v $name";
#print STDERR "Executing: $cmd \n";
`$cmd`;
$cmd = "sed -i 's\/[ \\t]*\$\/\/' \"$name\"";
#print STDERR "Executing: $cmd \n";
`$cmd`;
}
}
}