moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-07 20:17:48 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'perf_baseline' of github.com:hieuhoang/mosesdecoder into perf_baseline

Browse Source
This commit is contained in:
Hieu Hoang 2015-10-17 22:25:13 +01:00
commit 1f6a787c2a
43 changed files with 148 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
OnDiskPt/OnDiskWrapper.cpp
View File

@ -25,6 +25,7 @@
#include "OnDiskWrapper.h" #include "OnDiskWrapper.h"
#include "moses/Factor.h" #include "moses/Factor.h"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -223,7 +224,8 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact
{ {
bool isNonTerminal = origWord.IsNonTerminal(); bool isNonTerminal = origWord.IsNonTerminal();
Word *newWord = new Word(isNonTerminal); Word *newWord = new Word(isNonTerminal);
stringstream strme;
util::StringStream strme;
size_t factorType = factorsVec[0]; size_t factorType = factorsVec[0];
const Moses::Factor *factor = origWord.GetFactor(factorType); const Moses::Factor *factor = origWord.GetFactor(factorType);

5
lm/filter/arpa_io.cc
View File

@ -33,10 +33,9 @@ template <class Stream> void WriteCounts(Stream &out, const std::vector<uint64_t
} }
size_t SizeNeededForCounts(const std::vector<uint64_t> &number) { size_t SizeNeededForCounts(const std::vector<uint64_t> &number) {
std::string buf; util::StringStream stream;
util::StringStream stream(buf);
WriteCounts(stream, number); WriteCounts(stream, number);
return buf.size(); return stream.str().size();
} }
bool IsEntirelyWhiteSpace(const StringPiece &line) { bool IsEntirelyWhiteSpace(const StringPiece &line) {

1
moses-cmd/Main.cpp
View File

@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
* Moses main wrapper for executable for single-threaded and multi-threaded, simply calling decoder_main. * Moses main wrapper for executable for single-threaded and multi-threaded, simply calling decoder_main.
**/ **/
#include "moses/ExportInterface.h" #include "moses/ExportInterface.h"
#include "util/string_stream.hh"
/** main function of the command line version of the decoder **/ /** main function of the command line version of the decoder **/
int main(int argc, char** argv) int main(int argc, char** argv)

14
moses/ChartHypothesisCollection.cpp
View File

@ -167,20 +167,6 @@ void ChartHypothesisCollection::Detach(const HCType::iterator &iter)
void ChartHypothesisCollection::Remove(const HCType::iterator &iter) void ChartHypothesisCollection::Remove(const HCType::iterator &iter)
{ {
ChartHypothesis *h = *iter; ChartHypothesis *h = *iter;
/*
stringstream strme("");
strme << h->GetOutputPhrase();
string toFind = "the goal of gene scientists is ";
size_t pos = toFind.find(strme.str());
if (pos == 0)
{
cerr << pos << " " << strme.str() << *h << endl;
cerr << *this << endl;
}
*/
Detach(iter); Detach(iter);
ChartHypothesis::Delete(h); ChartHypothesis::Delete(h);
} }

6
moses/FF/FeatureFunction.cpp
View File

@ -112,12 +112,10 @@ void FeatureFunction::ParseLine(const std::string &line)
if (m_description == "") { if (m_description == "") {
size_t index = description_counts.count(nameStub); size_t index = description_counts.count(nameStub);
ostringstream dstream; string descr = SPrint(nameStub) + SPrint(index);
dstream << nameStub;
dstream << index;
description_counts.insert(nameStub); description_counts.insert(nameStub);
m_description = dstream.str(); m_description = descr;
} }
} }

11
moses/FF/GlobalLexicalModelUnlimited.cpp
View File

@ -5,6 +5,7 @@
#include "moses/Hypothesis.h" #include "moses/Hypothesis.h"
#include "moses/TranslationTask.h" #include "moses/TranslationTask.h"
#include "util/string_piece_hash.hh" #include "util/string_piece_hash.hh"
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -131,7 +132,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
} }
if (m_biasFeature) { if (m_biasFeature) {
stringstream feature; util::StringStream feature;
feature << "glm_"; feature << "glm_";
feature << targetString; feature << targetString;
feature << "~"; feature << "~";
@ -165,7 +166,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
if (m_sourceContext) { if (m_sourceContext) {
if (sourceIndex == 0) { if (sourceIndex == 0) {
// add <s> trigger feature for source // add <s> trigger feature for source
stringstream feature; util::StringStream feature;
feature << "glm_"; feature << "glm_";
feature << targetString; feature << targetString;
feature << "~"; feature << "~";
@ -183,7 +184,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
contextExists = FindStringPiece(m_vocabSource, contextString ) != m_vocabSource.end(); contextExists = FindStringPiece(m_vocabSource, contextString ) != m_vocabSource.end();
if (m_unrestricted || contextExists) { if (m_unrestricted || contextExists) {
stringstream feature; util::StringStream feature;
feature << "glm_"; feature << "glm_";
feature << targetString; feature << targetString;
feature << "~"; feature << "~";
@ -304,7 +305,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
} }
} }
} else { } else {
stringstream feature; util::StringStream feature;
feature << "glm_"; feature << "glm_";
feature << targetString; feature << targetString;
feature << "~"; feature << "~";
@ -323,7 +324,7 @@ void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulat
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece sourceTrigger, StringPiece sourceWord,
StringPiece targetTrigger, StringPiece targetWord) const StringPiece targetTrigger, StringPiece targetWord) const
{ {
stringstream feature; util::StringStream feature;
feature << "glm_"; feature << "glm_";
feature << targetTrigger; feature << targetTrigger;
feature << ","; feature << ",";

5
moses/FF/LexicalReordering/SparseReordering.cpp
View File

@ -8,6 +8,7 @@
#include "util/file_piece.hh" #include "util/file_piece.hh"
#include "util/string_piece.hh" #include "util/string_piece.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh" #include "util/tokenize_piece.hh"
#include "LexicalReordering.h" #include "LexicalReordering.h"
@ -26,7 +27,7 @@ const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
{ {
static string kSep = "-"; static string kSep = "-";
static string name; static string name;
ostringstream buf; util::StringStream buf;
// type side position id word reotype // type side position id word reotype
if (type == Phrase) { if (type == Phrase) {
buf << "phr"; buf << "phr";
@ -88,7 +89,7 @@ SparseReordering::SparseReordering(const map<string,string>& config, const Lexic
ReadWeightMap(i->second); ReadWeightMap(i->second);
m_useWeightMap = true; m_useWeightMap = true;
for (int reoType=0; reoType<=LRModel::MAX; ++reoType) { for (int reoType=0; reoType<=LRModel::MAX; ++reoType) {
ostringstream buf; util::StringStream buf;
buf << reoType; buf << reoType;
m_featureMap2.push_back(m_producer->GetFeatureName(buf.str())); m_featureMap2.push_back(m_producer->GetFeatureName(buf.str()));
} }

6
moses/FF/OSM-Feature/osmHyp.cpp
View File

@ -165,11 +165,7 @@ int osmHypothesis :: firstOpenGap(vector <int> & coverageVector)
string osmHypothesis :: intToString(int num) string osmHypothesis :: intToString(int num)
{ {
return SPrint(num);
std::ostringstream stm;
stm<<num;
return stm.str();
} }

3
moses/FF/PhraseBoundaryFeature.cpp
View File

@ -3,6 +3,7 @@
#include "moses/Hypothesis.h" #include "moses/Hypothesis.h"
#include "moses/TranslationOption.h" #include "moses/TranslationOption.h"
#include "moses/InputPath.h" #include "moses/InputPath.h"
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -53,7 +54,7 @@ void PhraseBoundaryFeature::AddFeatures(
ScoreComponentCollection* scores) const ScoreComponentCollection* scores) const
{ {
for (size_t i = 0; i < factors.size(); ++i) { for (size_t i = 0; i < factors.size(); ++i) {
ostringstream name; util::StringStream name;
name << side << ":"; name << side << ":";
name << factors[i]; name << factors[i];
name << ":"; name << ":";

7
moses/FF/PhraseLengthFeature.cpp
View File

@ -3,6 +3,7 @@
#include "moses/Hypothesis.h" #include "moses/Hypothesis.h"
#include "moses/ScoreComponentCollection.h" #include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h" #include "moses/TranslationOption.h"
#include "util/string_stream.hh"
namespace Moses namespace Moses
{ {
@ -25,13 +26,13 @@ void PhraseLengthFeature::EvaluateInIsolation(const Phrase &source
size_t sourceLength = source.GetSize(); size_t sourceLength = source.GetSize();
// create feature names // create feature names
stringstream nameSource; util::StringStream nameSource;
nameSource << "s" << sourceLength; nameSource << "s" << sourceLength;
stringstream nameTarget; util::StringStream nameTarget;
nameTarget << "t" << targetLength; nameTarget << "t" << targetLength;
stringstream nameBoth; util::StringStream nameBoth;
nameBoth << sourceLength << "," << targetLength; nameBoth << sourceLength << "," << targetLength;
// increase feature counts // increase feature counts

17
moses/FF/PhrasePairFeature.cpp
View File

@ -7,6 +7,7 @@
#include "moses/TranslationOption.h" #include "moses/TranslationOption.h"
#include "moses/InputPath.h" #include "moses/InputPath.h"
#include "util/string_piece_hash.hh" #include "util/string_piece_hash.hh"
#include "util/string_stream.hh"
#include "util/exception.hh" #include "util/exception.hh"
using namespace std; using namespace std;
@ -126,7 +127,8 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
const bool use_topicid_prob = isnt.GetUseTopicIdAndProb(); const bool use_topicid_prob = isnt.GetUseTopicIdAndProb();
// compute pair // compute pair
ostringstream pair; util::StringStream pair;
pair << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() ); pair << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() );
for (size_t i = 1; i < source.GetSize(); ++i) { for (size_t i = 1; i < source.GetSize(); ++i) {
const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId); const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId);
@ -145,7 +147,8 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
if(use_topicid) { if(use_topicid) {
// use topicid as trigger // use topicid as trigger
const long topicid = isnt.GetTopicId(); const long topicid = isnt.GetTopicId();
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
if (topicid == -1) if (topicid == -1)
feature << "unk"; feature << "unk";
@ -159,13 +162,13 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
// use topic probabilities // use topic probabilities
const vector<string> &topicid_prob = *(isnt.GetTopicIdAndProb()); const vector<string> &topicid_prob = *(isnt.GetTopicIdAndProb());
if (atol(topicid_prob[0].c_str()) == -1) { if (atol(topicid_prob[0].c_str()) == -1) {
stringstream feature; util::StringStream feature;
feature << m_description << "_unk_"; feature << m_description << "_unk_";
feature << pair.str(); feature << pair.str();
scoreBreakdown.SparsePlusEquals(feature.str(), 1); scoreBreakdown.SparsePlusEquals(feature.str(), 1);
} else { } else {
for (size_t i=0; i+1 < topicid_prob.size(); i+=2) { for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
feature << topicid_prob[i]; feature << topicid_prob[i];
feature << "_"; feature << "_";
@ -179,7 +182,7 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
const long docid = isnt.GetDocumentId(); const long docid = isnt.GetDocumentId();
for (set<string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) { for (set<string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
string sourceTrigger = *p; string sourceTrigger = *p;
ostringstream namestr; util::StringStream namestr;
namestr << m_description << "_"; namestr << m_description << "_";
namestr << sourceTrigger; namestr << sourceTrigger;
namestr << "_"; namestr << "_";
@ -207,7 +210,7 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end(); sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
if (m_unrestricted || sourceTriggerExists) { if (m_unrestricted || sourceTriggerExists) {
ostringstream namestr; util::StringStream namestr;
namestr << m_description << "_"; namestr << m_description << "_";
namestr << sourceTrigger; namestr << sourceTrigger;
namestr << "~"; namestr << "~";
@ -237,7 +240,7 @@ void PhrasePairFeature::EvaluateInIsolation(const Phrase &source
, ScoreComponentCollection &estimatedFutureScore) const , ScoreComponentCollection &estimatedFutureScore) const
{ {
if (m_simple) { if (m_simple) {
ostringstream namestr; util::StringStream namestr;
namestr << m_description << "_"; namestr << m_description << "_";
namestr << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() ); namestr << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() );
for (size_t i = 1; i < source.GetSize(); ++i) { for (size_t i = 1; i < source.GetSize(); ++i) {

4
moses/FF/RulePairUnlexicalizedSource.cpp
View File

@ -4,7 +4,7 @@
#include "moses/ScoreComponentCollection.h" #include "moses/ScoreComponentCollection.h"
#include "moses/FactorCollection.h" #include "moses/FactorCollection.h"
#include <sstream> #include <sstream>
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -58,7 +58,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
} }
} }
ostringstream namestr; util::StringStream namestr;
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) { for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
const Word &wordT = targetPhrase.GetWord(posT); const Word &wordT = targetPhrase.GetWord(posT);

3
moses/FF/SparseHieroReorderingFeature.cpp
View File

@ -6,6 +6,7 @@
#include "moses/Sentence.h" #include "moses/Sentence.h"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#include "SparseHieroReorderingFeature.h" #include "SparseHieroReorderingFeature.h"
@ -202,7 +203,7 @@ void SparseHieroReorderingFeature::EvaluateWhenApplied(
targetLeftRulePos < targetRightRulePos))) { targetLeftRulePos < targetRightRulePos))) {
isMonotone = false; isMonotone = false;
} }
stringstream buf; util::StringStream buf;
buf << "h_"; //sparse reordering, Huck buf << "h_"; //sparse reordering, Huck
if (m_type == SourceLeft || m_type == SourceCombined) { if (m_type == SourceLeft || m_type == SourceCombined) {
buf << GetFactor(sourceLeftBoundaryWord,m_sourceVocab,m_sourceFactor)->GetString(); buf << GetFactor(sourceLeftBoundaryWord,m_sourceVocab,m_sourceFactor)->GetString();

16
moses/FF/TargetNgramFeature.cpp
View File

@ -115,7 +115,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
// extract all ngrams from current hypothesis // extract all ngrams from current hypothesis
vector<Word> prev_words(tnState->GetWords()); vector<Word> prev_words(tnState->GetWords());
stringstream curr_ngram; util::StringStream curr_ngram;
bool skip = false; bool skip = false;
// include lower order ngrams? // include lower order ngrams?
@ -173,7 +173,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
if (cur_hypo.GetWordsBitmap().IsComplete()) { if (cur_hypo.GetWordsBitmap().IsComplete()) {
for (size_t n = m_n; n >= smallest_n; --n) { for (size_t n = m_n; n >= smallest_n; --n) {
stringstream last_ngram; util::StringStream last_ngram;
skip = false; skip = false;
for (size_t i = cur_hypo.GetSize() - n + 1; i < cur_hypo.GetSize() && !skip; ++i) for (size_t i = cur_hypo.GetSize() - n + 1; i < cur_hypo.GetSize() && !skip; ++i)
appendNgram(cur_hypo.GetWord(i), skip, last_ngram); appendNgram(cur_hypo.GetWord(i), skip, last_ngram);
@ -203,7 +203,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
return new TargetNgramState(new_prev_words); return new TargetNgramState(new_prev_words);
} }
void TargetNgramFeature::appendNgram(const Word& word, bool& skip, stringstream &ngram) const void TargetNgramFeature::appendNgram(const Word& word, bool& skip, util::StringStream &ngram) const
{ {
// const string& w = word.GetFactor(m_factorType)->GetString(); // const string& w = word.GetFactor(m_factorType)->GetString();
const StringPiece w = word.GetString(m_factorType); const StringPiece w = word.GetString(m_factorType);
@ -256,7 +256,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
suffixTerminals++; suffixTerminals++;
// everything else // everything else
else { else {
stringstream ngram; util::StringStream ngram;
ngram << m_baseName; ngram << m_baseName;
if (m_factorType == 0) if (m_factorType == 0)
ngram << factorZero; ngram << factorZero;
@ -367,7 +367,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
suffixTerminals = 0; suffixTerminals = 0;
// remove duplicates // remove duplicates
stringstream curr_ngram; util::StringStream curr_ngram;
curr_ngram << m_baseName; curr_ngram << m_baseName;
curr_ngram << (*contextFactor[m_n-2]).GetString(m_factorType); curr_ngram << (*contextFactor[m_n-2]).GetString(m_factorType);
curr_ngram << ":"; curr_ngram << ":";
@ -393,7 +393,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
// remove duplicates // remove duplicates
size_t size = contextFactor.size(); size_t size = contextFactor.size();
if (makePrefix && makeSuffix && (size <= m_n)) { if (makePrefix && makeSuffix && (size <= m_n)) {
stringstream curr_ngram; util::StringStream curr_ngram;
curr_ngram << m_baseName; curr_ngram << m_baseName;
for (size_t i = 0; i < size; ++i) { for (size_t i = 0; i < size; ++i) {
curr_ngram << (*contextFactor[i]).GetString(m_factorType); curr_ngram << (*contextFactor[i]).GetString(m_factorType);
@ -411,7 +411,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos, size_t offset) const void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos, size_t offset) const
{ {
stringstream ngram; util::StringStream ngram;
size_t size = contextFactor.size(); size_t size = contextFactor.size();
for (size_t k = 0; k < numberOfStartPos; ++k) { for (size_t k = 0; k < numberOfStartPos; ++k) {
size_t max_end = (size < m_n+k+offset)? size: m_n+k+offset; size_t max_end = (size < m_n+k+offset)? size: m_n+k+offset;
@ -436,7 +436,7 @@ void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFacto
void TargetNgramFeature::MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfEndPos, size_t offset) const void TargetNgramFeature::MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfEndPos, size_t offset) const
{ {
stringstream ngram; util::StringStream ngram;
for (size_t k = 0; k < numberOfEndPos; ++k) { for (size_t k = 0; k < numberOfEndPos; ++k) {
size_t end_pos = contextFactor.size()-1-k-offset; size_t end_pos = contextFactor.size()-1-k-offset;
for (int start_pos=end_pos-1; (start_pos >= 0) && (end_pos-start_pos < m_n); --start_pos) { for (int start_pos=end_pos-1; (start_pos >= 0) && (end_pos-start_pos < m_n); --start_pos) {

3
moses/FF/TargetNgramFeature.h
View File

@ -12,6 +12,7 @@
#include "moses/LM/SingleFactor.h" #include "moses/LM/SingleFactor.h"
#include "moses/ChartHypothesis.h" #include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h" #include "moses/ChartManager.h"
#include "util/string_stream.hh"
namespace Moses namespace Moses
{ {
@ -244,7 +245,7 @@ private:
std::string m_baseName; std::string m_baseName;
void appendNgram(const Word& word, bool& skip, std::stringstream& ngram) const; void appendNgram(const Word& word, bool& skip, util::StringStream& ngram) const;
void MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, void MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator,
size_t numberOfStartPos = 1, size_t offset = 0) const; size_t numberOfStartPos = 1, size_t offset = 0) const;
void MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, void MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator,

16
moses/FF/WordTranslationFeature.cpp
View File

@ -179,7 +179,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
if (m_simple) { if (m_simple) {
// construct feature name // construct feature name
stringstream featureName; util::StringStream featureName;
featureName << m_description << "_"; featureName << m_description << "_";
featureName << sourceWord; featureName << sourceWord;
featureName << "~"; featureName << "~";
@ -193,7 +193,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
if(use_topicid) { if(use_topicid) {
// use topicid as trigger // use topicid as trigger
const long topicid = sentence.GetTopicId(); const long topicid = sentence.GetTopicId();
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
if (topicid == -1) if (topicid == -1)
feature << "unk"; feature << "unk";
@ -209,7 +209,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
// use topic probabilities // use topic probabilities
const vector<string> &topicid_prob = *(input.GetTopicIdAndProb()); const vector<string> &topicid_prob = *(input.GetTopicIdAndProb());
if (atol(topicid_prob[0].c_str()) == -1) { if (atol(topicid_prob[0].c_str()) == -1) {
stringstream feature; util::StringStream feature;
feature << m_description << "_unk_"; feature << m_description << "_unk_";
feature << sourceWord; feature << sourceWord;
feature << "~"; feature << "~";
@ -217,7 +217,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
scoreBreakdown.SparsePlusEquals(feature.str(), 1); scoreBreakdown.SparsePlusEquals(feature.str(), 1);
} else { } else {
for (size_t i=0; i+1 < topicid_prob.size(); i+=2) { for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
feature << topicid_prob[i]; feature << topicid_prob[i];
feature << "_"; feature << "_";
@ -233,7 +233,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
const long docid = input.GetDocumentId(); const long docid = input.GetDocumentId();
for (boost::unordered_set<std::string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) { for (boost::unordered_set<std::string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
string sourceTrigger = *p; string sourceTrigger = *p;
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
feature << sourceTrigger; feature << sourceTrigger;
feature << "_"; feature << "_";
@ -248,7 +248,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
size_t globalSourceIndex = inputPath.GetWordsRange().GetStartPos() + sourceIndex; size_t globalSourceIndex = inputPath.GetWordsRange().GetStartPos() + sourceIndex;
if (!m_domainTrigger && globalSourceIndex == 0) { if (!m_domainTrigger && globalSourceIndex == 0) {
// add <s> trigger feature for source // add <s> trigger feature for source
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
feature << "<s>,"; feature << "<s>,";
feature << sourceWord; feature << sourceWord;
@ -278,7 +278,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
if (m_domainTrigger) { if (m_domainTrigger) {
if (sourceTriggerExists) { if (sourceTriggerExists) {
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
feature << sourceTrigger; feature << sourceTrigger;
feature << "_"; feature << "_";
@ -288,7 +288,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
scoreBreakdown.SparsePlusEquals(feature.str(), 1); scoreBreakdown.SparsePlusEquals(feature.str(), 1);
} }
} else if (m_unrestricted || sourceTriggerExists) { } else if (m_unrestricted || sourceTriggerExists) {
stringstream feature; util::StringStream feature;
feature << m_description << "_"; feature << m_description << "_";
if (contextIndex < globalSourceIndex) { if (contextIndex < globalSourceIndex) {
feature << sourceTrigger; feature << sourceTrigger;

3
moses/FeatureVector.cpp
View File

@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FeatureVector.h" #include "FeatureVector.h"
#include "util/string_piece_hash.hh" #include "util/string_piece_hash.hh"
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -204,7 +205,7 @@ void FVector::save(const string& filename) const
{ {
ofstream out(filename.c_str()); ofstream out(filename.c_str());
if (!out) { if (!out) {
ostringstream msg; util::StringStream msg;
msg << "Unable to open " << filename; msg << "Unable to open " << filename;
throw runtime_error(msg.str()); throw runtime_error(msg.str());
} }

3
moses/File.h
View File

@ -12,6 +12,7 @@
#include <sstream> #include <sstream>
#include <vector> #include <vector>
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#include "TypeDef.h" #include "TypeDef.h"
#include "Util.h" #include "Util.h"
@ -147,7 +148,7 @@ inline OFF_T fTell(FILE* f)
inline void fSeek(FILE* f,OFF_T o) inline void fSeek(FILE* f,OFF_T o)
{ {
if(FSEEKO(f,o,SEEK_SET)<0) { if(FSEEKO(f,o,SEEK_SET)<0) {
std::stringstream strme; util::StringStream strme;
strme << "ERROR: could not fseeko position " << o <<"\n"; strme << "ERROR: could not fseeko position " << o <<"\n";
if(o==InvalidOffT) strme << "You tried to seek for 'InvalidOffT'!\n"; if(o==InvalidOffT) strme << "You tried to seek for 'InvalidOffT'!\n";
UTIL_THROW2(strme.str()); UTIL_THROW2(strme.str());

5
moses/GenerationDictionary.cpp
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "InputFileStream.h" #include "InputFileStream.h"
#include "StaticData.h" #include "StaticData.h"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -84,9 +85,9 @@ void GenerationDictionary::Load()
size_t numFeaturesInFile = token.size() - 2; size_t numFeaturesInFile = token.size() - 2;
if (numFeaturesInFile < numFeatureValuesInConfig) { if (numFeaturesInFile < numFeatureValuesInConfig) {
stringstream strme; util::StringStream strme;
strme << m_filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig strme << m_filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig
<< " feature values, but found " << numFeaturesInFile << std::endl; << " feature values, but found " << numFeaturesInFile << "\n";
throw strme.str(); throw strme.str();
} }
std::vector<float> scores(numFeatureValuesInConfig, 0.0f); std::vector<float> scores(numFeatureValuesInConfig, 0.0f);

2
moses/Hypothesis.h
View File

@ -209,7 +209,7 @@ public:
if (m_prevHypo != NULL) { if (m_prevHypo != NULL) {
m_prevHypo->ToStream(out); m_prevHypo->ToStream(out);
} }
out << (Phrase) GetCurrTargetPhrase(); out << (const Phrase&) GetCurrTargetPhrase();
} }
std::string GetOutputString() const { std::string GetOutputString() const {

3
moses/LM/Ken.cpp
View File

@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "lm/model.hh" #include "lm/model.hh"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/tokenize_piece.hh" #include "util/tokenize_piece.hh"
#include "util/string_stream.hh"
#include "Ken.h" #include "Ken.h"
#include "Base.h" #include "Base.h"
@ -474,7 +475,7 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig)
util::TokenIter<util::SingleCharacter, true> argument(lineOrig, ' '); util::TokenIter<util::SingleCharacter, true> argument(lineOrig, ' ');
++argument; // KENLM ++argument; // KENLM
stringstream line; util::StringStream line;
line << "KENLM"; line << "KENLM";
for (; argument; ++argument) { for (; argument; ++argument) {

8
moses/LM/Remote.cpp
View File

@ -5,6 +5,7 @@
#include <sys/types.h> #include <sys/types.h>
#include "Remote.h" #include "Remote.h"
#include "moses/Factor.h" #include "moses/Factor.h"
#include "util/string_stream.hh"
#if !defined(_WIN32) && !defined(_WIN64) #if !defined(_WIN32) && !defined(_WIN64)
#include <arpa/inet.h> #include <arpa/inet.h>
@ -96,7 +97,7 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
cur->boState = *reinterpret_cast<const State*>(&m_curId); cur->boState = *reinterpret_cast<const State*>(&m_curId);
++m_curId; ++m_curId;
std::ostringstream os; util::StringStream os;
os << "prob "; os << "prob ";
if (event_word == NULL) { if (event_word == NULL) {
os << "</s>"; os << "</s>";
@ -111,9 +112,8 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
os << ' ' << f->GetString(); os << ' ' << f->GetString();
} }
} }
os << std::endl; os << "\n";
std::string out = os.str(); write(sock, os.str().c_str(), os.str().size());
write(sock, out.c_str(), out.size());
char res[6]; char res[6];
int r = read(sock, res, 6); int r = read(sock, res, 6);
int errors = 0; int errors = 0;

3
moses/Manager.cpp
View File

@ -56,6 +56,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "util/exception.hh" #include "util/exception.hh"
#include "util/random.hh" #include "util/random.hh"
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -1971,7 +1972,7 @@ void Manager::OutputSearchGraphSLF() const
// Output search graph in HTK standard lattice format (SLF) // Output search graph in HTK standard lattice format (SLF)
bool slf = staticData.GetOutputSearchGraphSLF(); bool slf = staticData.GetOutputSearchGraphSLF();
if (slf) { if (slf) {
stringstream fileName; util::StringStream fileName;
string dir; string dir;
staticData.GetParameter().SetParameter<string>(dir, "output-search-graph-slf", ""); staticData.GetParameter().SetParameter<string>(dir, "output-search-graph-slf", "");

7
moses/Parameter.cpp
View File

@ -30,6 +30,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Util.h" #include "Util.h"
#include "InputFileStream.h" #include "InputFileStream.h"
#include "StaticData.h" #include "StaticData.h"
#include "util/string_stream.hh"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/random.hh" #include "util/random.hh"
#include <boost/program_options.hpp> #include <boost/program_options.hpp>
@ -701,7 +702,7 @@ ConvertWeightArgsPhraseModel(const string &oldWeightName)
size_t currOldInd = 0; size_t currOldInd = 0;
for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) { for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
stringstream ptLine; util::StringStream ptLine;
vector<string> token = Tokenize(translationVector[currDict]); vector<string> token = Tokenize(translationVector[currDict]);
@ -860,7 +861,7 @@ ConvertWeightArgsDistortion()
} }
SetWeight("LexicalReordering", indTable, weights); SetWeight("LexicalReordering", indTable, weights);
stringstream strme; util::StringStream strme;
strme << "LexicalReordering " strme << "LexicalReordering "
<< "type=" << toks[1] << " "; << "type=" << toks[1] << " ";
@ -1007,7 +1008,7 @@ ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string
} }
SetWeight(newWeightName, indTable, weights); SetWeight(newWeightName, indTable, weights);
stringstream strme; util::StringStream strme;
strme << "Generation " strme << "Generation "
<< "input-factor=" << modelToks[0] << " " << "input-factor=" << modelToks[0] << " "
<< "output-factor=" << modelToks[1] << " " << "output-factor=" << modelToks[1] << " "

3
moses/Phrase.cpp
View File

@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "StaticData.h" // GetMaxNumFactors #include "StaticData.h" // GetMaxNumFactors
#include "util/string_piece.hh" #include "util/string_piece.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh" #include "util/tokenize_piece.hh"
using namespace std; using namespace std;
@ -117,7 +118,7 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const
{ {
bool markUnknown = StaticData::Instance().GetMarkUnknown(); bool markUnknown = StaticData::Instance().GetMarkUnknown();
stringstream strme; util::StringStream strme;
for (size_t pos = 0 ; pos < GetSize() ; pos++) { for (size_t pos = 0 ; pos < GetSize() ; pos++) {
if (markUnknown && GetWord(pos).IsOOV()) { if (markUnknown && GetWord(pos).IsOOV()) {
strme << StaticData::Instance().GetUnknownWordPrefix(); strme << StaticData::Instance().GetUnknownWordPrefix();

13
moses/ScoreComponentCollection.cpp
View File

@ -4,6 +4,7 @@
#include <boost/foreach.hpp> #include <boost/foreach.hpp>
#include <boost/format.hpp> #include <boost/format.hpp>
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#include "ScoreComponentCollection.h" #include "ScoreComponentCollection.h"
#include "StaticData.h" #include "StaticData.h"
#include "moses/FF/StatelessFeatureFunction.h" #include "moses/FF/StatelessFeatureFunction.h"
@ -88,9 +89,8 @@ void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float s
{ {
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP; std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) { for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
std::stringstream name; const std::string &name = i->first.name();
name << i->first; if (starts_with(name, prefix))
if (starts_with(name.str(), prefix))
m_scores[i->first] = i->second * scalar; m_scores[i->first] = i->second * scalar;
} }
} }
@ -101,9 +101,8 @@ size_t ScoreComponentCollection::GetNumberWeights(const FeatureFunction* sp)
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP; std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
size_t weights = 0; size_t weights = 0;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) { for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
std::stringstream name; const std::string &name = i->first.name();
name << i->first; if (starts_with(name, prefix))
if (starts_with(name.str(), prefix))
weights++; weights++;
} }
return weights; return weights;
@ -215,7 +214,7 @@ void ScoreComponentCollection::Save(const string& filename) const
{ {
ofstream out(filename.c_str()); ofstream out(filename.c_str());
if (!out) { if (!out) {
ostringstream msg; util::StringStream msg;
msg << "Unable to open " << filename; msg << "Unable to open " << filename;
throw runtime_error(msg.str()); throw runtime_error(msg.str());
} }

4
moses/StaticData.cpp
View File

@ -640,7 +640,7 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const
switch (decodeType) { switch (decodeType) {
case Translate: case Translate:
if(index>=pts.size()) { if(index>=pts.size()) {
stringstream strme; util::StringStream strme;
strme << "No phrase dictionary with index " strme << "No phrase dictionary with index "
<< index << " available!"; << index << " available!";
UTIL_THROW(util::Exception, strme.str()); UTIL_THROW(util::Exception, strme.str());
@ -649,7 +649,7 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const
break; break;
case Generate: case Generate:
if(index>=gens.size()) { if(index>=gens.size()) {
stringstream strme; util::StringStream strme;
strme << "No generation dictionary with index " strme << "No generation dictionary with index "
<< index << " available!"; << index << " available!";
UTIL_THROW(util::Exception, strme.str()); UTIL_THROW(util::Exception, strme.str());

3
moses/Syntax/F2S/GlueRuleSynthesizer.cpp
View File

@ -4,6 +4,7 @@
#include "moses/FF/UnknownWordPenaltyProducer.h" #include "moses/FF/UnknownWordPenaltyProducer.h"
#include "moses/StaticData.h" #include "moses/StaticData.h"
#include "util/string_stream.hh"
namespace Moses namespace Moses
{ {
@ -55,7 +56,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
TargetPhrase *targetPhrase = new TargetPhrase(); TargetPhrase *targetPhrase = new TargetPhrase();
std::ostringstream alignmentSS; util::StringStream alignmentSS;
for (std::size_t i = 0; i < e.tail.size(); ++i) { for (std::size_t i = 0; i < e.tail.size(); ++i) {
const Word &symbol = e.tail[i]->pvertex.symbol; const Word &symbol = e.tail[i]->pvertex.symbol;
if (symbol.IsNonTerminal()) { if (symbol.IsNonTerminal()) {

2
moses/Syntax/T2S/GlueRuleSynthesizer.cpp
View File

@ -47,7 +47,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
TargetPhrase *targetPhrase = new TargetPhrase(); TargetPhrase *targetPhrase = new TargetPhrase();
std::ostringstream alignmentSS; util::StringStream alignmentSS;
for (std::size_t i = 0; i < node.children.size(); ++i) { for (std::size_t i = 0; i < node.children.size(); ++i) {
const Word &symbol = node.children[i]->pvertex.symbol; const Word &symbol = node.children[i]->pvertex.symbol;
if (symbol.IsNonTerminal()) { if (symbol.IsNonTerminal()) {

6
moses/TranslationAnalysis.cpp
View File

@ -10,6 +10,7 @@
#include "moses/FF/StatefulFeatureFunction.h" #include "moses/FF/StatefulFeatureFunction.h"
#include "moses/FF/StatelessFeatureFunction.h" #include "moses/FF/StatelessFeatureFunction.h"
#include "moses/LM/Base.h" #include "moses/LM/Base.h"
#include "util/string_stream.hh"
using namespace Moses; using namespace Moses;
@ -40,8 +41,9 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo)
if (doLMStats) if (doLMStats)
lmAcc.resize((*tpi)->GetLMStats()->size(), 0); lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
for (; tpi != translationPath.end(); ++tpi) { for (; tpi != translationPath.end(); ++tpi) {
std::ostringstream sms; util::StringStream sms;
std::ostringstream tms;
util::StringStream tms;
std::string target = (*tpi)->GetTargetPhraseStringRep(); std::string target = (*tpi)->GetTargetPhraseStringRep();
std::string source = (*tpi)->GetSourcePhraseStringRep(); std::string source = (*tpi)->GetSourcePhraseStringRep();
WordsRange twr = (*tpi)->GetCurrTargetWordsRange(); WordsRange twr = (*tpi)->GetCurrTargetWordsRange();

9
moses/TranslationModel/CompactPT/BlockHashIndex.cpp
View File

@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "BlockHashIndex.h" #include "BlockHashIndex.h"
#include "CmphStringVectorAdapter.h" #include "CmphStringVectorAdapter.h"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#ifdef HAVE_CMPH #ifdef HAVE_CMPH
#include "cmph.h" #include "cmph.h"
@ -367,10 +368,10 @@ void BlockHashIndex::CalcHash(size_t current, void* source_void)
if(lastKey > temp) { if(lastKey > temp) {
if(source->nkeys != 2 || temp != "###DUMMY_KEY###") { if(source->nkeys != 2 || temp != "###DUMMY_KEY###") {
std::stringstream strme; util::StringStream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl; strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
strme << "1: " << lastKey << std::endl; strme << "1: " << lastKey << "\n";
strme << "2: " << temp << std::endl; strme << "2: " << temp << "\n";
UTIL_THROW2(strme.str()); UTIL_THROW2(strme.str());
} }
} }

9
moses/TranslationModel/CompactPT/BlockHashIndex.h
View File

@ -34,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "StringVector.h" #include "StringVector.h"
#include "PackedArray.h" #include "PackedArray.h"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#ifdef WITH_THREADS #ifdef WITH_THREADS
#include "moses/ThreadPool.h" #include "moses/ThreadPool.h"
@ -145,10 +146,10 @@ public:
size_t current = m_landmarks.size(); size_t current = m_landmarks.size();
if(m_landmarks.size() && m_landmarks.back().str() >= keys[0]) { if(m_landmarks.size() && m_landmarks.back().str() >= keys[0]) {
std::stringstream strme; util::StringStream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl; strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
strme << "1: " << m_landmarks.back().str() << std::endl; strme << "1: " << m_landmarks.back().str() << "\n";
strme << "2: " << keys[0] << std::endl; strme << "2: " << keys[0] << "\n";
UTIL_THROW2(strme.str()); UTIL_THROW2(strme.str());
} }

5
moses/TranslationModel/PhraseDictionaryMultiModel.cpp
View File

@ -17,6 +17,7 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/ ***********************************************************************/
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#include "moses/TranslationModel/PhraseDictionaryMultiModel.h" #include "moses/TranslationModel/PhraseDictionaryMultiModel.h"
@ -38,7 +39,7 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
} else if (m_mode == "all" || m_mode == "all-restrict") { } else if (m_mode == "all" || m_mode == "all-restrict") {
UTIL_THROW2("Implementation has moved: use PhraseDictionaryGroup with restrict=true/false"); UTIL_THROW2("Implementation has moved: use PhraseDictionaryGroup with restrict=true/false");
} else { } else {
ostringstream msg; util::StringStream msg;
msg << "combination mode unknown: " << m_mode; msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str()); throw runtime_error(msg.str());
} }
@ -210,7 +211,7 @@ std::vector<std::vector<float> > PhraseDictionaryMultiModel::getWeights(size_t n
raw_weights.push_back(1.0/m_numModels); //uniform weights created online raw_weights.push_back(1.0/m_numModels); //uniform weights created online
} }
} else if(weights_ptr->size() != m_numModels && weights_ptr->size() != m_numModels * numWeights) { } else if(weights_ptr->size() != m_numModels && weights_ptr->size() != m_numModels * numWeights) {
std::stringstream strme; util::StringStream strme;
strme << "Must have either one multimodel weight per model (" << m_numModels << "), or one per weighted feature and model (" << numWeights << "*" << m_numModels << "). You have " << weights_ptr->size() << "."; strme << "Must have either one multimodel weight per model (" << m_numModels << "), or one per weighted feature and model (" << numWeights << "*" << m_numModels << "). You have " << weights_ptr->size() << ".";
UTIL_THROW(util::Exception, strme.str()); UTIL_THROW(util::Exception, strme.str());
} else { } else {

3
moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp
View File

@ -18,6 +18,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/ ***********************************************************************/
#include "util/exception.hh" #include "util/exception.hh"
#include "util/tokenize.hh" #include "util/tokenize.hh"
#include "util/string_stream.hh"
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h" #include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h"
using namespace std; using namespace std;
@ -56,7 +57,7 @@ void PhraseDictionaryMultiModelCounts::SetParameter(const std::string& key, cons
else if (m_mode == "interpolate") else if (m_mode == "interpolate")
m_combineFunction = LinearInterpolationFromCounts; m_combineFunction = LinearInterpolationFromCounts;
else { else {
ostringstream msg; util::StringStream msg;
msg << "combination mode unknown: " << m_mode; msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str()); throw runtime_error(msg.str());
} }

3
moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp
View File

@ -15,6 +15,7 @@
#include "moses/PDTAimp.h" #include "moses/PDTAimp.h"
#include "moses/TranslationTask.h" #include "moses/TranslationTask.h"
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std; using namespace std;
@ -52,7 +53,7 @@ void PhraseDictionaryTreeAdaptor::InitializeForInput(ttasksptr const& ttask)
vector<float> weight = staticData.GetWeights(this); vector<float> weight = staticData.GetWeights(this);
if(m_numScoreComponents!=weight.size()) { if(m_numScoreComponents!=weight.size()) {
std::stringstream strme; util::StringStream strme;
UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size() UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size()
<< " " << m_numScoreComponents); << " " << m_numScoreComponents);
} }

4
moses/TranslationModel/RuleTable/LoaderStandard.cpp
View File

@ -126,14 +126,14 @@ void ReformatHieroRule(const string &lineOrig, string &out)
ReformatHieroRule(1, targetPhraseString, ntAlign); ReformatHieroRule(1, targetPhraseString, ntAlign);
ReformateHieroScore(scoreString); ReformateHieroScore(scoreString);
stringstream align; util::StringStream align;
map<size_t, pair<size_t, size_t> >::const_iterator iterAlign; map<size_t, pair<size_t, size_t> >::const_iterator iterAlign;
for (iterAlign = ntAlign.begin(); iterAlign != ntAlign.end(); ++iterAlign) { for (iterAlign = ntAlign.begin(); iterAlign != ntAlign.end(); ++iterAlign) {
const pair<size_t, size_t> &alignPoint = iterAlign->second; const pair<size_t, size_t> &alignPoint = iterAlign->second;
align << alignPoint.first << "-" << alignPoint.second << " "; align << alignPoint.first << "-" << alignPoint.second << " ";
} }
stringstream ret; util::StringStream ret;
ret << sourcePhraseString << " ||| " ret << sourcePhraseString << " ||| "
<< targetPhraseString << " ||| " << targetPhraseString << " ||| "
<< scoreString << " ||| " << scoreString << " ||| "

3
moses/TranslationModel/fuzzy-match/SentenceAlignment.cpp
View File

@ -7,13 +7,14 @@
// //
#include <iostream> #include <iostream>
#include "util/string_stream.hh"
#include "SentenceAlignment.h" #include "SentenceAlignment.h"
namespace tmmt namespace tmmt
{ {
std::string SentenceAlignment::getTargetString(const Vocabulary &vocab) const std::string SentenceAlignment::getTargetString(const Vocabulary &vocab) const
{ {
std::stringstream strme; util::StringStream strme;
for (size_t i = 0; i < target.size(); ++i) { for (size_t i = 0; i < target.size(); ++i) {
const WORD &word = vocab.GetWord(target[i]); const WORD &word = vocab.GetWord(target[i]);
strme << word << " "; strme << word << " ";

3
moses/TranslationModel/fuzzy-match/SentenceAlignment.h
View File

@ -12,6 +12,7 @@
#include <sstream> #include <sstream>
#include <vector> #include <vector>
#include "Vocabulary.h" #include "Vocabulary.h"
#include "util/string_stream.hh"
namespace tmmt namespace tmmt
{ {
@ -27,7 +28,7 @@ struct SentenceAlignment {
std::string getTargetString(const Vocabulary &vocab) const; std::string getTargetString(const Vocabulary &vocab) const;
std::string getAlignmentString() const { std::string getAlignmentString() const {
std::stringstream strme; util::StringStream strme;
for (size_t i = 0; i < alignment.size(); ++i) { for (size_t i = 0; i < alignment.size(); ++i) {
const std::pair<int,int> &alignPair = alignment[i]; const std::pair<int,int> &alignPair = alignment[i];
strme << alignPair.first << "-" << alignPair.second << " "; strme << alignPair.first << "-" << alignPair.second << " ";

5
moses/Util.h
View File

@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#include "TypeDef.h" #include "TypeDef.h"
namespace Moses namespace Moses
@ -343,7 +344,7 @@ inline std::vector<std::string> TokenizeFirstOnly(const std::string& str,
template <typename T> template <typename T>
std::string Join(const std::string& delimiter, const std::vector<T>& items) std::string Join(const std::string& delimiter, const std::vector<T>& items)
{ {
std::ostringstream outstr; util::StringStream outstr;
if(items.size() == 0) return ""; if(items.size() == 0) return "";
outstr << items[0]; outstr << items[0];
for(unsigned int i = 1; i < items.size(); i++) for(unsigned int i = 1; i < items.size(); i++)
@ -357,7 +358,7 @@ std::string Join(const std::string& delimiter, const std::vector<T>& items)
template<typename It> template<typename It>
std::string Join(const std::string &delim, It begin, It end) std::string Join(const std::string &delim, It begin, It end)
{ {
std::ostringstream outstr; util::StringStream outstr;
if (begin != end) if (begin != end)
outstr << *begin++; outstr << *begin++;
for ( ; begin != end; ++begin) for ( ; begin != end; ++begin)

7
moses/Word.cpp
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FactorCollection.h" #include "FactorCollection.h"
#include "StaticData.h" // needed to determine the FactorDelimiter #include "StaticData.h" // needed to determine the FactorDelimiter
#include "util/exception.hh" #include "util/exception.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh" #include "util/tokenize_piece.hh"
using namespace std; using namespace std;
@ -94,7 +95,7 @@ void Word::Merge(const Word &sourceWord)
std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlank) const std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlank) const
{ {
stringstream strme; util::StringStream strme;
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
bool firstPass = true; bool firstPass = true;
unsigned int stop = min(max_fax(),factorType.size()); unsigned int stop = min(max_fax(),factorType.size());
@ -210,7 +211,7 @@ TO_STRING_BODY(Word);
// friend // friend
ostream& operator<<(ostream& out, const Word& word) ostream& operator<<(ostream& out, const Word& word)
{ {
stringstream strme; util::StringStream strme;
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
bool firstPass = true; bool firstPass = true;
unsigned int stop = max_fax(); unsigned int stop = max_fax();
@ -223,7 +224,7 @@ ostream& operator<<(ostream& out, const Word& word)
} else { } else {
strme << factorDelimiter; strme << factorDelimiter;
} }
strme << *factor; strme << factor->GetString();
} }
} }
out << strme.str() << " "; out << strme.str() << " ";

3
util/exception.cc
View File

@ -25,7 +25,8 @@ void Exception::SetLocation(const char *file, unsigned int line, const char *fun
*/ */
std::string old_text; std::string old_text;
std::swap(old_text, what_); std::swap(old_text, what_);
StringStream stream(what_); StringStream stream;
stream << what_;
stream << file << ':' << line; stream << file << ':' << line;
if (func) stream << " in " << func << " threw "; if (func) stream << " in " << func << " threw ";
if (child_name) { if (child_name) {

17
util/string_stream.hh
View File

@ -11,9 +11,13 @@ namespace util {
class StringStream : public FakeOStream<StringStream> { class StringStream : public FakeOStream<StringStream> {
public: public:
// Semantics: appends to string. Remember to clear first! // Semantics: appends to string. Remember to clear first!
explicit StringStream(std::string &out)
: out_(out) {}
explicit StringStream()
{}
/*
explicit StringStream(std::string &out)
: out_(out) {}
*/
StringStream &flush() { return *this; } StringStream &flush() { return *this; }
StringStream &write(const void *data, std::size_t length) { StringStream &write(const void *data, std::size_t length) {
@ -21,6 +25,13 @@ class StringStream : public FakeOStream<StringStream> {
return *this; return *this;
} }
const std::string &str() const
{ return out_; }
void str(const std::string &val)
{
out_ = val;
}
protected: protected:
friend class FakeOStream<StringStream>; friend class FakeOStream<StringStream>;
char *Ensure(std::size_t amount) { char *Ensure(std::size_t amount) {
@ -36,7 +47,7 @@ class StringStream : public FakeOStream<StringStream> {
} }
private: private:
std::string &out_; std::string out_;
}; };
} // namespace } // namespace

18
util/string_stream_test.cc
View File

@ -11,9 +11,9 @@
namespace util { namespace { namespace util { namespace {
template <class T> void TestEqual(const T value) { template <class T> void TestEqual(const T value) {
std::string str; StringStream strme;
StringStream(str) << value; strme << value;
BOOST_CHECK_EQUAL(boost::lexical_cast<std::string>(value), str); BOOST_CHECK_EQUAL(boost::lexical_cast<std::string>(value), strme.str());
} }
template <class T> void TestCorners() { template <class T> void TestCorners() {
@ -66,15 +66,15 @@ BOOST_AUTO_TEST_CASE(Strings) {
non_const[0] = 'b'; non_const[0] = 'b';
non_const[1] = 'c'; non_const[1] = 'c';
non_const[2] = 0; non_const[2] = 0;
std::string out;
StringStream(out) << "a" << non_const << 'c'; StringStream out;
BOOST_CHECK_EQUAL("abcc", out); out << "a" << non_const << 'c';
BOOST_CHECK_EQUAL("abcc", out.str());
// Now test as a separate object. // Now test as a separate object.
out.clear(); StringStream stream;
StringStream stream(out);
stream << "a" << non_const << 'c' << piece; stream << "a" << non_const << 'c' << piece;
BOOST_CHECK_EQUAL("abccabcdef", out); BOOST_CHECK_EQUAL("abccabcdef", stream.str());
} }
}} // namespaces }} // namespaces