moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-06 11:38:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'perf_baseline' of github.com:hieuhoang/mosesdecoder into perf_baseline

Browse Source
This commit is contained in:
Hieu Hoang 2015-10-17 22:25:13 +01:00
commit 1f6a787c2a
43 changed files with 148 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
OnDiskPt/OnDiskWrapper.cpp
View File

@ -25,6 +25,7 @@
#include "OnDiskWrapper.h"
#include "moses/Factor.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std;
@ -223,7 +224,8 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact
{
bool isNonTerminal = origWord.IsNonTerminal();
Word *newWord = new Word(isNonTerminal);
stringstream strme;
util::StringStream strme;
size_t factorType = factorsVec[0];
const Moses::Factor *factor = origWord.GetFactor(factorType);

5
lm/filter/arpa_io.cc
View File

@ -33,10 +33,9 @@ template <class Stream> void WriteCounts(Stream &out, const std::vector<uint64_t
}
size_t SizeNeededForCounts(const std::vector<uint64_t> &number) {
std::string buf;
util::StringStream stream(buf);
util::StringStream stream;
WriteCounts(stream, number);
return buf.size();
return stream.str().size();
}
bool IsEntirelyWhiteSpace(const StringPiece &line) {

1
moses-cmd/Main.cpp
View File

@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
* Moses main wrapper for executable for single-threaded and multi-threaded, simply calling decoder_main.
**/
#include "moses/ExportInterface.h"
#include "util/string_stream.hh"
/** main function of the command line version of the decoder **/
int main(int argc, char** argv)

14
moses/ChartHypothesisCollection.cpp
View File

@ -167,20 +167,6 @@ void ChartHypothesisCollection::Detach(const HCType::iterator &iter)
void ChartHypothesisCollection::Remove(const HCType::iterator &iter)
{
ChartHypothesis *h = *iter;
/*
stringstream strme("");
strme << h->GetOutputPhrase();
string toFind = "the goal of gene scientists is ";
size_t pos = toFind.find(strme.str());
if (pos == 0)
{
cerr << pos << " " << strme.str() << *h << endl;
cerr << *this << endl;
}
*/
Detach(iter);
ChartHypothesis::Delete(h);
}

6
moses/FF/FeatureFunction.cpp
View File

@ -112,12 +112,10 @@ void FeatureFunction::ParseLine(const std::string &line)
if (m_description == "") {
size_t index = description_counts.count(nameStub);
ostringstream dstream;
dstream << nameStub;
dstream << index;
string descr = SPrint(nameStub) + SPrint(index);
description_counts.insert(nameStub);
m_description = dstream.str();
m_description = descr;
}
}

11
moses/FF/GlobalLexicalModelUnlimited.cpp
View File

@ -5,6 +5,7 @@
#include "moses/Hypothesis.h"
#include "moses/TranslationTask.h"
#include "util/string_piece_hash.hh"
#include "util/string_stream.hh"
using namespace std;
@ -131,7 +132,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
}
if (m_biasFeature) {
stringstream feature;
util::StringStream feature;
feature << "glm_";
feature << targetString;
feature << "~";
@ -165,7 +166,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
if (m_sourceContext) {
if (sourceIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
util::StringStream feature;
feature << "glm_";
feature << targetString;
feature << "~";
@ -183,7 +184,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
contextExists = FindStringPiece(m_vocabSource, contextString ) != m_vocabSource.end();
if (m_unrestricted || contextExists) {
stringstream feature;
util::StringStream feature;
feature << "glm_";
feature << targetString;
feature << "~";
@ -304,7 +305,7 @@ void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo
}
}
} else {
stringstream feature;
util::StringStream feature;
feature << "glm_";
feature << targetString;
feature << "~";
@ -323,7 +324,7 @@ void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulat
StringPiece sourceTrigger, StringPiece sourceWord,
StringPiece targetTrigger, StringPiece targetWord) const
{
stringstream feature;
util::StringStream feature;
feature << "glm_";
feature << targetTrigger;
feature << ",";

5
moses/FF/LexicalReordering/SparseReordering.cpp
View File

@ -8,6 +8,7 @@
#include "util/file_piece.hh"
#include "util/string_piece.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh"
#include "LexicalReordering.h"
@ -26,7 +27,7 @@ const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
{
static string kSep = "-";
static string name;
ostringstream buf;
util::StringStream buf;
// type side position id word reotype
if (type == Phrase) {
buf << "phr";
@ -88,7 +89,7 @@ SparseReordering::SparseReordering(const map<string,string>& config, const Lexic
ReadWeightMap(i->second);
m_useWeightMap = true;
for (int reoType=0; reoType<=LRModel::MAX; ++reoType) {
ostringstream buf;
util::StringStream buf;
buf << reoType;
m_featureMap2.push_back(m_producer->GetFeatureName(buf.str()));
}

6
moses/FF/OSM-Feature/osmHyp.cpp
View File

@ -165,11 +165,7 @@ int osmHypothesis :: firstOpenGap(vector <int> & coverageVector)
string osmHypothesis :: intToString(int num)
{
std::ostringstream stm;
stm<<num;
return stm.str();
return SPrint(num);
}

3
moses/FF/PhraseBoundaryFeature.cpp
View File

@ -3,6 +3,7 @@
#include "moses/Hypothesis.h"
#include "moses/TranslationOption.h"
#include "moses/InputPath.h"
#include "util/string_stream.hh"
using namespace std;
@ -53,7 +54,7 @@ void PhraseBoundaryFeature::AddFeatures(
ScoreComponentCollection* scores) const
{
for (size_t i = 0; i < factors.size(); ++i) {
ostringstream name;
util::StringStream name;
name << side << ":";
name << factors[i];
name << ":";

7
moses/FF/PhraseLengthFeature.cpp
View File

@ -3,6 +3,7 @@
#include "moses/Hypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h"
#include "util/string_stream.hh"
namespace Moses
{
@ -25,13 +26,13 @@ void PhraseLengthFeature::EvaluateInIsolation(const Phrase &source
size_t sourceLength = source.GetSize();
// create feature names
stringstream nameSource;
util::StringStream nameSource;
nameSource << "s" << sourceLength;
stringstream nameTarget;
util::StringStream nameTarget;
nameTarget << "t" << targetLength;
stringstream nameBoth;
util::StringStream nameBoth;
nameBoth << sourceLength << "," << targetLength;
// increase feature counts

17
moses/FF/PhrasePairFeature.cpp
View File

@ -7,6 +7,7 @@
#include "moses/TranslationOption.h"
#include "moses/InputPath.h"
#include "util/string_piece_hash.hh"
#include "util/string_stream.hh"
#include "util/exception.hh"
using namespace std;
@ -126,7 +127,8 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
const bool use_topicid_prob = isnt.GetUseTopicIdAndProb();
// compute pair
ostringstream pair;
util::StringStream pair;
pair << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() );
for (size_t i = 1; i < source.GetSize(); ++i) {
const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId);
@ -145,7 +147,8 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
if(use_topicid) {
// use topicid as trigger
const long topicid = isnt.GetTopicId();
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
if (topicid == -1)
feature << "unk";
@ -159,13 +162,13 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
// use topic probabilities
const vector<string> &topicid_prob = *(isnt.GetTopicIdAndProb());
if (atol(topicid_prob[0].c_str()) == -1) {
stringstream feature;
util::StringStream feature;
feature << m_description << "_unk_";
feature << pair.str();
scoreBreakdown.SparsePlusEquals(feature.str(), 1);
} else {
for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
feature << topicid_prob[i];
feature << "_";
@ -179,7 +182,7 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
const long docid = isnt.GetDocumentId();
for (set<string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
string sourceTrigger = *p;
ostringstream namestr;
util::StringStream namestr;
namestr << m_description << "_";
namestr << sourceTrigger;
namestr << "_";
@ -207,7 +210,7 @@ void PhrasePairFeature::EvaluateWithSourceContext(const InputType &input
sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
if (m_unrestricted || sourceTriggerExists) {
ostringstream namestr;
util::StringStream namestr;
namestr << m_description << "_";
namestr << sourceTrigger;
namestr << "~";
@ -237,7 +240,7 @@ void PhrasePairFeature::EvaluateInIsolation(const Phrase &source
, ScoreComponentCollection &estimatedFutureScore) const
{
if (m_simple) {
ostringstream namestr;
util::StringStream namestr;
namestr << m_description << "_";
namestr << ReplaceTilde( source.GetWord(0).GetFactor(m_sourceFactorId)->GetString() );
for (size_t i = 1; i < source.GetSize(); ++i) {

4
moses/FF/RulePairUnlexicalizedSource.cpp
View File

@ -4,7 +4,7 @@
#include "moses/ScoreComponentCollection.h"
#include "moses/FactorCollection.h"
#include <sstream>
#include "util/string_stream.hh"
using namespace std;
@ -58,7 +58,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
}
}
ostringstream namestr;
util::StringStream namestr;
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
const Word &wordT = targetPhrase.GetWord(posT);

3
moses/FF/SparseHieroReorderingFeature.cpp
View File

@ -6,6 +6,7 @@
#include "moses/Sentence.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "SparseHieroReorderingFeature.h"
@ -202,7 +203,7 @@ void SparseHieroReorderingFeature::EvaluateWhenApplied(
targetLeftRulePos < targetRightRulePos))) {
isMonotone = false;
}
stringstream buf;
util::StringStream buf;
buf << "h_"; //sparse reordering, Huck
if (m_type == SourceLeft || m_type == SourceCombined) {
buf << GetFactor(sourceLeftBoundaryWord,m_sourceVocab,m_sourceFactor)->GetString();

16
moses/FF/TargetNgramFeature.cpp
View File

@ -115,7 +115,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
// extract all ngrams from current hypothesis
vector<Word> prev_words(tnState->GetWords());
stringstream curr_ngram;
util::StringStream curr_ngram;
bool skip = false;
// include lower order ngrams?
@ -173,7 +173,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
if (cur_hypo.GetWordsBitmap().IsComplete()) {
for (size_t n = m_n; n >= smallest_n; --n) {
stringstream last_ngram;
util::StringStream last_ngram;
skip = false;
for (size_t i = cur_hypo.GetSize() - n + 1; i < cur_hypo.GetSize() && !skip; ++i)
appendNgram(cur_hypo.GetWord(i), skip, last_ngram);
@ -203,7 +203,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
return new TargetNgramState(new_prev_words);
}
void TargetNgramFeature::appendNgram(const Word& word, bool& skip, stringstream &ngram) const
void TargetNgramFeature::appendNgram(const Word& word, bool& skip, util::StringStream &ngram) const
{
// const string& w = word.GetFactor(m_factorType)->GetString();
const StringPiece w = word.GetString(m_factorType);
@ -256,7 +256,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
suffixTerminals++;
// everything else
else {
stringstream ngram;
util::StringStream ngram;
ngram << m_baseName;
if (m_factorType == 0)
ngram << factorZero;
@ -367,7 +367,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
suffixTerminals = 0;
// remove duplicates
stringstream curr_ngram;
util::StringStream curr_ngram;
curr_ngram << m_baseName;
curr_ngram << (*contextFactor[m_n-2]).GetString(m_factorType);
curr_ngram << ":";
@ -393,7 +393,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
// remove duplicates
size_t size = contextFactor.size();
if (makePrefix && makeSuffix && (size <= m_n)) {
stringstream curr_ngram;
util::StringStream curr_ngram;
curr_ngram << m_baseName;
for (size_t i = 0; i < size; ++i) {
curr_ngram << (*contextFactor[i]).GetString(m_factorType);
@ -411,7 +411,7 @@ FFState* TargetNgramFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos, size_t offset) const
{
stringstream ngram;
util::StringStream ngram;
size_t size = contextFactor.size();
for (size_t k = 0; k < numberOfStartPos; ++k) {
size_t max_end = (size < m_n+k+offset)? size: m_n+k+offset;
@ -436,7 +436,7 @@ void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFacto
void TargetNgramFeature::MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfEndPos, size_t offset) const
{
stringstream ngram;
util::StringStream ngram;
for (size_t k = 0; k < numberOfEndPos; ++k) {
size_t end_pos = contextFactor.size()-1-k-offset;
for (int start_pos=end_pos-1; (start_pos >= 0) && (end_pos-start_pos < m_n); --start_pos) {

3
moses/FF/TargetNgramFeature.h
View File

@ -12,6 +12,7 @@
#include "moses/LM/SingleFactor.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"
#include "util/string_stream.hh"
namespace Moses
{
@ -244,7 +245,7 @@ private:
std::string m_baseName;
void appendNgram(const Word& word, bool& skip, std::stringstream& ngram) const;
void appendNgram(const Word& word, bool& skip, util::StringStream& ngram) const;
void MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator,
size_t numberOfStartPos = 1, size_t offset = 0) const;
void MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator,

16
moses/FF/WordTranslationFeature.cpp
View File

@ -179,7 +179,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
if (m_simple) {
// construct feature name
stringstream featureName;
util::StringStream featureName;
featureName << m_description << "_";
featureName << sourceWord;
featureName << "~";
@ -193,7 +193,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
if(use_topicid) {
// use topicid as trigger
const long topicid = sentence.GetTopicId();
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
if (topicid == -1)
feature << "unk";
@ -209,7 +209,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
// use topic probabilities
const vector<string> &topicid_prob = *(input.GetTopicIdAndProb());
if (atol(topicid_prob[0].c_str()) == -1) {
stringstream feature;
util::StringStream feature;
feature << m_description << "_unk_";
feature << sourceWord;
feature << "~";
@ -217,7 +217,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
scoreBreakdown.SparsePlusEquals(feature.str(), 1);
} else {
for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
feature << topicid_prob[i];
feature << "_";
@ -233,7 +233,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
const long docid = input.GetDocumentId();
for (boost::unordered_set<std::string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
string sourceTrigger = *p;
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
feature << sourceTrigger;
feature << "_";
@ -248,7 +248,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
size_t globalSourceIndex = inputPath.GetWordsRange().GetStartPos() + sourceIndex;
if (!m_domainTrigger && globalSourceIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
feature << "<s>,";
feature << sourceWord;
@ -278,7 +278,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
if (m_domainTrigger) {
if (sourceTriggerExists) {
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
feature << sourceTrigger;
feature << "_";
@ -288,7 +288,7 @@ void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
scoreBreakdown.SparsePlusEquals(feature.str(), 1);
}
} else if (m_unrestricted || sourceTriggerExists) {
stringstream feature;
util::StringStream feature;
feature << m_description << "_";
if (contextIndex < globalSourceIndex) {
feature << sourceTrigger;

3
moses/FeatureVector.cpp
View File

@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FeatureVector.h"
#include "util/string_piece_hash.hh"
#include "util/string_stream.hh"
using namespace std;
@ -204,7 +205,7 @@ void FVector::save(const string& filename) const
{
ofstream out(filename.c_str());
if (!out) {
ostringstream msg;
util::StringStream msg;
msg << "Unable to open " << filename;
throw runtime_error(msg.str());
}

3
moses/File.h
View File

@ -12,6 +12,7 @@
#include <sstream>
#include <vector>
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "TypeDef.h"
#include "Util.h"
@ -147,7 +148,7 @@ inline OFF_T fTell(FILE* f)
inline void fSeek(FILE* f,OFF_T o)
{
if(FSEEKO(f,o,SEEK_SET)<0) {
std::stringstream strme;
util::StringStream strme;
strme << "ERROR: could not fseeko position " << o <<"\n";
if(o==InvalidOffT) strme << "You tried to seek for 'InvalidOffT'!\n";
UTIL_THROW2(strme.str());

5
moses/GenerationDictionary.cpp
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "InputFileStream.h"
#include "StaticData.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std;
@ -84,9 +85,9 @@ void GenerationDictionary::Load()
size_t numFeaturesInFile = token.size() - 2;
if (numFeaturesInFile < numFeatureValuesInConfig) {
stringstream strme;
util::StringStream strme;
strme << m_filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig
<< " feature values, but found " << numFeaturesInFile << std::endl;
<< " feature values, but found " << numFeaturesInFile << "\n";
throw strme.str();
}
std::vector<float> scores(numFeatureValuesInConfig, 0.0f);

2
moses/Hypothesis.h
View File

@ -209,7 +209,7 @@ public:
if (m_prevHypo != NULL) {
m_prevHypo->ToStream(out);
}
out << (Phrase) GetCurrTargetPhrase();
out << (const Phrase&) GetCurrTargetPhrase();
}
std::string GetOutputString() const {

3
moses/LM/Ken.cpp
View File

@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "lm/model.hh"
#include "util/exception.hh"
#include "util/tokenize_piece.hh"
#include "util/string_stream.hh"
#include "Ken.h"
#include "Base.h"
@ -474,7 +475,7 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig)
util::TokenIter<util::SingleCharacter, true> argument(lineOrig, ' ');
++argument; // KENLM
stringstream line;
util::StringStream line;
line << "KENLM";
for (; argument; ++argument) {

8
moses/LM/Remote.cpp
View File

@ -5,6 +5,7 @@
#include <sys/types.h>
#include "Remote.h"
#include "moses/Factor.h"
#include "util/string_stream.hh"
#if !defined(_WIN32) && !defined(_WIN64)
#include <arpa/inet.h>
@ -96,7 +97,7 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
cur->boState = *reinterpret_cast<const State*>(&m_curId);
++m_curId;
std::ostringstream os;
util::StringStream os;
os << "prob ";
if (event_word == NULL) {
os << "</s>";
@ -111,9 +112,8 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
os << ' ' << f->GetString();
}
}
os << std::endl;
std::string out = os.str();
write(sock, out.c_str(), out.size());
os << "\n";
write(sock, os.str().c_str(), os.str().size());
char res[6];
int r = read(sock, res, 6);
int errors = 0;

3
moses/Manager.cpp
View File

@ -56,6 +56,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "util/exception.hh"
#include "util/random.hh"
#include "util/string_stream.hh"
using namespace std;
@ -1971,7 +1972,7 @@ void Manager::OutputSearchGraphSLF() const
// Output search graph in HTK standard lattice format (SLF)
bool slf = staticData.GetOutputSearchGraphSLF();
if (slf) {
stringstream fileName;
util::StringStream fileName;
string dir;
staticData.GetParameter().SetParameter<string>(dir, "output-search-graph-slf", "");

7
moses/Parameter.cpp
View File

@ -30,6 +30,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Util.h"
#include "InputFileStream.h"
#include "StaticData.h"
#include "util/string_stream.hh"
#include "util/exception.hh"
#include "util/random.hh"
#include <boost/program_options.hpp>
@ -701,7 +702,7 @@ ConvertWeightArgsPhraseModel(const string &oldWeightName)
size_t currOldInd = 0;
for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
stringstream ptLine;
util::StringStream ptLine;
vector<string> token = Tokenize(translationVector[currDict]);
@ -860,7 +861,7 @@ ConvertWeightArgsDistortion()
}
SetWeight("LexicalReordering", indTable, weights);
stringstream strme;
util::StringStream strme;
strme << "LexicalReordering "
<< "type=" << toks[1] << " ";
@ -1007,7 +1008,7 @@ ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string
}
SetWeight(newWeightName, indTable, weights);
stringstream strme;
util::StringStream strme;
strme << "Generation "
<< "input-factor=" << modelToks[0] << " "
<< "output-factor=" << modelToks[1] << " "

3
moses/Phrase.cpp
View File

@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "StaticData.h" // GetMaxNumFactors
#include "util/string_piece.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh"
using namespace std;
@ -117,7 +118,7 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const
{
bool markUnknown = StaticData::Instance().GetMarkUnknown();
stringstream strme;
util::StringStream strme;
for (size_t pos = 0 ; pos < GetSize() ; pos++) {
if (markUnknown && GetWord(pos).IsOOV()) {
strme << StaticData::Instance().GetUnknownWordPrefix();

13
moses/ScoreComponentCollection.cpp
View File

@ -4,6 +4,7 @@
#include <boost/foreach.hpp>
#include <boost/format.hpp>
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "ScoreComponentCollection.h"
#include "StaticData.h"
#include "moses/FF/StatelessFeatureFunction.h"
@ -88,9 +89,8 @@ void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float s
{
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
std::stringstream name;
name << i->first;
if (starts_with(name.str(), prefix))
const std::string &name = i->first.name();
if (starts_with(name, prefix))
m_scores[i->first] = i->second * scalar;
}
}
@ -101,9 +101,8 @@ size_t ScoreComponentCollection::GetNumberWeights(const FeatureFunction* sp)
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
size_t weights = 0;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
std::stringstream name;
name << i->first;
if (starts_with(name.str(), prefix))
const std::string &name = i->first.name();
if (starts_with(name, prefix))
weights++;
}
return weights;
@ -215,7 +214,7 @@ void ScoreComponentCollection::Save(const string& filename) const
{
ofstream out(filename.c_str());
if (!out) {
ostringstream msg;
util::StringStream msg;
msg << "Unable to open " << filename;
throw runtime_error(msg.str());
}

4
moses/StaticData.cpp
View File

@ -640,7 +640,7 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const
switch (decodeType) {
case Translate:
if(index>=pts.size()) {
stringstream strme;
util::StringStream strme;
strme << "No phrase dictionary with index "
<< index << " available!";
UTIL_THROW(util::Exception, strme.str());
@ -649,7 +649,7 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const
break;
case Generate:
if(index>=gens.size()) {
stringstream strme;
util::StringStream strme;
strme << "No generation dictionary with index "
<< index << " available!";
UTIL_THROW(util::Exception, strme.str());

3
moses/Syntax/F2S/GlueRuleSynthesizer.cpp
View File

@ -4,6 +4,7 @@
#include "moses/FF/UnknownWordPenaltyProducer.h"
#include "moses/StaticData.h"
#include "util/string_stream.hh"
namespace Moses
{
@ -55,7 +56,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
TargetPhrase *targetPhrase = new TargetPhrase();
std::ostringstream alignmentSS;
util::StringStream alignmentSS;
for (std::size_t i = 0; i < e.tail.size(); ++i) {
const Word &symbol = e.tail[i]->pvertex.symbol;
if (symbol.IsNonTerminal()) {

2
moses/Syntax/T2S/GlueRuleSynthesizer.cpp
View File

@ -47,7 +47,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
TargetPhrase *targetPhrase = new TargetPhrase();
std::ostringstream alignmentSS;
util::StringStream alignmentSS;
for (std::size_t i = 0; i < node.children.size(); ++i) {
const Word &symbol = node.children[i]->pvertex.symbol;
if (symbol.IsNonTerminal()) {

6
moses/TranslationAnalysis.cpp
View File

@ -10,6 +10,7 @@
#include "moses/FF/StatefulFeatureFunction.h"
#include "moses/FF/StatelessFeatureFunction.h"
#include "moses/LM/Base.h"
#include "util/string_stream.hh"
using namespace Moses;
@ -40,8 +41,9 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo)
if (doLMStats)
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
for (; tpi != translationPath.end(); ++tpi) {
std::ostringstream sms;
std::ostringstream tms;
util::StringStream sms;
util::StringStream tms;
std::string target = (*tpi)->GetTargetPhraseStringRep();
std::string source = (*tpi)->GetSourcePhraseStringRep();
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();

9
moses/TranslationModel/CompactPT/BlockHashIndex.cpp
View File

@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "BlockHashIndex.h"
#include "CmphStringVectorAdapter.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
#ifdef HAVE_CMPH
#include "cmph.h"
@ -367,10 +368,10 @@ void BlockHashIndex::CalcHash(size_t current, void* source_void)
if(lastKey > temp) {
if(source->nkeys != 2 || temp != "###DUMMY_KEY###") {
std::stringstream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl;
strme << "1: " << lastKey << std::endl;
strme << "2: " << temp << std::endl;
util::StringStream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
strme << "1: " << lastKey << "\n";
strme << "2: " << temp << "\n";
UTIL_THROW2(strme.str());
}
}

9
moses/TranslationModel/CompactPT/BlockHashIndex.h
View File

@ -34,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "StringVector.h"
#include "PackedArray.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
#ifdef WITH_THREADS
#include "moses/ThreadPool.h"
@ -145,10 +146,10 @@ public:
size_t current = m_landmarks.size();
if(m_landmarks.size() && m_landmarks.back().str() >= keys[0]) {
std::stringstream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl;
strme << "1: " << m_landmarks.back().str() << std::endl;
strme << "2: " << keys[0] << std::endl;
util::StringStream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
strme << "1: " << m_landmarks.back().str() << "\n";
strme << "2: " << keys[0] << "\n";
UTIL_THROW2(strme.str());
}

5
moses/TranslationModel/PhraseDictionaryMultiModel.cpp
View File

@ -17,6 +17,7 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "moses/TranslationModel/PhraseDictionaryMultiModel.h"
@ -38,7 +39,7 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
} else if (m_mode == "all" || m_mode == "all-restrict") {
UTIL_THROW2("Implementation has moved: use PhraseDictionaryGroup with restrict=true/false");
} else {
ostringstream msg;
util::StringStream msg;
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
}
@ -210,7 +211,7 @@ std::vector<std::vector<float> > PhraseDictionaryMultiModel::getWeights(size_t n
raw_weights.push_back(1.0/m_numModels); //uniform weights created online
}
} else if(weights_ptr->size() != m_numModels && weights_ptr->size() != m_numModels * numWeights) {
std::stringstream strme;
util::StringStream strme;
strme << "Must have either one multimodel weight per model (" << m_numModels << "), or one per weighted feature and model (" << numWeights << "*" << m_numModels << "). You have " << weights_ptr->size() << ".";
UTIL_THROW(util::Exception, strme.str());
} else {

3
moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp
View File

@ -18,6 +18,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "util/exception.hh"
#include "util/tokenize.hh"
#include "util/string_stream.hh"
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h"
using namespace std;
@ -56,7 +57,7 @@ void PhraseDictionaryMultiModelCounts::SetParameter(const std::string& key, cons
else if (m_mode == "interpolate")
m_combineFunction = LinearInterpolationFromCounts;
else {
ostringstream msg;
util::StringStream msg;
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
}

3
moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp
View File

@ -15,6 +15,7 @@
#include "moses/PDTAimp.h"
#include "moses/TranslationTask.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std;
@ -52,7 +53,7 @@ void PhraseDictionaryTreeAdaptor::InitializeForInput(ttasksptr const& ttask)
vector<float> weight = staticData.GetWeights(this);
if(m_numScoreComponents!=weight.size()) {
std::stringstream strme;
util::StringStream strme;
UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size()
<< " " << m_numScoreComponents);
}

4
moses/TranslationModel/RuleTable/LoaderStandard.cpp
View File

@ -126,14 +126,14 @@ void ReformatHieroRule(const string &lineOrig, string &out)
ReformatHieroRule(1, targetPhraseString, ntAlign);
ReformateHieroScore(scoreString);
stringstream align;
util::StringStream align;
map<size_t, pair<size_t, size_t> >::const_iterator iterAlign;
for (iterAlign = ntAlign.begin(); iterAlign != ntAlign.end(); ++iterAlign) {
const pair<size_t, size_t> &alignPoint = iterAlign->second;
align << alignPoint.first << "-" << alignPoint.second << " ";
}
stringstream ret;
util::StringStream ret;
ret << sourcePhraseString << " ||| "
<< targetPhraseString << " ||| "
<< scoreString << " ||| "

3
moses/TranslationModel/fuzzy-match/SentenceAlignment.cpp
View File

@ -7,13 +7,14 @@
//
#include <iostream>
#include "util/string_stream.hh"
#include "SentenceAlignment.h"
namespace tmmt
{
std::string SentenceAlignment::getTargetString(const Vocabulary &vocab) const
{
std::stringstream strme;
util::StringStream strme;
for (size_t i = 0; i < target.size(); ++i) {
const WORD &word = vocab.GetWord(target[i]);
strme << word << " ";

3
moses/TranslationModel/fuzzy-match/SentenceAlignment.h
View File

@ -12,6 +12,7 @@
#include <sstream>
#include <vector>
#include "Vocabulary.h"
#include "util/string_stream.hh"
namespace tmmt
{
@ -27,7 +28,7 @@ struct SentenceAlignment {
std::string getTargetString(const Vocabulary &vocab) const;
std::string getAlignmentString() const {
std::stringstream strme;
util::StringStream strme;
for (size_t i = 0; i < alignment.size(); ++i) {
const std::pair<int,int> &alignPair = alignment[i];
strme << alignPair.first << "-" << alignPair.second << " ";

5
moses/Util.h
View File

@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <cstdlib>
#include <cstring>
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "TypeDef.h"
namespace Moses
@ -343,7 +344,7 @@ inline std::vector<std::string> TokenizeFirstOnly(const std::string& str,
template <typename T>
std::string Join(const std::string& delimiter, const std::vector<T>& items)
{
std::ostringstream outstr;
util::StringStream outstr;
if(items.size() == 0) return "";
outstr << items[0];
for(unsigned int i = 1; i < items.size(); i++)
@ -357,7 +358,7 @@ std::string Join(const std::string& delimiter, const std::vector<T>& items)
template<typename It>
std::string Join(const std::string &delim, It begin, It end)
{
std::ostringstream outstr;
util::StringStream outstr;
if (begin != end)
outstr << *begin++;
for ( ; begin != end; ++begin)

7
moses/Word.cpp
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FactorCollection.h"
#include "StaticData.h" // needed to determine the FactorDelimiter
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh"
using namespace std;
@ -94,7 +95,7 @@ void Word::Merge(const Word &sourceWord)
std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlank) const
{
stringstream strme;
util::StringStream strme;
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
bool firstPass = true;
unsigned int stop = min(max_fax(),factorType.size());
@ -210,7 +211,7 @@ TO_STRING_BODY(Word);
// friend
ostream& operator<<(ostream& out, const Word& word)
{
stringstream strme;
util::StringStream strme;
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
bool firstPass = true;
unsigned int stop = max_fax();
@ -223,7 +224,7 @@ ostream& operator<<(ostream& out, const Word& word)
} else {
strme << factorDelimiter;
}
strme << *factor;
strme << factor->GetString();
}
}
out << strme.str() << " ";

3
util/exception.cc
View File

@ -25,7 +25,8 @@ void Exception::SetLocation(const char *file, unsigned int line, const char *fun
*/
std::string old_text;
std::swap(old_text, what_);
StringStream stream(what_);
StringStream stream;
stream << what_;
stream << file << ':' << line;
if (func) stream << " in " << func << " threw ";
if (child_name) {

17
util/string_stream.hh
View File

@ -11,9 +11,13 @@ namespace util {
class StringStream : public FakeOStream<StringStream> {
public:
// Semantics: appends to string. Remember to clear first!
explicit StringStream(std::string &out)
: out_(out) {}
explicit StringStream()
{}
/*
explicit StringStream(std::string &out)
: out_(out) {}
*/
StringStream &flush() { return *this; }
StringStream &write(const void *data, std::size_t length) {
@ -21,6 +25,13 @@ class StringStream : public FakeOStream<StringStream> {
return *this;
}
const std::string &str() const
{ return out_; }
void str(const std::string &val)
{
out_ = val;
}
protected:
friend class FakeOStream<StringStream>;
char *Ensure(std::size_t amount) {
@ -36,7 +47,7 @@ class StringStream : public FakeOStream<StringStream> {
}
private:
std::string &out_;
std::string out_;
};
} // namespace

18
util/string_stream_test.cc
View File

@ -11,9 +11,9 @@
namespace util { namespace {
template <class T> void TestEqual(const T value) {
std::string str;
StringStream(str) << value;
BOOST_CHECK_EQUAL(boost::lexical_cast<std::string>(value), str);
StringStream strme;
strme << value;
BOOST_CHECK_EQUAL(boost::lexical_cast<std::string>(value), strme.str());
}
template <class T> void TestCorners() {
@ -66,15 +66,15 @@ BOOST_AUTO_TEST_CASE(Strings) {
non_const[0] = 'b';
non_const[1] = 'c';
non_const[2] = 0;
std::string out;
StringStream(out) << "a" << non_const << 'c';
BOOST_CHECK_EQUAL("abcc", out);
StringStream out;
out << "a" << non_const << 'c';
BOOST_CHECK_EQUAL("abcc", out.str());
// Now test as a separate object.
out.clear();
StringStream stream(out);
StringStream stream;
stream << "a" << non_const << 'c' << piece;
BOOST_CHECK_EQUAL("abccabcdef", out);
BOOST_CHECK_EQUAL("abccabcdef", stream.str());
}
}} // namespaces