moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-17 23:40:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

use util::StringStream

Browse Source
This commit is contained in:
Hieu Hoang 2015-10-02 20:12:39 +01:00
parent 786258cb28
commit cfb1ab34c7
25 changed files with 124 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
OnDiskPt/OnDiskWrapper.cpp
View File

@ -25,6 +25,7 @@
#include "OnDiskWrapper.h"
#include "moses/Factor.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std;
@ -223,7 +224,9 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact
{
bool isNonTerminal = origWord.IsNonTerminal();
Word *newWord = new Word(isNonTerminal);
stringstream strme;
string str;
util::StringStream strme(str);
size_t factorType = factorsVec[0];
const Moses::Factor *factor = origWord.GetFactor(factorType);
@ -243,7 +246,7 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact
} // for (size_t factorType
bool found;
uint64_t vocabId = m_vocab.GetVocabId(strme.str(), found);
uint64_t vocabId = m_vocab.GetVocabId(str, found);
if (!found) {
// factor not in phrase table -> phrse definately not in. exit
delete newWord;

8
moses-cmd/Main.cpp
View File

@ -23,10 +23,18 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
* Moses main wrapper for executable for single-threaded and multi-threaded, simply calling decoder_main.
**/
#include "moses/ExportInterface.h"
#include "util/string_stream.hh"
/** main function of the command line version of the decoder **/
int main(int argc, char** argv)
{
std::string output;
util::StringStream strme2(output);
strme2 << StringPiece("JJJJ");
strme2 << std::string("kkkk");
std::cout << output;
return decoder_main(argc, argv);
}

11
moses/FF/LexicalReordering/SparseReordering.cpp
View File

@ -8,6 +8,7 @@
#include "util/file_piece.hh"
#include "util/string_piece.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh"
#include "LexicalReordering.h"
@ -26,7 +27,8 @@ const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
{
static string kSep = "-";
static string name;
ostringstream buf;
std::string str;
util::StringStream buf(str);
// type side position id word reotype
if (type == Phrase) {
buf << "phr";
@ -54,7 +56,7 @@ const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
buf << word->GetString();
buf << kSep;
buf << reoType;
name = buf.str();
name = str;
return name;
}
@ -88,9 +90,10 @@ SparseReordering::SparseReordering(const map<string,string>& config, const Lexic
ReadWeightMap(i->second);
m_useWeightMap = true;
for (int reoType=0; reoType<=LRModel::MAX; ++reoType) {
ostringstream buf;
std::string str;
util::StringStream buf(str);
buf << reoType;
m_featureMap2.push_back(m_producer->GetFeatureName(buf.str()));
m_featureMap2.push_back(m_producer->GetFeatureName(str));
}
} else if (fields[0] == "phrase") {

6
moses/FF/OSM-Feature/osmHyp.cpp
View File

@ -182,11 +182,7 @@ int osmHypothesis :: firstOpenGap(vector <int> & coverageVector)
string osmHypothesis :: intToString(int num)
{
std::ostringstream stm;
stm<<num;
return stm.str();
return SPrint(num);
}

6
moses/FeatureVector.cpp
View File

@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FeatureVector.h"
#include "util/string_piece_hash.hh"
#include "util/string_stream.hh"
using namespace std;
@ -204,9 +205,10 @@ void FVector::save(const string& filename) const
{
ofstream out(filename.c_str());
if (!out) {
ostringstream msg;
std::string str;
util::StringStream msg(str);
msg << "Unable to open " << filename;
throw runtime_error(msg.str());
throw runtime_error(str);
}
write(out);
out.close();

8
moses/GenerationDictionary.cpp
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "InputFileStream.h"
#include "StaticData.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std;
@ -84,10 +85,11 @@ void GenerationDictionary::Load()
size_t numFeaturesInFile = token.size() - 2;
if (numFeaturesInFile < numFeatureValuesInConfig) {
stringstream strme;
std::string str;
util::StringStream strme(str);
strme << m_filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig
<< " feature values, but found " << numFeaturesInFile << std::endl;
throw strme.str();
<< " feature values, but found " << numFeaturesInFile << "\n";
throw str;
}
std::vector<float> scores(numFeatureValuesInConfig, 0.0f);
for (size_t i = 0; i < numFeatureValuesInConfig; i++)

2
moses/Hypothesis.h
View File

@ -209,7 +209,7 @@ public:
if (m_prevHypo != NULL) {
m_prevHypo->ToStream(out);
}
out << (Phrase) GetCurrTargetPhrase();
out << (const Phrase&) GetCurrTargetPhrase();
}
std::string GetOutputString() const {

6
moses/LM/Ken.cpp
View File

@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "lm/model.hh"
#include "util/exception.hh"
#include "util/tokenize_piece.hh"
#include "util/string_stream.hh"
#include "Ken.h"
#include "Base.h"
@ -487,7 +488,8 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig)
util::TokenIter<util::SingleCharacter, true> argument(lineOrig, ' ');
++argument; // KENLM
stringstream line;
string str;
util::StringStream line(str);
line << "KENLM";
for (; argument; ++argument) {
@ -510,7 +512,7 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig)
}
}
return ConstructKenLM(line.str(), filePath, factorType, lazy);
return ConstructKenLM(str, filePath, factorType, lazy);
}
LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy)

7
moses/LM/Remote.cpp
View File

@ -5,6 +5,7 @@
#include <sys/types.h>
#include "Remote.h"
#include "moses/Factor.h"
#include "util/string_stream.hh"
#if !defined(_WIN32) && !defined(_WIN64)
#include <arpa/inet.h>
@ -96,7 +97,8 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
cur->boState = *reinterpret_cast<const State*>(&m_curId);
++m_curId;
std::ostringstream os;
std::string out;
util::StringStream os(out);
os << "prob ";
if (event_word == NULL) {
os << "</s>";
@ -111,8 +113,7 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
os << ' ' << f->GetString();
}
}
os << std::endl;
std::string out = os.str();
os << "\n";
write(sock, out.c_str(), out.size());
char res[6];
int r = read(sock, res, 6);

16
moses/Parameter.cpp
View File

@ -30,6 +30,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Util.h"
#include "InputFileStream.h"
#include "StaticData.h"
#include "util/string_stream.hh"
#include "util/exception.hh"
#include "util/random.hh"
#include <boost/program_options.hpp>
@ -701,7 +702,8 @@ ConvertWeightArgsPhraseModel(const string &oldWeightName)
size_t currOldInd = 0;
for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
stringstream ptLine;
string ptLineStr;
util::StringStream ptLine(ptLineStr);
vector<string> token = Tokenize(translationVector[currDict]);
@ -797,7 +799,7 @@ ConvertWeightArgsPhraseModel(const string &oldWeightName)
ptLine << "alignment-path=" << token[6] << " ";
}
AddFeature(ptLine.str());
AddFeature(ptLineStr);
} // for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
} // if (GetParam("ttable-file").size() > 0) {
@ -860,7 +862,8 @@ ConvertWeightArgsDistortion()
}
SetWeight("LexicalReordering", indTable, weights);
stringstream strme;
string str;
util::StringStream strme(str);
strme << "LexicalReordering "
<< "type=" << toks[1] << " ";
@ -874,7 +877,7 @@ ConvertWeightArgsDistortion()
strme << "num-features=" << toks[2] << " ";
strme << "path=" << toks[3];
AddFeature(strme.str());
AddFeature(str);
}
}
@ -1007,13 +1010,14 @@ ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string
}
SetWeight(newWeightName, indTable, weights);
stringstream strme;
string str;
util::StringStream strme(str);
strme << "Generation "
<< "input-factor=" << modelToks[0] << " "
<< "output-factor=" << modelToks[1] << " "
<< "num-features=" << modelToks[2] << " "
<< "path=" << modelToks[3];
AddFeature(strme.str());
AddFeature(str);
}
}

6
moses/Phrase.cpp
View File

@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "StaticData.h" // GetMaxNumFactors
#include "util/string_piece.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh"
using namespace std;
@ -117,7 +118,8 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const
{
bool markUnknown = StaticData::Instance().GetMarkUnknown();
stringstream strme;
string str;
util::StringStream strme(str);
for (size_t pos = 0 ; pos < GetSize() ; pos++) {
if (markUnknown && GetWord(pos).IsOOV()) {
strme << StaticData::Instance().GetUnknownWordPrefix();
@ -128,7 +130,7 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const
}
}
return strme.str();
return str;
}
Word &Phrase::AddWord()

10
moses/ScoreComponentCollection.cpp
View File

@ -88,9 +88,8 @@ void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float s
{
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
std::stringstream name;
name << i->first;
if (starts_with(name.str(), prefix))
const std::string &name = i->first.name();
if (starts_with(name, prefix))
m_scores[i->first] = i->second * scalar;
}
}
@ -101,9 +100,8 @@ size_t ScoreComponentCollection::GetNumberWeights(const FeatureFunction* sp)
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
size_t weights = 0;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
std::stringstream name;
name << i->first;
if (starts_with(name.str(), prefix))
const std::string &name = i->first.name();
if (starts_with(name, prefix))
weights++;
}
return weights;

10
moses/StaticData.cpp
View File

@ -640,19 +640,21 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const
switch (decodeType) {
case Translate:
if(index>=pts.size()) {
stringstream strme;
string str;
util::StringStream strme(str);
strme << "No phrase dictionary with index "
<< index << " available!";
UTIL_THROW(util::Exception, strme.str());
UTIL_THROW(util::Exception, str);
}
decodeStep = new DecodeStepTranslation(pts[index], prev, *featuresRemaining);
break;
case Generate:
if(index>=gens.size()) {
stringstream strme;
string str;
util::StringStream strme(str);
strme << "No generation dictionary with index "
<< index << " available!";
UTIL_THROW(util::Exception, strme.str());
UTIL_THROW(util::Exception, str);
}
decodeStep = new DecodeStepGeneration(gens[index], prev, *featuresRemaining);
break;

6
moses/Syntax/F2S/GlueRuleSynthesizer.cpp
View File

@ -4,6 +4,7 @@
#include "moses/FF/UnknownWordPenaltyProducer.h"
#include "moses/StaticData.h"
#include "util/string_stream.hh"
namespace Moses
{
@ -55,7 +56,8 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
TargetPhrase *targetPhrase = new TargetPhrase();
std::ostringstream alignmentSS;
std::string alignmentSSStr;
util::StringStream alignmentSS(alignmentSSStr);
for (std::size_t i = 0; i < e.tail.size(); ++i) {
const Word &symbol = e.tail[i]->pvertex.symbol;
if (symbol.IsNonTerminal()) {
@ -75,7 +77,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
targetPhrase->EvaluateInIsolation(m_dummySourcePhrase);
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
targetPhrase->SetTargetLHS(targetLhs);
targetPhrase->SetAlignmentInfo(alignmentSS.str());
targetPhrase->SetAlignmentInfo(alignmentSSStr);
return targetPhrase;
}

5
moses/Syntax/T2S/GlueRuleSynthesizer.cpp
View File

@ -47,7 +47,8 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
TargetPhrase *targetPhrase = new TargetPhrase();
std::ostringstream alignmentSS;
std::string alignmentSSStr;
util::StringStream alignmentSS(alignmentSSStr);
for (std::size_t i = 0; i < node.children.size(); ++i) {
const Word &symbol = node.children[i]->pvertex.symbol;
if (symbol.IsNonTerminal()) {
@ -67,7 +68,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
targetPhrase->EvaluateInIsolation(sourceRhs);
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
targetPhrase->SetTargetLHS(targetLhs);
targetPhrase->SetAlignmentInfo(alignmentSS.str());
targetPhrase->SetAlignmentInfo(alignmentSSStr);
return targetPhrase;
}

12
moses/TranslationAnalysis.cpp
View File

@ -10,6 +10,7 @@
#include "moses/FF/StatefulFeatureFunction.h"
#include "moses/FF/StatelessFeatureFunction.h"
#include "moses/LM/Base.h"
#include "util/string_stream.hh"
using namespace Moses;
@ -40,8 +41,11 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo)
if (doLMStats)
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
for (; tpi != translationPath.end(); ++tpi) {
std::ostringstream sms;
std::ostringstream tms;
std::string smsStr;
util::StringStream sms(smsStr);
std::string tmsStr;
util::StringStream tms(tmsStr);
std::string target = (*tpi)->GetTargetPhraseStringRep();
std::string source = (*tpi)->GetSourcePhraseStringRep();
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();
@ -89,8 +93,8 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo)
for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) {
tms << '-' << swr_i;
}
if (!epsilon) targetMap.push_back(sms.str());
sourceMap.push_back(tms.str());
if (!epsilon) targetMap.push_back(smsStr);
sourceMap.push_back(tmsStr);
}
std::vector<std::string>::iterator si = sourceMap.begin();
std::vector<std::string>::iterator ti = targetMap.begin();

12
moses/TranslationModel/CompactPT/BlockHashIndex.cpp
View File

@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "BlockHashIndex.h"
#include "CmphStringVectorAdapter.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
#ifdef HAVE_CMPH
#include "cmph.h"
@ -367,11 +368,12 @@ void BlockHashIndex::CalcHash(size_t current, void* source_void)
if(lastKey > temp) {
if(source->nkeys != 2 || temp != "###DUMMY_KEY###") {
std::stringstream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl;
strme << "1: " << lastKey << std::endl;
strme << "2: " << temp << std::endl;
UTIL_THROW2(strme.str());
std::string str;
util::StringStream strme(str);
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
strme << "1: " << lastKey << "\n";
strme << "2: " << temp << "\n";
UTIL_THROW2(str);
}
}
lastKey = temp;

12
moses/TranslationModel/CompactPT/BlockHashIndex.h
View File

@ -34,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "StringVector.h"
#include "PackedArray.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
#ifdef WITH_THREADS
#include "moses/ThreadPool.h"
@ -145,11 +146,12 @@ public:
size_t current = m_landmarks.size();
if(m_landmarks.size() && m_landmarks.back().str() >= keys[0]) {
std::stringstream strme;
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl;
strme << "1: " << m_landmarks.back().str() << std::endl;
strme << "2: " << keys[0] << std::endl;
UTIL_THROW2(strme.str());
std::string str;
util::StringStream strme(str);
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
strme << "1: " << m_landmarks.back().str() << "\n";
strme << "2: " << keys[0] << "\n";
UTIL_THROW2(str);
}
m_landmarks.push_back(keys[0]);

11
moses/TranslationModel/PhraseDictionaryMultiModel.cpp
View File

@ -17,6 +17,7 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "moses/TranslationModel/PhraseDictionaryMultiModel.h"
@ -38,9 +39,10 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
} else if (m_mode == "all" || m_mode == "all-restrict") {
UTIL_THROW2("Implementation has moved: use PhraseDictionaryGroup with restrict=true/false");
} else {
ostringstream msg;
string str;
util::StringStream msg(str);
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
throw runtime_error(str);
}
}
@ -210,9 +212,10 @@ std::vector<std::vector<float> > PhraseDictionaryMultiModel::getWeights(size_t n
raw_weights.push_back(1.0/m_numModels); //uniform weights created online
}
} else if(weights_ptr->size() != m_numModels && weights_ptr->size() != m_numModels * numWeights) {
std::stringstream strme;
string str;
util::StringStream strme(str);
strme << "Must have either one multimodel weight per model (" << m_numModels << "), or one per weighted feature and model (" << numWeights << "*" << m_numModels << "). You have " << weights_ptr->size() << ".";
UTIL_THROW(util::Exception, strme.str());
UTIL_THROW(util::Exception, str);
} else {
raw_weights = *weights_ptr;
}

6
moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp
View File

@ -18,6 +18,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "util/exception.hh"
#include "util/tokenize.hh"
#include "util/string_stream.hh"
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h"
using namespace std;
@ -56,9 +57,10 @@ void PhraseDictionaryMultiModelCounts::SetParameter(const std::string& key, cons
else if (m_mode == "interpolate")
m_combineFunction = LinearInterpolationFromCounts;
else {
ostringstream msg;
string str;
util::StringStream msg(str);
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
throw runtime_error(str);
}
} else if (key == "lex-e2f") {
m_lexE2FStr = Tokenize(value, ",");

4
moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp
View File

@ -15,6 +15,7 @@
#include "moses/PDTAimp.h"
#include "moses/TranslationTask.h"
#include "util/exception.hh"
#include "util/string_stream.hh"
using namespace std;
@ -52,7 +53,8 @@ void PhraseDictionaryTreeAdaptor::InitializeForInput(ttasksptr const& ttask)
vector<float> weight = staticData.GetWeights(this);
if(m_numScoreComponents!=weight.size()) {
std::stringstream strme;
string str;
util::StringStream strme(str);
UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size()
<< " " << m_numScoreComponents);
}

10
moses/TranslationModel/RuleTable/LoaderStandard.cpp
View File

@ -126,20 +126,22 @@ void ReformatHieroRule(const string &lineOrig, string &out)
ReformatHieroRule(1, targetPhraseString, ntAlign);
ReformateHieroScore(scoreString);
stringstream align;
std::string alignStr;
util::StringStream align(alignStr);
map<size_t, pair<size_t, size_t> >::const_iterator iterAlign;
for (iterAlign = ntAlign.begin(); iterAlign != ntAlign.end(); ++iterAlign) {
const pair<size_t, size_t> &alignPoint = iterAlign->second;
align << alignPoint.first << "-" << alignPoint.second << " ";
}
stringstream ret;
std::string str;
util::StringStream ret(str);
ret << sourcePhraseString << " ||| "
<< targetPhraseString << " ||| "
<< scoreString << " ||| "
<< align.str();
<< alignStr;
out = ret.str();
out = str;
}
bool RuleTableLoaderStandard::Load(FormatType format

6
moses/TranslationModel/fuzzy-match/SentenceAlignment.cpp
View File

@ -7,18 +7,20 @@
//
#include <iostream>
#include "util/string_stream.hh"
#include "SentenceAlignment.h"
namespace tmmt
{
std::string SentenceAlignment::getTargetString(const Vocabulary &vocab) const
{
std::stringstream strme;
std::string str;
util::StringStream strme(str);
for (size_t i = 0; i < target.size(); ++i) {
const WORD &word = vocab.GetWord(target[i]);
strme << word << " ";
}
return strme.str();
return str;
}
}

6
moses/TranslationModel/fuzzy-match/SentenceAlignment.h
View File

@ -12,6 +12,7 @@
#include <sstream>
#include <vector>
#include "Vocabulary.h"
#include "util/string_stream.hh"
namespace tmmt
{
@ -27,12 +28,13 @@ struct SentenceAlignment {
std::string getTargetString(const Vocabulary &vocab) const;
std::string getAlignmentString() const {
std::stringstream strme;
std::string str;
util::StringStream strme(str);
for (size_t i = 0; i < alignment.size(); ++i) {
const std::pair<int,int> &alignPair = alignment[i];
strme << alignPair.first << "-" << alignPair.second << " ";
}
return strme.str();
return str;
}
};

6
moses/Word.cpp
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FactorCollection.h"
#include "StaticData.h" // needed to determine the FactorDelimiter
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "util/tokenize_piece.hh"
using namespace std;
@ -79,7 +80,8 @@ void Word::Merge(const Word &sourceWord)
std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlank) const
{
stringstream strme;
string str;
util::StringStream strme(str);
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
bool firstPass = true;
unsigned int stop = min(max_fax(),factorType.size());
@ -99,7 +101,7 @@ std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlan
}
}
if(endWithBlank) strme << " ";
return strme.str();
return str;
}
StringPiece Word::GetString(FactorType factorType) const