mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
use util::StringStream
This commit is contained in:
parent
786258cb28
commit
cfb1ab34c7
@ -25,6 +25,7 @@
|
|||||||
#include "OnDiskWrapper.h"
|
#include "OnDiskWrapper.h"
|
||||||
#include "moses/Factor.h"
|
#include "moses/Factor.h"
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
@ -223,7 +224,9 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact
|
|||||||
{
|
{
|
||||||
bool isNonTerminal = origWord.IsNonTerminal();
|
bool isNonTerminal = origWord.IsNonTerminal();
|
||||||
Word *newWord = new Word(isNonTerminal);
|
Word *newWord = new Word(isNonTerminal);
|
||||||
stringstream strme;
|
|
||||||
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
|
|
||||||
size_t factorType = factorsVec[0];
|
size_t factorType = factorsVec[0];
|
||||||
const Moses::Factor *factor = origWord.GetFactor(factorType);
|
const Moses::Factor *factor = origWord.GetFactor(factorType);
|
||||||
@ -243,7 +246,7 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact
|
|||||||
} // for (size_t factorType
|
} // for (size_t factorType
|
||||||
|
|
||||||
bool found;
|
bool found;
|
||||||
uint64_t vocabId = m_vocab.GetVocabId(strme.str(), found);
|
uint64_t vocabId = m_vocab.GetVocabId(str, found);
|
||||||
if (!found) {
|
if (!found) {
|
||||||
// factor not in phrase table -> phrse definately not in. exit
|
// factor not in phrase table -> phrse definately not in. exit
|
||||||
delete newWord;
|
delete newWord;
|
||||||
|
@ -23,10 +23,18 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
* Moses main wrapper for executable for single-threaded and multi-threaded, simply calling decoder_main.
|
* Moses main wrapper for executable for single-threaded and multi-threaded, simply calling decoder_main.
|
||||||
**/
|
**/
|
||||||
#include "moses/ExportInterface.h"
|
#include "moses/ExportInterface.h"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
/** main function of the command line version of the decoder **/
|
/** main function of the command line version of the decoder **/
|
||||||
int main(int argc, char** argv)
|
int main(int argc, char** argv)
|
||||||
{
|
{
|
||||||
|
std::string output;
|
||||||
|
util::StringStream strme2(output);
|
||||||
|
strme2 << StringPiece("JJJJ");
|
||||||
|
strme2 << std::string("kkkk");
|
||||||
|
|
||||||
|
std::cout << output;
|
||||||
|
|
||||||
return decoder_main(argc, argv);
|
return decoder_main(argc, argv);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8,6 +8,7 @@
|
|||||||
|
|
||||||
#include "util/file_piece.hh"
|
#include "util/file_piece.hh"
|
||||||
#include "util/string_piece.hh"
|
#include "util/string_piece.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
#include "util/tokenize_piece.hh"
|
#include "util/tokenize_piece.hh"
|
||||||
|
|
||||||
#include "LexicalReordering.h"
|
#include "LexicalReordering.h"
|
||||||
@ -26,7 +27,8 @@ const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
|
|||||||
{
|
{
|
||||||
static string kSep = "-";
|
static string kSep = "-";
|
||||||
static string name;
|
static string name;
|
||||||
ostringstream buf;
|
std::string str;
|
||||||
|
util::StringStream buf(str);
|
||||||
// type side position id word reotype
|
// type side position id word reotype
|
||||||
if (type == Phrase) {
|
if (type == Phrase) {
|
||||||
buf << "phr";
|
buf << "phr";
|
||||||
@ -54,7 +56,7 @@ const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
|
|||||||
buf << word->GetString();
|
buf << word->GetString();
|
||||||
buf << kSep;
|
buf << kSep;
|
||||||
buf << reoType;
|
buf << reoType;
|
||||||
name = buf.str();
|
name = str;
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -88,9 +90,10 @@ SparseReordering::SparseReordering(const map<string,string>& config, const Lexic
|
|||||||
ReadWeightMap(i->second);
|
ReadWeightMap(i->second);
|
||||||
m_useWeightMap = true;
|
m_useWeightMap = true;
|
||||||
for (int reoType=0; reoType<=LRModel::MAX; ++reoType) {
|
for (int reoType=0; reoType<=LRModel::MAX; ++reoType) {
|
||||||
ostringstream buf;
|
std::string str;
|
||||||
|
util::StringStream buf(str);
|
||||||
buf << reoType;
|
buf << reoType;
|
||||||
m_featureMap2.push_back(m_producer->GetFeatureName(buf.str()));
|
m_featureMap2.push_back(m_producer->GetFeatureName(str));
|
||||||
}
|
}
|
||||||
|
|
||||||
} else if (fields[0] == "phrase") {
|
} else if (fields[0] == "phrase") {
|
||||||
|
@ -182,11 +182,7 @@ int osmHypothesis :: firstOpenGap(vector <int> & coverageVector)
|
|||||||
|
|
||||||
string osmHypothesis :: intToString(int num)
|
string osmHypothesis :: intToString(int num)
|
||||||
{
|
{
|
||||||
|
return SPrint(num);
|
||||||
std::ostringstream stm;
|
|
||||||
stm<<num;
|
|
||||||
|
|
||||||
return stm.str();
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
|
|
||||||
#include "FeatureVector.h"
|
#include "FeatureVector.h"
|
||||||
#include "util/string_piece_hash.hh"
|
#include "util/string_piece_hash.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
@ -204,9 +205,10 @@ void FVector::save(const string& filename) const
|
|||||||
{
|
{
|
||||||
ofstream out(filename.c_str());
|
ofstream out(filename.c_str());
|
||||||
if (!out) {
|
if (!out) {
|
||||||
ostringstream msg;
|
std::string str;
|
||||||
|
util::StringStream msg(str);
|
||||||
msg << "Unable to open " << filename;
|
msg << "Unable to open " << filename;
|
||||||
throw runtime_error(msg.str());
|
throw runtime_error(str);
|
||||||
}
|
}
|
||||||
write(out);
|
write(out);
|
||||||
out.close();
|
out.close();
|
||||||
|
@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "InputFileStream.h"
|
#include "InputFileStream.h"
|
||||||
#include "StaticData.h"
|
#include "StaticData.h"
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
@ -84,10 +85,11 @@ void GenerationDictionary::Load()
|
|||||||
|
|
||||||
size_t numFeaturesInFile = token.size() - 2;
|
size_t numFeaturesInFile = token.size() - 2;
|
||||||
if (numFeaturesInFile < numFeatureValuesInConfig) {
|
if (numFeaturesInFile < numFeatureValuesInConfig) {
|
||||||
stringstream strme;
|
std::string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
strme << m_filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig
|
strme << m_filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig
|
||||||
<< " feature values, but found " << numFeaturesInFile << std::endl;
|
<< " feature values, but found " << numFeaturesInFile << "\n";
|
||||||
throw strme.str();
|
throw str;
|
||||||
}
|
}
|
||||||
std::vector<float> scores(numFeatureValuesInConfig, 0.0f);
|
std::vector<float> scores(numFeatureValuesInConfig, 0.0f);
|
||||||
for (size_t i = 0; i < numFeatureValuesInConfig; i++)
|
for (size_t i = 0; i < numFeatureValuesInConfig; i++)
|
||||||
|
@ -209,7 +209,7 @@ public:
|
|||||||
if (m_prevHypo != NULL) {
|
if (m_prevHypo != NULL) {
|
||||||
m_prevHypo->ToStream(out);
|
m_prevHypo->ToStream(out);
|
||||||
}
|
}
|
||||||
out << (Phrase) GetCurrTargetPhrase();
|
out << (const Phrase&) GetCurrTargetPhrase();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string GetOutputString() const {
|
std::string GetOutputString() const {
|
||||||
|
@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "lm/model.hh"
|
#include "lm/model.hh"
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
#include "util/tokenize_piece.hh"
|
#include "util/tokenize_piece.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
#include "Ken.h"
|
#include "Ken.h"
|
||||||
#include "Base.h"
|
#include "Base.h"
|
||||||
@ -487,7 +488,8 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig)
|
|||||||
util::TokenIter<util::SingleCharacter, true> argument(lineOrig, ' ');
|
util::TokenIter<util::SingleCharacter, true> argument(lineOrig, ' ');
|
||||||
++argument; // KENLM
|
++argument; // KENLM
|
||||||
|
|
||||||
stringstream line;
|
string str;
|
||||||
|
util::StringStream line(str);
|
||||||
line << "KENLM";
|
line << "KENLM";
|
||||||
|
|
||||||
for (; argument; ++argument) {
|
for (; argument; ++argument) {
|
||||||
@ -510,7 +512,7 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return ConstructKenLM(line.str(), filePath, factorType, lazy);
|
return ConstructKenLM(str, filePath, factorType, lazy);
|
||||||
}
|
}
|
||||||
|
|
||||||
LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
|
LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
|
||||||
|
@ -5,6 +5,7 @@
|
|||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include "Remote.h"
|
#include "Remote.h"
|
||||||
#include "moses/Factor.h"
|
#include "moses/Factor.h"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
#if !defined(_WIN32) && !defined(_WIN64)
|
#if !defined(_WIN32) && !defined(_WIN64)
|
||||||
#include <arpa/inet.h>
|
#include <arpa/inet.h>
|
||||||
@ -96,7 +97,8 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
|
|||||||
cur->boState = *reinterpret_cast<const State*>(&m_curId);
|
cur->boState = *reinterpret_cast<const State*>(&m_curId);
|
||||||
++m_curId;
|
++m_curId;
|
||||||
|
|
||||||
std::ostringstream os;
|
std::string out;
|
||||||
|
util::StringStream os(out);
|
||||||
os << "prob ";
|
os << "prob ";
|
||||||
if (event_word == NULL) {
|
if (event_word == NULL) {
|
||||||
os << "</s>";
|
os << "</s>";
|
||||||
@ -111,8 +113,7 @@ LMResult LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFa
|
|||||||
os << ' ' << f->GetString();
|
os << ' ' << f->GetString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
os << std::endl;
|
os << "\n";
|
||||||
std::string out = os.str();
|
|
||||||
write(sock, out.c_str(), out.size());
|
write(sock, out.c_str(), out.size());
|
||||||
char res[6];
|
char res[6];
|
||||||
int r = read(sock, res, 6);
|
int r = read(sock, res, 6);
|
||||||
|
@ -30,6 +30,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "Util.h"
|
#include "Util.h"
|
||||||
#include "InputFileStream.h"
|
#include "InputFileStream.h"
|
||||||
#include "StaticData.h"
|
#include "StaticData.h"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
#include "util/random.hh"
|
#include "util/random.hh"
|
||||||
#include <boost/program_options.hpp>
|
#include <boost/program_options.hpp>
|
||||||
@ -701,7 +702,8 @@ ConvertWeightArgsPhraseModel(const string &oldWeightName)
|
|||||||
|
|
||||||
size_t currOldInd = 0;
|
size_t currOldInd = 0;
|
||||||
for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
|
for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
|
||||||
stringstream ptLine;
|
string ptLineStr;
|
||||||
|
util::StringStream ptLine(ptLineStr);
|
||||||
|
|
||||||
vector<string> token = Tokenize(translationVector[currDict]);
|
vector<string> token = Tokenize(translationVector[currDict]);
|
||||||
|
|
||||||
@ -797,7 +799,7 @@ ConvertWeightArgsPhraseModel(const string &oldWeightName)
|
|||||||
ptLine << "alignment-path=" << token[6] << " ";
|
ptLine << "alignment-path=" << token[6] << " ";
|
||||||
}
|
}
|
||||||
|
|
||||||
AddFeature(ptLine.str());
|
AddFeature(ptLineStr);
|
||||||
} // for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
|
} // for(size_t currDict = 0 ; currDict < translationVector.size(); currDict++) {
|
||||||
} // if (GetParam("ttable-file").size() > 0) {
|
} // if (GetParam("ttable-file").size() > 0) {
|
||||||
|
|
||||||
@ -860,7 +862,8 @@ ConvertWeightArgsDistortion()
|
|||||||
}
|
}
|
||||||
SetWeight("LexicalReordering", indTable, weights);
|
SetWeight("LexicalReordering", indTable, weights);
|
||||||
|
|
||||||
stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
strme << "LexicalReordering "
|
strme << "LexicalReordering "
|
||||||
<< "type=" << toks[1] << " ";
|
<< "type=" << toks[1] << " ";
|
||||||
|
|
||||||
@ -874,7 +877,7 @@ ConvertWeightArgsDistortion()
|
|||||||
strme << "num-features=" << toks[2] << " ";
|
strme << "num-features=" << toks[2] << " ";
|
||||||
strme << "path=" << toks[3];
|
strme << "path=" << toks[3];
|
||||||
|
|
||||||
AddFeature(strme.str());
|
AddFeature(str);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1007,13 +1010,14 @@ ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string
|
|||||||
}
|
}
|
||||||
SetWeight(newWeightName, indTable, weights);
|
SetWeight(newWeightName, indTable, weights);
|
||||||
|
|
||||||
stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
strme << "Generation "
|
strme << "Generation "
|
||||||
<< "input-factor=" << modelToks[0] << " "
|
<< "input-factor=" << modelToks[0] << " "
|
||||||
<< "output-factor=" << modelToks[1] << " "
|
<< "output-factor=" << modelToks[1] << " "
|
||||||
<< "num-features=" << modelToks[2] << " "
|
<< "num-features=" << modelToks[2] << " "
|
||||||
<< "path=" << modelToks[3];
|
<< "path=" << modelToks[3];
|
||||||
AddFeature(strme.str());
|
AddFeature(str);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "StaticData.h" // GetMaxNumFactors
|
#include "StaticData.h" // GetMaxNumFactors
|
||||||
|
|
||||||
#include "util/string_piece.hh"
|
#include "util/string_piece.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
#include "util/tokenize_piece.hh"
|
#include "util/tokenize_piece.hh"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
@ -117,7 +118,8 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const
|
|||||||
{
|
{
|
||||||
bool markUnknown = StaticData::Instance().GetMarkUnknown();
|
bool markUnknown = StaticData::Instance().GetMarkUnknown();
|
||||||
|
|
||||||
stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
for (size_t pos = 0 ; pos < GetSize() ; pos++) {
|
for (size_t pos = 0 ; pos < GetSize() ; pos++) {
|
||||||
if (markUnknown && GetWord(pos).IsOOV()) {
|
if (markUnknown && GetWord(pos).IsOOV()) {
|
||||||
strme << StaticData::Instance().GetUnknownWordPrefix();
|
strme << StaticData::Instance().GetUnknownWordPrefix();
|
||||||
@ -128,7 +130,7 @@ std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return strme.str();
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
Word &Phrase::AddWord()
|
Word &Phrase::AddWord()
|
||||||
|
@ -88,9 +88,8 @@ void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float s
|
|||||||
{
|
{
|
||||||
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
|
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
|
||||||
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
|
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
|
||||||
std::stringstream name;
|
const std::string &name = i->first.name();
|
||||||
name << i->first;
|
if (starts_with(name, prefix))
|
||||||
if (starts_with(name.str(), prefix))
|
|
||||||
m_scores[i->first] = i->second * scalar;
|
m_scores[i->first] = i->second * scalar;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -101,9 +100,8 @@ size_t ScoreComponentCollection::GetNumberWeights(const FeatureFunction* sp)
|
|||||||
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
|
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
|
||||||
size_t weights = 0;
|
size_t weights = 0;
|
||||||
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
|
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
|
||||||
std::stringstream name;
|
const std::string &name = i->first.name();
|
||||||
name << i->first;
|
if (starts_with(name, prefix))
|
||||||
if (starts_with(name.str(), prefix))
|
|
||||||
weights++;
|
weights++;
|
||||||
}
|
}
|
||||||
return weights;
|
return weights;
|
||||||
|
@ -640,19 +640,21 @@ void StaticData::LoadDecodeGraphsOld(const vector<string> &mappingVector, const
|
|||||||
switch (decodeType) {
|
switch (decodeType) {
|
||||||
case Translate:
|
case Translate:
|
||||||
if(index>=pts.size()) {
|
if(index>=pts.size()) {
|
||||||
stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
strme << "No phrase dictionary with index "
|
strme << "No phrase dictionary with index "
|
||||||
<< index << " available!";
|
<< index << " available!";
|
||||||
UTIL_THROW(util::Exception, strme.str());
|
UTIL_THROW(util::Exception, str);
|
||||||
}
|
}
|
||||||
decodeStep = new DecodeStepTranslation(pts[index], prev, *featuresRemaining);
|
decodeStep = new DecodeStepTranslation(pts[index], prev, *featuresRemaining);
|
||||||
break;
|
break;
|
||||||
case Generate:
|
case Generate:
|
||||||
if(index>=gens.size()) {
|
if(index>=gens.size()) {
|
||||||
stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
strme << "No generation dictionary with index "
|
strme << "No generation dictionary with index "
|
||||||
<< index << " available!";
|
<< index << " available!";
|
||||||
UTIL_THROW(util::Exception, strme.str());
|
UTIL_THROW(util::Exception, str);
|
||||||
}
|
}
|
||||||
decodeStep = new DecodeStepGeneration(gens[index], prev, *featuresRemaining);
|
decodeStep = new DecodeStepGeneration(gens[index], prev, *featuresRemaining);
|
||||||
break;
|
break;
|
||||||
|
@ -4,6 +4,7 @@
|
|||||||
|
|
||||||
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
||||||
#include "moses/StaticData.h"
|
#include "moses/StaticData.h"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
@ -55,7 +56,8 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
|
|
||||||
TargetPhrase *targetPhrase = new TargetPhrase();
|
TargetPhrase *targetPhrase = new TargetPhrase();
|
||||||
|
|
||||||
std::ostringstream alignmentSS;
|
std::string alignmentSSStr;
|
||||||
|
util::StringStream alignmentSS(alignmentSSStr);
|
||||||
for (std::size_t i = 0; i < e.tail.size(); ++i) {
|
for (std::size_t i = 0; i < e.tail.size(); ++i) {
|
||||||
const Word &symbol = e.tail[i]->pvertex.symbol;
|
const Word &symbol = e.tail[i]->pvertex.symbol;
|
||||||
if (symbol.IsNonTerminal()) {
|
if (symbol.IsNonTerminal()) {
|
||||||
@ -75,7 +77,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
targetPhrase->EvaluateInIsolation(m_dummySourcePhrase);
|
targetPhrase->EvaluateInIsolation(m_dummySourcePhrase);
|
||||||
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
|
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
|
||||||
targetPhrase->SetTargetLHS(targetLhs);
|
targetPhrase->SetTargetLHS(targetLhs);
|
||||||
targetPhrase->SetAlignmentInfo(alignmentSS.str());
|
targetPhrase->SetAlignmentInfo(alignmentSSStr);
|
||||||
|
|
||||||
return targetPhrase;
|
return targetPhrase;
|
||||||
}
|
}
|
||||||
|
@ -47,7 +47,8 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
|
|
||||||
TargetPhrase *targetPhrase = new TargetPhrase();
|
TargetPhrase *targetPhrase = new TargetPhrase();
|
||||||
|
|
||||||
std::ostringstream alignmentSS;
|
std::string alignmentSSStr;
|
||||||
|
util::StringStream alignmentSS(alignmentSSStr);
|
||||||
for (std::size_t i = 0; i < node.children.size(); ++i) {
|
for (std::size_t i = 0; i < node.children.size(); ++i) {
|
||||||
const Word &symbol = node.children[i]->pvertex.symbol;
|
const Word &symbol = node.children[i]->pvertex.symbol;
|
||||||
if (symbol.IsNonTerminal()) {
|
if (symbol.IsNonTerminal()) {
|
||||||
@ -67,7 +68,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
targetPhrase->EvaluateInIsolation(sourceRhs);
|
targetPhrase->EvaluateInIsolation(sourceRhs);
|
||||||
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
|
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
|
||||||
targetPhrase->SetTargetLHS(targetLhs);
|
targetPhrase->SetTargetLHS(targetLhs);
|
||||||
targetPhrase->SetAlignmentInfo(alignmentSS.str());
|
targetPhrase->SetAlignmentInfo(alignmentSSStr);
|
||||||
|
|
||||||
return targetPhrase;
|
return targetPhrase;
|
||||||
}
|
}
|
||||||
|
@ -10,6 +10,7 @@
|
|||||||
#include "moses/FF/StatefulFeatureFunction.h"
|
#include "moses/FF/StatefulFeatureFunction.h"
|
||||||
#include "moses/FF/StatelessFeatureFunction.h"
|
#include "moses/FF/StatelessFeatureFunction.h"
|
||||||
#include "moses/LM/Base.h"
|
#include "moses/LM/Base.h"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
using namespace Moses;
|
using namespace Moses;
|
||||||
|
|
||||||
@ -40,8 +41,11 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo)
|
|||||||
if (doLMStats)
|
if (doLMStats)
|
||||||
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
|
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
|
||||||
for (; tpi != translationPath.end(); ++tpi) {
|
for (; tpi != translationPath.end(); ++tpi) {
|
||||||
std::ostringstream sms;
|
std::string smsStr;
|
||||||
std::ostringstream tms;
|
util::StringStream sms(smsStr);
|
||||||
|
|
||||||
|
std::string tmsStr;
|
||||||
|
util::StringStream tms(tmsStr);
|
||||||
std::string target = (*tpi)->GetTargetPhraseStringRep();
|
std::string target = (*tpi)->GetTargetPhraseStringRep();
|
||||||
std::string source = (*tpi)->GetSourcePhraseStringRep();
|
std::string source = (*tpi)->GetSourcePhraseStringRep();
|
||||||
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();
|
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();
|
||||||
@ -89,8 +93,8 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo)
|
|||||||
for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) {
|
for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) {
|
||||||
tms << '-' << swr_i;
|
tms << '-' << swr_i;
|
||||||
}
|
}
|
||||||
if (!epsilon) targetMap.push_back(sms.str());
|
if (!epsilon) targetMap.push_back(smsStr);
|
||||||
sourceMap.push_back(tms.str());
|
sourceMap.push_back(tmsStr);
|
||||||
}
|
}
|
||||||
std::vector<std::string>::iterator si = sourceMap.begin();
|
std::vector<std::string>::iterator si = sourceMap.begin();
|
||||||
std::vector<std::string>::iterator ti = targetMap.begin();
|
std::vector<std::string>::iterator ti = targetMap.begin();
|
||||||
|
@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "BlockHashIndex.h"
|
#include "BlockHashIndex.h"
|
||||||
#include "CmphStringVectorAdapter.h"
|
#include "CmphStringVectorAdapter.h"
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
#ifdef HAVE_CMPH
|
#ifdef HAVE_CMPH
|
||||||
#include "cmph.h"
|
#include "cmph.h"
|
||||||
@ -367,11 +368,12 @@ void BlockHashIndex::CalcHash(size_t current, void* source_void)
|
|||||||
|
|
||||||
if(lastKey > temp) {
|
if(lastKey > temp) {
|
||||||
if(source->nkeys != 2 || temp != "###DUMMY_KEY###") {
|
if(source->nkeys != 2 || temp != "###DUMMY_KEY###") {
|
||||||
std::stringstream strme;
|
std::string str;
|
||||||
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl;
|
util::StringStream strme(str);
|
||||||
strme << "1: " << lastKey << std::endl;
|
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
|
||||||
strme << "2: " << temp << std::endl;
|
strme << "1: " << lastKey << "\n";
|
||||||
UTIL_THROW2(strme.str());
|
strme << "2: " << temp << "\n";
|
||||||
|
UTIL_THROW2(str);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
lastKey = temp;
|
lastKey = temp;
|
||||||
|
@ -34,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "StringVector.h"
|
#include "StringVector.h"
|
||||||
#include "PackedArray.h"
|
#include "PackedArray.h"
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
#ifdef WITH_THREADS
|
#ifdef WITH_THREADS
|
||||||
#include "moses/ThreadPool.h"
|
#include "moses/ThreadPool.h"
|
||||||
@ -145,11 +146,12 @@ public:
|
|||||||
size_t current = m_landmarks.size();
|
size_t current = m_landmarks.size();
|
||||||
|
|
||||||
if(m_landmarks.size() && m_landmarks.back().str() >= keys[0]) {
|
if(m_landmarks.size() && m_landmarks.back().str() >= keys[0]) {
|
||||||
std::stringstream strme;
|
std::string str;
|
||||||
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort" << std::endl;
|
util::StringStream strme(str);
|
||||||
strme << "1: " << m_landmarks.back().str() << std::endl;
|
strme << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n";
|
||||||
strme << "2: " << keys[0] << std::endl;
|
strme << "1: " << m_landmarks.back().str() << "\n";
|
||||||
UTIL_THROW2(strme.str());
|
strme << "2: " << keys[0] << "\n";
|
||||||
|
UTIL_THROW2(str);
|
||||||
}
|
}
|
||||||
|
|
||||||
m_landmarks.push_back(keys[0]);
|
m_landmarks.push_back(keys[0]);
|
||||||
|
@ -17,6 +17,7 @@ License along with this library; if not, write to the Free Software
|
|||||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||||
***********************************************************************/
|
***********************************************************************/
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
#include "moses/TranslationModel/PhraseDictionaryMultiModel.h"
|
#include "moses/TranslationModel/PhraseDictionaryMultiModel.h"
|
||||||
|
|
||||||
@ -38,9 +39,10 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
|
|||||||
} else if (m_mode == "all" || m_mode == "all-restrict") {
|
} else if (m_mode == "all" || m_mode == "all-restrict") {
|
||||||
UTIL_THROW2("Implementation has moved: use PhraseDictionaryGroup with restrict=true/false");
|
UTIL_THROW2("Implementation has moved: use PhraseDictionaryGroup with restrict=true/false");
|
||||||
} else {
|
} else {
|
||||||
ostringstream msg;
|
string str;
|
||||||
|
util::StringStream msg(str);
|
||||||
msg << "combination mode unknown: " << m_mode;
|
msg << "combination mode unknown: " << m_mode;
|
||||||
throw runtime_error(msg.str());
|
throw runtime_error(str);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -210,9 +212,10 @@ std::vector<std::vector<float> > PhraseDictionaryMultiModel::getWeights(size_t n
|
|||||||
raw_weights.push_back(1.0/m_numModels); //uniform weights created online
|
raw_weights.push_back(1.0/m_numModels); //uniform weights created online
|
||||||
}
|
}
|
||||||
} else if(weights_ptr->size() != m_numModels && weights_ptr->size() != m_numModels * numWeights) {
|
} else if(weights_ptr->size() != m_numModels && weights_ptr->size() != m_numModels * numWeights) {
|
||||||
std::stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
strme << "Must have either one multimodel weight per model (" << m_numModels << "), or one per weighted feature and model (" << numWeights << "*" << m_numModels << "). You have " << weights_ptr->size() << ".";
|
strme << "Must have either one multimodel weight per model (" << m_numModels << "), or one per weighted feature and model (" << numWeights << "*" << m_numModels << "). You have " << weights_ptr->size() << ".";
|
||||||
UTIL_THROW(util::Exception, strme.str());
|
UTIL_THROW(util::Exception, str);
|
||||||
} else {
|
} else {
|
||||||
raw_weights = *weights_ptr;
|
raw_weights = *weights_ptr;
|
||||||
}
|
}
|
||||||
|
@ -18,6 +18,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
***********************************************************************/
|
***********************************************************************/
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
#include "util/tokenize.hh"
|
#include "util/tokenize.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h"
|
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
@ -56,9 +57,10 @@ void PhraseDictionaryMultiModelCounts::SetParameter(const std::string& key, cons
|
|||||||
else if (m_mode == "interpolate")
|
else if (m_mode == "interpolate")
|
||||||
m_combineFunction = LinearInterpolationFromCounts;
|
m_combineFunction = LinearInterpolationFromCounts;
|
||||||
else {
|
else {
|
||||||
ostringstream msg;
|
string str;
|
||||||
|
util::StringStream msg(str);
|
||||||
msg << "combination mode unknown: " << m_mode;
|
msg << "combination mode unknown: " << m_mode;
|
||||||
throw runtime_error(msg.str());
|
throw runtime_error(str);
|
||||||
}
|
}
|
||||||
} else if (key == "lex-e2f") {
|
} else if (key == "lex-e2f") {
|
||||||
m_lexE2FStr = Tokenize(value, ",");
|
m_lexE2FStr = Tokenize(value, ",");
|
||||||
|
@ -15,6 +15,7 @@
|
|||||||
#include "moses/PDTAimp.h"
|
#include "moses/PDTAimp.h"
|
||||||
#include "moses/TranslationTask.h"
|
#include "moses/TranslationTask.h"
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
@ -52,7 +53,8 @@ void PhraseDictionaryTreeAdaptor::InitializeForInput(ttasksptr const& ttask)
|
|||||||
|
|
||||||
vector<float> weight = staticData.GetWeights(this);
|
vector<float> weight = staticData.GetWeights(this);
|
||||||
if(m_numScoreComponents!=weight.size()) {
|
if(m_numScoreComponents!=weight.size()) {
|
||||||
std::stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size()
|
UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size()
|
||||||
<< " " << m_numScoreComponents);
|
<< " " << m_numScoreComponents);
|
||||||
}
|
}
|
||||||
|
@ -126,20 +126,22 @@ void ReformatHieroRule(const string &lineOrig, string &out)
|
|||||||
ReformatHieroRule(1, targetPhraseString, ntAlign);
|
ReformatHieroRule(1, targetPhraseString, ntAlign);
|
||||||
ReformateHieroScore(scoreString);
|
ReformateHieroScore(scoreString);
|
||||||
|
|
||||||
stringstream align;
|
std::string alignStr;
|
||||||
|
util::StringStream align(alignStr);
|
||||||
map<size_t, pair<size_t, size_t> >::const_iterator iterAlign;
|
map<size_t, pair<size_t, size_t> >::const_iterator iterAlign;
|
||||||
for (iterAlign = ntAlign.begin(); iterAlign != ntAlign.end(); ++iterAlign) {
|
for (iterAlign = ntAlign.begin(); iterAlign != ntAlign.end(); ++iterAlign) {
|
||||||
const pair<size_t, size_t> &alignPoint = iterAlign->second;
|
const pair<size_t, size_t> &alignPoint = iterAlign->second;
|
||||||
align << alignPoint.first << "-" << alignPoint.second << " ";
|
align << alignPoint.first << "-" << alignPoint.second << " ";
|
||||||
}
|
}
|
||||||
|
|
||||||
stringstream ret;
|
std::string str;
|
||||||
|
util::StringStream ret(str);
|
||||||
ret << sourcePhraseString << " ||| "
|
ret << sourcePhraseString << " ||| "
|
||||||
<< targetPhraseString << " ||| "
|
<< targetPhraseString << " ||| "
|
||||||
<< scoreString << " ||| "
|
<< scoreString << " ||| "
|
||||||
<< align.str();
|
<< alignStr;
|
||||||
|
|
||||||
out = ret.str();
|
out = str;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool RuleTableLoaderStandard::Load(FormatType format
|
bool RuleTableLoaderStandard::Load(FormatType format
|
||||||
|
@ -7,18 +7,20 @@
|
|||||||
//
|
//
|
||||||
|
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
|
#include "util/string_stream.hh"
|
||||||
#include "SentenceAlignment.h"
|
#include "SentenceAlignment.h"
|
||||||
|
|
||||||
namespace tmmt
|
namespace tmmt
|
||||||
{
|
{
|
||||||
std::string SentenceAlignment::getTargetString(const Vocabulary &vocab) const
|
std::string SentenceAlignment::getTargetString(const Vocabulary &vocab) const
|
||||||
{
|
{
|
||||||
std::stringstream strme;
|
std::string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
for (size_t i = 0; i < target.size(); ++i) {
|
for (size_t i = 0; i < target.size(); ++i) {
|
||||||
const WORD &word = vocab.GetWord(target[i]);
|
const WORD &word = vocab.GetWord(target[i]);
|
||||||
strme << word << " ";
|
strme << word << " ";
|
||||||
}
|
}
|
||||||
return strme.str();
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include "Vocabulary.h"
|
#include "Vocabulary.h"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
|
|
||||||
namespace tmmt
|
namespace tmmt
|
||||||
{
|
{
|
||||||
@ -27,12 +28,13 @@ struct SentenceAlignment {
|
|||||||
std::string getTargetString(const Vocabulary &vocab) const;
|
std::string getTargetString(const Vocabulary &vocab) const;
|
||||||
|
|
||||||
std::string getAlignmentString() const {
|
std::string getAlignmentString() const {
|
||||||
std::stringstream strme;
|
std::string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
for (size_t i = 0; i < alignment.size(); ++i) {
|
for (size_t i = 0; i < alignment.size(); ++i) {
|
||||||
const std::pair<int,int> &alignPair = alignment[i];
|
const std::pair<int,int> &alignPair = alignment[i];
|
||||||
strme << alignPair.first << "-" << alignPair.second << " ";
|
strme << alignPair.first << "-" << alignPair.second << " ";
|
||||||
}
|
}
|
||||||
return strme.str();
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "FactorCollection.h"
|
#include "FactorCollection.h"
|
||||||
#include "StaticData.h" // needed to determine the FactorDelimiter
|
#include "StaticData.h" // needed to determine the FactorDelimiter
|
||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
|
#include "util/string_stream.hh"
|
||||||
#include "util/tokenize_piece.hh"
|
#include "util/tokenize_piece.hh"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
@ -79,7 +80,8 @@ void Word::Merge(const Word &sourceWord)
|
|||||||
|
|
||||||
std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlank) const
|
std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlank) const
|
||||||
{
|
{
|
||||||
stringstream strme;
|
string str;
|
||||||
|
util::StringStream strme(str);
|
||||||
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
|
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
|
||||||
bool firstPass = true;
|
bool firstPass = true;
|
||||||
unsigned int stop = min(max_fax(),factorType.size());
|
unsigned int stop = min(max_fax(),factorType.size());
|
||||||
@ -99,7 +101,7 @@ std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlan
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(endWithBlank) strme << " ";
|
if(endWithBlank) strme << " ";
|
||||||
return strme.str();
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
StringPiece Word::GetString(FactorType factorType) const
|
StringPiece Word::GetString(FactorType factorType) const
|
||||||
|
Loading…
Reference in New Issue
Block a user