2013-05-29 21:16:15 +04:00
|
|
|
// $Id$
|
|
|
|
// vim:tabstop=2
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2006 University of Edinburgh
|
|
|
|
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
2012-08-03 18:38:45 +04:00
|
|
|
|
2014-01-13 22:32:22 +04:00
|
|
|
#include <sstream>
|
2012-08-02 20:32:55 +04:00
|
|
|
#include "LexicalReorderingTableCreator.h"
|
2012-08-06 22:07:43 +04:00
|
|
|
#include "ThrowingFwrite.h"
|
2012-11-12 23:56:18 +04:00
|
|
|
#include "moses/Util.h"
|
2013-01-23 00:11:02 +04:00
|
|
|
#include "util/file.hh"
|
2014-01-13 18:37:05 +04:00
|
|
|
#include "util/exception.hh"
|
2013-01-23 00:11:02 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
2012-08-02 20:32:55 +04:00
|
|
|
|
|
|
|
LexicalReorderingTableCreator::LexicalReorderingTableCreator(
|
2013-01-23 00:11:02 +04:00
|
|
|
std::string inPath, std::string outPath, std::string tempfilePath,
|
2012-08-02 20:32:55 +04:00
|
|
|
size_t orderBits, size_t fingerPrintBits, bool multipleScoreTrees,
|
|
|
|
size_t quantize
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
, size_t threads
|
|
|
|
#endif
|
2013-05-29 21:16:15 +04:00
|
|
|
)
|
2013-01-23 00:11:02 +04:00
|
|
|
: m_inPath(inPath), m_outPath(outPath), m_tempfilePath(tempfilePath),
|
2013-05-29 21:16:15 +04:00
|
|
|
m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits),
|
|
|
|
m_numScoreComponent(0), m_multipleScoreTrees(multipleScoreTrees),
|
|
|
|
m_quantize(quantize), m_separator(" ||| "),
|
|
|
|
m_hash(m_orderBits, m_fingerPrintBits), m_lastFlushedLine(-1)
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
, m_threads(threads)
|
2012-08-02 20:32:55 +04:00
|
|
|
#endif
|
2013-05-29 21:16:15 +04:00
|
|
|
{
|
2012-08-02 20:32:55 +04:00
|
|
|
PrintInfo();
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
m_outFile = std::fopen(m_outPath.c_str(), "w");
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "Pass 1/2: Creating phrase index + Counting scores" << std::endl;
|
2013-05-29 21:16:15 +04:00
|
|
|
m_hash.BeginSave(m_outFile);
|
2013-01-23 00:11:02 +04:00
|
|
|
|
|
|
|
|
|
|
|
if(tempfilePath.size()) {
|
|
|
|
MmapAllocator<unsigned char> allocEncoded(util::FMakeTemp(tempfilePath));
|
|
|
|
m_encodedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>(allocEncoded);
|
2013-05-29 21:16:15 +04:00
|
|
|
} else {
|
2013-01-23 00:11:02 +04:00
|
|
|
m_encodedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>();
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
EncodeScores();
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "Intermezzo: Calculating Huffman code sets" << std::endl;
|
|
|
|
CalcHuffmanCodes();
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "Pass 2/2: Compressing scores" << std::endl;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
|
|
|
|
if(tempfilePath.size()) {
|
2013-01-23 00:11:02 +04:00
|
|
|
MmapAllocator<unsigned char> allocCompressed(util::FMakeTemp(tempfilePath));
|
|
|
|
m_compressedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>(allocCompressed);
|
2013-05-29 21:16:15 +04:00
|
|
|
} else {
|
2013-01-23 00:11:02 +04:00
|
|
|
m_compressedScores = new StringVector<unsigned char, unsigned long, MmapAllocator>();
|
|
|
|
}
|
2012-08-02 20:32:55 +04:00
|
|
|
CompressScores();
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "Saving to " << m_outPath << std::endl;
|
|
|
|
Save();
|
|
|
|
std::cerr << "Done" << std::endl;
|
|
|
|
std::fclose(m_outFile);
|
|
|
|
}
|
|
|
|
|
|
|
|
void LexicalReorderingTableCreator::PrintInfo()
|
2013-05-29 21:16:15 +04:00
|
|
|
{
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "Used options:" << std::endl;
|
|
|
|
std::cerr << "\tText reordering table will be read from: " << m_inPath << std::endl;
|
2012-08-14 18:43:24 +04:00
|
|
|
std::cerr << "\tOutput reordering table will be written to: " << m_outPath << std::endl;
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "\tStep size for source landmark phrases: 2^" << m_orderBits << "=" << (1ul << m_orderBits) << std::endl;
|
|
|
|
std::cerr << "\tPhrase fingerprint size: " << m_fingerPrintBits << " bits / P(fp)=" << (float(1)/(1ul << m_fingerPrintBits)) << std::endl;
|
2013-05-29 21:16:15 +04:00
|
|
|
std::cerr << "\tSingle Huffman code set for score components: " << (m_multipleScoreTrees ? "no" : "yes") << std::endl;
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "\tUsing score quantization: ";
|
|
|
|
if(m_quantize)
|
|
|
|
std::cerr << m_quantize << " best" << std::endl;
|
|
|
|
else
|
|
|
|
std::cerr << "no" << std::endl;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
#ifdef WITH_THREADS
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "\tRunning with " << m_threads << " threads" << std::endl;
|
|
|
|
#endif
|
|
|
|
std::cerr << std::endl;
|
|
|
|
}
|
|
|
|
|
2012-08-13 17:35:31 +04:00
|
|
|
LexicalReorderingTableCreator::~LexicalReorderingTableCreator()
|
|
|
|
{
|
|
|
|
for(size_t i = 0; i < m_scoreTrees.size(); i++) {
|
|
|
|
delete m_scoreTrees[i];
|
|
|
|
delete m_scoreCounters[i];
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2013-01-23 00:11:02 +04:00
|
|
|
delete m_encodedScores;
|
|
|
|
delete m_compressedScores;
|
2012-08-13 17:35:31 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
void LexicalReorderingTableCreator::EncodeScores()
|
|
|
|
{
|
|
|
|
InputFileStream inFile(m_inPath);
|
|
|
|
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::thread_group threads;
|
2013-05-29 21:16:15 +04:00
|
|
|
for (size_t i = 0; i < m_threads; ++i) {
|
|
|
|
EncodingTaskReordering* et = new EncodingTaskReordering(inFile, *this);
|
2012-08-02 20:32:55 +04:00
|
|
|
threads.create_thread(*et);
|
|
|
|
}
|
|
|
|
threads.join_all();
|
|
|
|
#else
|
|
|
|
EncodingTaskReordering* et = new EncodingTaskReordering(inFile, *this);
|
|
|
|
(*et)();
|
|
|
|
delete et;
|
|
|
|
#endif
|
|
|
|
FlushEncodedQueue(true);
|
|
|
|
}
|
|
|
|
|
|
|
|
void LexicalReorderingTableCreator::CalcHuffmanCodes()
|
2013-05-29 21:16:15 +04:00
|
|
|
{
|
2012-08-02 20:32:55 +04:00
|
|
|
std::vector<ScoreTree*>::iterator treeIt = m_scoreTrees.begin();
|
|
|
|
for(std::vector<ScoreCounter*>::iterator it = m_scoreCounters.begin();
|
2013-05-29 21:16:15 +04:00
|
|
|
it != m_scoreCounters.end(); it++) {
|
2012-08-02 20:32:55 +04:00
|
|
|
if(m_quantize)
|
2013-05-29 21:16:15 +04:00
|
|
|
(*it)->Quantize(m_quantize);
|
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::cerr << "\tCreating Huffman codes for " << (*it)->Size()
|
2013-05-29 21:16:15 +04:00
|
|
|
<< " scores" << std::endl;
|
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
*treeIt = new ScoreTree((*it)->Begin(), (*it)->End());
|
|
|
|
treeIt++;
|
|
|
|
}
|
|
|
|
std::cerr << std::endl;
|
|
|
|
}
|
|
|
|
|
|
|
|
void LexicalReorderingTableCreator::CompressScores()
|
|
|
|
{
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::thread_group threads;
|
|
|
|
for (size_t i = 0; i < m_threads; ++i) {
|
2013-05-29 21:16:15 +04:00
|
|
|
CompressionTaskReordering* ct = new CompressionTaskReordering(*m_encodedScores, *this);
|
2012-08-02 20:32:55 +04:00
|
|
|
threads.create_thread(*ct);
|
|
|
|
}
|
|
|
|
threads.join_all();
|
|
|
|
#else
|
2013-01-23 00:11:02 +04:00
|
|
|
CompressionTaskReordering* ct = new CompressionTaskReordering(*m_encodedScores, *this);
|
2012-08-02 20:32:55 +04:00
|
|
|
(*ct)();
|
|
|
|
delete ct;
|
|
|
|
#endif
|
|
|
|
FlushCompressedQueue(true);
|
|
|
|
}
|
|
|
|
|
|
|
|
void LexicalReorderingTableCreator::Save()
|
2013-05-29 21:16:15 +04:00
|
|
|
{
|
2012-08-06 22:07:43 +04:00
|
|
|
ThrowingFwrite(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, m_outFile);
|
|
|
|
ThrowingFwrite(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, m_outFile);
|
2012-08-02 20:32:55 +04:00
|
|
|
for(size_t i = 0; i < m_scoreTrees.size(); i++)
|
|
|
|
m_scoreTrees[i]->Save(m_outFile);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2013-01-23 00:11:02 +04:00
|
|
|
m_compressedScores->save(m_outFile);
|
2012-08-02 20:32:55 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
std::string LexicalReorderingTableCreator::MakeSourceTargetKey(std::string &source, std::string &target)
|
|
|
|
{
|
2012-08-23 12:26:22 +04:00
|
|
|
std::string key = source + m_separator;
|
|
|
|
if(!target.empty())
|
|
|
|
key += target + m_separator;
|
|
|
|
return key;
|
2012-08-02 20:32:55 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
std::string LexicalReorderingTableCreator::EncodeLine(std::vector<std::string>& tokens)
|
|
|
|
{
|
2012-08-23 12:26:22 +04:00
|
|
|
std::string scoresString = tokens.back();
|
2012-08-02 20:32:55 +04:00
|
|
|
std::stringstream scoresStream;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::vector<float> scores;
|
|
|
|
Tokenize<float>(scores, scoresString);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-23 12:26:22 +04:00
|
|
|
if(!m_numScoreComponent) {
|
|
|
|
m_numScoreComponent = scores.size();
|
|
|
|
m_scoreCounters.resize(m_multipleScoreTrees ? m_numScoreComponent : 1);
|
|
|
|
for(std::vector<ScoreCounter*>::iterator it = m_scoreCounters.begin();
|
|
|
|
it != m_scoreCounters.end(); it++)
|
2013-05-29 21:16:15 +04:00
|
|
|
*it = new ScoreCounter();
|
2012-08-23 12:26:22 +04:00
|
|
|
m_scoreTrees.resize(m_multipleScoreTrees ? m_numScoreComponent : 1);
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-23 12:26:22 +04:00
|
|
|
if(m_numScoreComponent != scores.size()) {
|
2014-01-13 22:32:22 +04:00
|
|
|
std::stringstream strme;
|
|
|
|
strme << "Error: Wrong number of scores detected ("
|
2014-01-15 19:49:57 +04:00
|
|
|
<< scores.size() << " != " << m_numScoreComponent << ") :" << std::endl;
|
2014-01-13 22:32:22 +04:00
|
|
|
strme << "Line: " << tokens[0] << " ||| ... ||| " << scoresString << std::endl;
|
|
|
|
UTIL_THROW2(strme.str());
|
2012-08-23 12:26:22 +04:00
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
size_t c = 0;
|
|
|
|
float score;
|
2013-05-29 21:16:15 +04:00
|
|
|
while(c < m_numScoreComponent) {
|
2012-08-02 20:32:55 +04:00
|
|
|
score = scores[c];
|
|
|
|
score = FloorScore(TransformScore(score));
|
|
|
|
scoresStream.write((char*)&score, sizeof(score));
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
m_scoreCounters[m_multipleScoreTrees ? c : 0]->Increase(score);
|
|
|
|
c++;
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
return scoresStream.str();
|
|
|
|
}
|
|
|
|
|
|
|
|
void LexicalReorderingTableCreator::AddEncodedLine(PackedItem& pi)
|
|
|
|
{
|
|
|
|
m_queue.push(pi);
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
void LexicalReorderingTableCreator::FlushEncodedQueue(bool force)
|
|
|
|
{
|
|
|
|
if(force || m_queue.size() > 10000) {
|
|
|
|
while(!m_queue.empty() && m_lastFlushedLine + 1 == m_queue.top().GetLine()) {
|
2012-08-02 20:32:55 +04:00
|
|
|
PackedItem pi = m_queue.top();
|
|
|
|
m_queue.pop();
|
|
|
|
m_lastFlushedLine++;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
m_lastRange.push_back(pi.GetSrc());
|
2013-01-23 00:11:02 +04:00
|
|
|
m_encodedScores->push_back(pi.GetTrg());
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
if((pi.GetLine()+1) % 100000 == 0)
|
2013-05-29 21:16:15 +04:00
|
|
|
std::cerr << ".";
|
2012-08-02 20:32:55 +04:00
|
|
|
if((pi.GetLine()+1) % 5000000 == 0)
|
2013-05-29 21:16:15 +04:00
|
|
|
std::cerr << "[" << (pi.GetLine()+1) << "]" << std::endl;
|
|
|
|
|
|
|
|
if(m_lastRange.size() == (1ul << m_orderBits)) {
|
2012-08-02 20:32:55 +04:00
|
|
|
m_hash.AddRange(m_lastRange);
|
|
|
|
m_hash.SaveLastRange();
|
|
|
|
m_hash.DropLastRange();
|
|
|
|
m_lastRange.clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
if(force) {
|
2012-08-02 20:32:55 +04:00
|
|
|
m_lastFlushedLine = -1;
|
|
|
|
|
2015-01-23 00:18:19 +03:00
|
|
|
if(!m_lastRange.empty()) {
|
|
|
|
m_hash.AddRange(m_lastRange);
|
|
|
|
m_lastRange.clear();
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
#ifdef WITH_THREADS
|
|
|
|
m_hash.WaitAll();
|
|
|
|
#endif
|
|
|
|
|
|
|
|
m_hash.SaveLastRange();
|
|
|
|
m_hash.DropLastRange();
|
|
|
|
m_hash.FinalizeSave();
|
|
|
|
|
|
|
|
std::cerr << std::endl << std::endl;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
std::string LexicalReorderingTableCreator::CompressEncodedScores(std::string &encodedScores)
|
|
|
|
{
|
2012-08-02 20:32:55 +04:00
|
|
|
std::stringstream encodedScoresStream(encodedScores);
|
|
|
|
encodedScoresStream.unsetf(std::ios::skipws);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::string compressedScores;
|
2012-08-11 22:10:58 +04:00
|
|
|
BitWrapper<> compressedScoresStream(compressedScores);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
size_t currScore = 0;
|
|
|
|
float score;
|
|
|
|
encodedScoresStream.read((char*) &score, sizeof(score));
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
while(encodedScoresStream) {
|
|
|
|
size_t index = currScore % m_scoreTrees.size();
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
if(m_quantize)
|
|
|
|
score = m_scoreCounters[index]->LowerBound(score);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-11 22:10:58 +04:00
|
|
|
m_scoreTrees[index]->Put(compressedScoresStream, score);
|
2012-08-02 20:32:55 +04:00
|
|
|
encodedScoresStream.read((char*) &score, sizeof(score));
|
|
|
|
currScore++;
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
return compressedScores;
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
void LexicalReorderingTableCreator::AddCompressedScores(PackedItem& pi)
|
|
|
|
{
|
|
|
|
m_queue.push(pi);
|
2012-08-02 20:32:55 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void LexicalReorderingTableCreator::FlushCompressedQueue(bool force)
|
2013-05-29 21:16:15 +04:00
|
|
|
{
|
|
|
|
if(force || m_queue.size() > 10000) {
|
|
|
|
while(!m_queue.empty() && m_lastFlushedLine + 1 == m_queue.top().GetLine()) {
|
2012-08-02 20:32:55 +04:00
|
|
|
PackedItem pi = m_queue.top();
|
|
|
|
m_queue.pop();
|
|
|
|
m_lastFlushedLine++;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2013-01-23 00:11:02 +04:00
|
|
|
m_compressedScores->push_back(pi.GetTrg());
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
if((pi.GetLine()+1) % 100000 == 0)
|
2013-05-29 21:16:15 +04:00
|
|
|
std::cerr << ".";
|
2012-08-02 20:32:55 +04:00
|
|
|
if((pi.GetLine()+1) % 5000000 == 0)
|
2013-05-29 21:16:15 +04:00
|
|
|
std::cerr << "[" << (pi.GetLine()+1) << "]" << std::endl;
|
2012-08-02 20:32:55 +04:00
|
|
|
}
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
if(force) {
|
2012-08-02 20:32:55 +04:00
|
|
|
m_lastFlushedLine = -1;
|
|
|
|
std::cerr << std::endl << std::endl;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
//****************************************************************************//
|
|
|
|
|
|
|
|
size_t EncodingTaskReordering::m_lineNum = 0;
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::mutex EncodingTaskReordering::m_mutex;
|
|
|
|
boost::mutex EncodingTaskReordering::m_fileMutex;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
EncodingTaskReordering::EncodingTaskReordering(InputFileStream& inFile, LexicalReorderingTableCreator& creator)
|
|
|
|
: m_inFile(inFile), m_creator(creator) {}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
void EncodingTaskReordering::operator()()
|
|
|
|
{
|
|
|
|
size_t lineNum = 0;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::vector<std::string> lines;
|
|
|
|
size_t max_lines = 1000;
|
|
|
|
lines.reserve(max_lines);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
{
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::mutex::scoped_lock lock(m_fileMutex);
|
|
|
|
#endif
|
|
|
|
std::string line;
|
|
|
|
while(lines.size() < max_lines && std::getline(m_inFile, line))
|
2013-05-29 21:16:15 +04:00
|
|
|
lines.push_back(line);
|
2012-08-02 20:32:55 +04:00
|
|
|
lineNum = m_lineNum;
|
|
|
|
m_lineNum += lines.size();
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::vector<PackedItem> result;
|
|
|
|
result.reserve(max_lines);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
while(lines.size()) {
|
|
|
|
for(size_t i = 0; i < lines.size(); i++) {
|
2012-08-02 20:32:55 +04:00
|
|
|
std::vector<std::string> tokens;
|
|
|
|
Moses::TokenizeMultiCharSeparator(tokens, lines[i], m_creator.m_separator);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
std::string encodedLine = m_creator.EncodeLine(tokens);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-23 12:26:22 +04:00
|
|
|
std::string f = tokens[0];
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-23 12:26:22 +04:00
|
|
|
std::string e;
|
|
|
|
if(tokens.size() > 2)
|
|
|
|
e = tokens[1];
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-23 12:26:22 +04:00
|
|
|
PackedItem packedItem(lineNum + i, m_creator.MakeSourceTargetKey(f, e),
|
2012-08-02 20:32:55 +04:00
|
|
|
encodedLine, i);
|
|
|
|
result.push_back(packedItem);
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
{
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::mutex::scoped_lock lock(m_mutex);
|
|
|
|
#endif
|
2013-05-29 21:16:15 +04:00
|
|
|
for(size_t i = 0; i < result.size(); i++)
|
2012-08-02 20:32:55 +04:00
|
|
|
m_creator.AddEncodedLine(result[i]);
|
2013-05-29 21:16:15 +04:00
|
|
|
m_creator.FlushEncodedQueue();
|
2012-08-02 20:32:55 +04:00
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2015-01-23 00:18:19 +03:00
|
|
|
lines.clear();
|
2012-08-02 20:32:55 +04:00
|
|
|
result.clear();
|
|
|
|
lines.reserve(max_lines);
|
|
|
|
result.reserve(max_lines);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::mutex::scoped_lock lock(m_fileMutex);
|
|
|
|
#endif
|
|
|
|
std::string line;
|
|
|
|
while(lines.size() < max_lines && std::getline(m_inFile, line))
|
|
|
|
lines.push_back(line);
|
|
|
|
lineNum = m_lineNum;
|
|
|
|
m_lineNum += lines.size();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
//****************************************************************************//
|
|
|
|
|
|
|
|
size_t CompressionTaskReordering::m_scoresNum = 0;
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::mutex CompressionTaskReordering::m_mutex;
|
|
|
|
#endif
|
|
|
|
|
2012-11-17 03:46:22 +04:00
|
|
|
CompressionTaskReordering::CompressionTaskReordering(StringVector<unsigned char, unsigned long,
|
2013-05-29 21:16:15 +04:00
|
|
|
MmapAllocator>& encodedScores,
|
|
|
|
LexicalReorderingTableCreator& creator)
|
2012-08-02 20:32:55 +04:00
|
|
|
: m_encodedScores(encodedScores), m_creator(creator)
|
|
|
|
{ }
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-02 20:32:55 +04:00
|
|
|
void CompressionTaskReordering::operator()()
|
|
|
|
{
|
|
|
|
size_t scoresNum;
|
|
|
|
{
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::mutex::scoped_lock lock(m_mutex);
|
|
|
|
#endif
|
|
|
|
scoresNum = m_scoresNum;
|
|
|
|
m_scoresNum++;
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
while(scoresNum < m_encodedScores.size()) {
|
2012-08-02 20:32:55 +04:00
|
|
|
std::string scores = m_encodedScores[scoresNum];
|
|
|
|
std::string compressedScores
|
2015-01-14 22:21:11 +03:00
|
|
|
= m_creator.CompressEncodedScores(scores);
|
2012-08-02 20:32:55 +04:00
|
|
|
|
|
|
|
std::string dummy;
|
|
|
|
PackedItem packedItem(scoresNum, dummy, compressedScores, 0);
|
|
|
|
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
boost::mutex::scoped_lock lock(m_mutex);
|
|
|
|
#endif
|
|
|
|
m_creator.AddCompressedScores(packedItem);
|
|
|
|
m_creator.FlushCompressedQueue();
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
scoresNum = m_scoresNum;
|
|
|
|
m_scoresNum++;
|
2012-08-02 20:32:55 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|