moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 22:14:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Hieu Hoang 2012-08-06 21:42:08 +01:00
commit ffb445c283
13 changed files with 115 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
misc/processLexicalTableMin.cpp
View File

@ -1,14 +1,10 @@
#include <iostream>
#include <string>
#include "Timer.h"
#include "InputFileStream.h"
#include "CompactPT/LexicalReorderingTableCreator.h"
using namespace Moses;
Timer timer;
void printHelp(char **argv)
{
std::cerr << "Usage " << argv[0] << ":\n"
@ -35,7 +31,10 @@ void printHelp(char **argv)
" volume = { 98 },\n"
" year = { 2012 },\n"
" note = { Proceedings of the MT Marathon 2012, Edinburgh },\n"
" }\n\n";
" }\n\n"
" Acknowledgments: Part of this research was carried out at and funded by\n"
" the World Intellectual Property Organization (WIPO) in Geneva.\n\n";
}
int main(int argc, char** argv)

5
misc/processPhraseTableMin.cpp
View File

@ -1,7 +1,6 @@
#include <iostream>
#include "TypeDef.h"
#include "CompactPT/PhraseTableCreator.h"
#include "CompactPT/CanonicalHuffman.h"
using namespace Moses;
@ -34,7 +33,9 @@ void printHelp(char **argv) {
" volume = { 98 },\n"
" year = { 2012 },\n"
" note = { Proceedings of the MT Marathon 2012, Edinburgh },\n"
" }\n\n";
" }\n\n"
" Acknowledgments: Part of this research was carried out at and funded by\n"
" the World Intellectual Property Organization (WIPO) in Geneva.\n\n";
}

15
moses/src/CompactPT/BlockHashIndex.cpp
View File

@ -19,6 +19,7 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "ThrowingFwrite.h"
#include "BlockHashIndex.h"
#include "CmphStringVectorAdapter.h"
@ -138,13 +139,13 @@ size_t BlockHashIndex::Save(std::string filename)
void BlockHashIndex::BeginSave(std::FILE * mphf)
{
m_fileHandle = mphf;
std::fwrite(&m_orderBits, sizeof(size_t), 1, m_fileHandle);
std::fwrite(&m_fingerPrintBits, sizeof(size_t), 1, m_fileHandle);
ThrowingFwrite(&m_orderBits, sizeof(size_t), 1, m_fileHandle);
ThrowingFwrite(&m_fingerPrintBits, sizeof(size_t), 1, m_fileHandle);
m_fileHandleStart = std::ftell(m_fileHandle);
size_t relIndexPos = 0;
std::fwrite(&relIndexPos, sizeof(size_t), 1, m_fileHandle);
ThrowingFwrite(&relIndexPos, sizeof(size_t), 1, m_fileHandle);
}
void BlockHashIndex::SaveRange(size_t i)
@ -219,16 +220,16 @@ size_t BlockHashIndex::FinalizeSave()
size_t relIndexPos = std::ftell(m_fileHandle) - m_fileHandleStart;
std::fseek(m_fileHandle, m_fileHandleStart, SEEK_SET);
std::fwrite(&relIndexPos, sizeof(size_t), 1, m_fileHandle);
ThrowingFwrite(&relIndexPos, sizeof(size_t), 1, m_fileHandle);
std::fseek(m_fileHandle, m_fileHandleStart + relIndexPos, SEEK_SET);
m_landmarks.save(m_fileHandle);
size_t seekIndexSize = m_seekIndex.size();
std::fwrite(&seekIndexSize, sizeof(size_t), 1, m_fileHandle);
std::fwrite(&m_seekIndex[0], sizeof(size_t), seekIndexSize, m_fileHandle);
ThrowingFwrite(&seekIndexSize, sizeof(size_t), 1, m_fileHandle);
ThrowingFwrite(&m_seekIndex[0], sizeof(size_t), seekIndexSize, m_fileHandle);
std::fwrite(&m_size, sizeof(size_t), 1, m_fileHandle);
ThrowingFwrite(&m_size, sizeof(size_t), 1, m_fileHandle);
size_t fileHandleStop = std::ftell(m_fileHandle);
return fileHandleStop - m_fileHandleStart + sizeof(m_orderBits)

14
moses/src/CompactPT/CanonicalHuffman.h
View File

@ -27,6 +27,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <boost/dynamic_bitset.hpp>
#include <boost/unordered_map.hpp>
#include "ThrowingFwrite.h"
namespace Moses {
template<typename PosType, typename DataType> class Hufftree;
@ -240,16 +242,16 @@ class CanonicalHuffman
size_t start = std::ftell(pFile);
size_t size = m_symbols.size();
std::fwrite(&size, sizeof(size_t), 1, pFile);
std::fwrite(&m_symbols[0], sizeof(Data), size, pFile);
ThrowingFwrite(&size, sizeof(size_t), 1, pFile);
ThrowingFwrite(&m_symbols[0], sizeof(Data), size, pFile);
size = m_firstCodes.size();
std::fwrite(&size, sizeof(size_t), 1, pFile);
std::fwrite(&m_firstCodes[0], sizeof(Code), size, pFile);
ThrowingFwrite(&size, sizeof(size_t), 1, pFile);
ThrowingFwrite(&m_firstCodes[0], sizeof(Code), size, pFile);
size = m_lengthIndex.size();
std::fwrite(&size, sizeof(size_t), 1, pFile);
std::fwrite(&m_lengthIndex[0], sizeof(size_t), size, pFile);
ThrowingFwrite(&size, sizeof(size_t), 1, pFile);
ThrowingFwrite(&m_lengthIndex[0], sizeof(size_t), size, pFile);
return std::ftell(pFile) - start;
}

10
moses/src/CompactPT/ConsistantPhrases.h
View File

@ -40,15 +40,6 @@ class ConsistantPhrases
{
bool operator()(Phrase a, Phrase b)
{
if(a.j < b.j)
return true;
if(a.j == b.j && a.n > b.n)
return true;
if(a.j == b.j && a.n == b.n && a.i < b.i)
return true;
if(a.j == b.j && a.n == b.n && a.i == b.i && a.m > b.m)
return true;
/*
if(a.n > b.n)
return true;
if(a.n == b.n && a.j < b.j)
@ -57,7 +48,6 @@ class ConsistantPhrases
return true;
if(a.n == b.n && a.j == b.j && a.m == b.m && a.i < b.i)
return true;
*/
return false;
}
};

4
moses/src/CompactPT/Jamfile
View File

@ -10,9 +10,7 @@ else {
alias cmph ;
}
alias sources : BlockHashIndex.cpp CmphStringVectorAdapter.cpp LexicalReorderingTableCompact.cpp
LexicalReorderingTableCreator.cpp MurmurHash3.cpp PhraseDecoder.cpp
PhraseDictionaryCompact.cpp PhraseTableCreator.cpp ;
alias sources : [ glob *.cpp ] ;
lib CompactPT : sources ..//moses_internal cmph : $(includes) ;

5
moses/src/CompactPT/LexicalReorderingTableCreator.cpp
View File

@ -20,6 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "LexicalReorderingTableCreator.h"
#include "ThrowingFwrite.h"
namespace Moses {
@ -145,8 +146,8 @@ void LexicalReorderingTableCreator::CompressScores()
void LexicalReorderingTableCreator::Save()
{
std::fwrite(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, m_outFile);
std::fwrite(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, m_outFile);
ThrowingFwrite(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, m_outFile);
ThrowingFwrite(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, m_outFile);
for(size_t i = 0; i < m_scoreTrees.size(); i++)
m_scoreTrees[i]->Save(m_outFile);

15
moses/src/CompactPT/MonotonicVector.h
View File

@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <cstdio>
#include <cassert>
#include "ThrowingFwrite.h"
#include "ListCoders.h"
#include "MmapAllocator.h"
@ -219,17 +220,17 @@ class MonotonicVector
commit();
bool byteSize = 0;
byteSize += fwrite(&m_final, sizeof(bool), 1, out) * sizeof(bool);
byteSize += fwrite(&m_size, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += fwrite(&m_last, sizeof(PosT), 1, out) * sizeof(PosT);
byteSize += ThrowingFwrite(&m_final, sizeof(bool), 1, out) * sizeof(bool);
byteSize += ThrowingFwrite(&m_size, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += ThrowingFwrite(&m_last, sizeof(PosT), 1, out) * sizeof(PosT);
size_t size = m_diffs.size();
byteSize += fwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += fwrite(&m_diffs[0], sizeof(unsigned int), size, out) * sizeof(unsigned int);
byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += ThrowingFwrite(&m_diffs[0], sizeof(unsigned int), size, out) * sizeof(unsigned int);
size = m_anchors.size();
byteSize += fwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += fwrite(&m_anchors[0], sizeof(NumT), size, out) * sizeof(NumT);
byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += ThrowingFwrite(&m_anchors[0], sizeof(NumT), size, out) * sizeof(NumT);
return byteSize;
}

8
moses/src/CompactPT/PackedArray.h
View File

@ -27,6 +27,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <cstring>
#include <cstdio>
#include "ThrowingFwrite.h"
namespace Moses
{
@ -147,9 +149,9 @@ class PackedArray
{
size_t a1 = std::ftell(out);
std::fwrite(&m_size, sizeof(m_size), 1, out);
std::fwrite(&m_storageSize, sizeof(m_storageSize), 1, out);
std::fwrite(m_storage, sizeof(D), m_storageSize, out);
ThrowingFwrite(&m_size, sizeof(m_size), 1, out);
ThrowingFwrite(&m_storageSize, sizeof(m_storageSize), 1, out);
ThrowingFwrite(m_storage, sizeof(D), m_storageSize, out);
size_t a2 = std::ftell(out);
return a2 - a1;

21
moses/src/CompactPT/PhraseTableCreator.cpp
View File

@ -22,6 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <cstdio>
#include "PhraseTableCreator.h"
#include "ThrowingFwrite.h"
namespace Moses
{
@ -153,11 +154,11 @@ void PhraseTableCreator::PrintInfo()
void PhraseTableCreator::Save()
{
// Save type of encoding
std::fwrite(&m_coding, sizeof(m_coding), 1, m_outFile);
std::fwrite(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, m_outFile);
std::fwrite(&m_useAlignmentInfo, sizeof(m_useAlignmentInfo), 1, m_outFile);
std::fwrite(&m_maxRank, sizeof(m_maxRank), 1, m_outFile);
std::fwrite(&m_maxPhraseLength, sizeof(m_maxPhraseLength), 1, m_outFile);
ThrowingFwrite(&m_coding, sizeof(m_coding), 1, m_outFile);
ThrowingFwrite(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, m_outFile);
ThrowingFwrite(&m_useAlignmentInfo, sizeof(m_useAlignmentInfo), 1, m_outFile);
ThrowingFwrite(&m_maxRank, sizeof(m_maxRank), 1, m_outFile);
ThrowingFwrite(&m_maxPhraseLength, sizeof(m_maxPhraseLength), 1, m_outFile);
if(m_coding == REnc)
{
@ -176,11 +177,11 @@ void PhraseTableCreator::Save()
// Save lexical translation table for REnc
size_t size = m_lexicalTableIndex.size();
std::fwrite(&size, sizeof(size_t), 1, m_outFile);
std::fwrite(&m_lexicalTableIndex[0], sizeof(size_t), size, m_outFile);
ThrowingFwrite(&size, sizeof(size_t), 1, m_outFile);
ThrowingFwrite(&m_lexicalTableIndex[0], sizeof(size_t), size, m_outFile);
size = m_lexicalTable.size();
std::fwrite(&size, sizeof(size_t), 1, m_outFile);
std::fwrite(&m_lexicalTable[0], sizeof(SrcTrg), size, m_outFile);
ThrowingFwrite(&size, sizeof(size_t), 1, m_outFile);
ThrowingFwrite(&m_lexicalTable[0], sizeof(SrcTrg), size, m_outFile);
}
// Save target language symbols
@ -200,7 +201,7 @@ void PhraseTableCreator::Save()
// Save number of Huffman code sets for scores and
// save Huffman code sets
std::fwrite(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, m_outFile);
ThrowingFwrite(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, m_outFile);
size_t numScoreTrees = m_scoreTrees.size();
for(size_t i = 0; i < numScoreTrees; i++)
m_scoreTrees[i]->Save(m_outFile);

7
moses/src/CompactPT/StringVector.h
View File

@ -31,6 +31,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <boost/iterator/iterator_facade.hpp>
#include "ThrowingFwrite.h"
#include "MonotonicVector.h"
#include "MmapAllocator.h"
@ -265,13 +266,13 @@ class StringVector
size_t save(std::FILE* out)
{
size_t byteSize = 0;
byteSize += std::fwrite(&m_sorted, sizeof(bool), 1, out) * sizeof(bool);
byteSize += ThrowingFwrite(&m_sorted, sizeof(bool), 1, out) * sizeof(bool);
byteSize += m_positions.save(out);
size_t valSize = size2();
byteSize += std::fwrite(&valSize, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += std::fwrite(&m_charArray[0], sizeof(ValueT), valSize, out) * sizeof(ValueT);
byteSize += ThrowingFwrite(&valSize, sizeof(size_t), 1, out) * sizeof(size_t);
byteSize += ThrowingFwrite(&m_charArray[0], sizeof(ValueT), valSize, out) * sizeof(ValueT);
return byteSize;
}

29
moses/src/CompactPT/ThrowingFwrite.cpp Normal file
View File

@ -0,0 +1,29 @@
// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "ThrowingFwrite.h"
size_t ThrowingFwrite(const void *ptr, size_t size, size_t count, FILE* stream) {
assert(size);
size_t returnValue = std::fwrite(ptr, size, count, stream);
UTIL_THROW_IF(count != returnValue, util::ErrnoException, "Short fwrite; requested size " << size);
return returnValue;
}

31
moses/src/CompactPT/ThrowingFwrite.h Normal file
View File

@ -0,0 +1,31 @@
// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef moses_ThrowingFwrite_h
#define moses_ThrowingFwrite_h
#include <cassert>
#include <cstdio>
#include "util/exception.hh"
size_t ThrowingFwrite(const void *ptr, size_t size, size_t count, FILE* stream);
#endif