moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-28 14:32:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

use ken's read map

Browse Source
This commit is contained in:
Hieu Hoang 2017-01-10 18:49:44 +00:00
parent 48f1bac05d
commit cf3b51d544
4 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
contrib/moses2/TranslationModel/ProbingPT/probing_hash_utils.cpp
View File

@ -6,14 +6,15 @@ namespace Moses2
{
//Read table from disk, return memory map location
char * readTable(const char * filename, size_t size)
char * readTable(const char * filename, size_t size, util::scoped_fd &file, util::scoped_memory &memory)
{
std::cerr << "filename=" << filename << std::endl;
util::scoped_fd file_(util::OpenReadOrThrow(filename));
uint64_t total_size_ = util::SizeFile(file_.get());
file.reset(util::OpenReadOrThrow(filename));
uint64_t total_size_ = util::SizeFile(file.get());
util::scoped_memory memory;
MapRead(util::LAZY, file_.get(), 0, total_size_, memory);
MapRead(util::LAZY, file.get(), 0, total_size_, memory);
//return memory.begin();
//Initial position of the file is the end of the file, thus we know the size
int fd;

2
contrib/moses2/TranslationModel/ProbingPT/probing_hash_utils.hh
View File

@ -38,7 +38,7 @@ typedef util::ProbingHashTable<Entry, boost::hash<uint64_t> > Table;
void serialize_table(char *mem, size_t size, const std::string &filename);
char * readTable(const char * filename, size_t size);
char * readTable(const char * filename, size_t size, util::scoped_fd &file, util::scoped_memory &memory);
uint64_t getKey(const uint64_t source_phrase[], size_t size);

2
contrib/moses2/TranslationModel/ProbingPT/querying.cpp
View File

@ -86,7 +86,7 @@ QueryEngine::QueryEngine(const char * filepath, util::LoadMethod load_method)
//Read hashtable
table_filesize = Table::Size(tablesize, 1.2);
mem = readTable(path_to_hashtable.c_str(), table_filesize);
mem = readTable(path_to_hashtable.c_str(), table_filesize, file_, memory_);
Table table_init(mem, table_filesize);
table = table_init;

3
contrib/moses2/TranslationModel/ProbingPT/querying.hh
View File

@ -26,6 +26,9 @@ class QueryEngine
size_t table_filesize;
bool is_reordering;
util::scoped_fd file_;
util::scoped_memory memory_;
void read_alignments(const std::string &alignPath);
void file_exits(const std::string &basePath);
void cat_files(const std::string &basePath);