moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-20 07:42:21 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix incorrect hashing in ProbingPT

Browse Source
This commit is contained in:
Nikolay Bogoychev 2014-12-16 11:15:12 +00:00
parent 4aa95516bf
commit d0f4402e86
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
moses/TranslationModel/ProbingPT/quering.cpp
View File

@ -73,7 +73,7 @@ std::pair<bool, std::vector<target_text> > QueryEngine::query(std::vector<uint64
//uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size());
uint64_t key = 0;
for (int i = 0; i < source_phrase.size(); i++){
key += source_phrase[i];
key += (source_phrase[i] << i);
}
@ -114,7 +114,7 @@ std::pair<bool, std::vector<target_text> > QueryEngine::query(StringPiece source
//uint64_t key = util::MurmurHashNative(&source_phrase_vid[0], source_phrase_vid.size());
uint64_t key = 0;
for (int i = 0; i < source_phrase_vid.size(); i++){
key += source_phrase_vid[i];
key += (source_phrase_vid[i] << i);
}
found = table.Find(key, entry);

7
moses/TranslationModel/ProbingPT/storing.cpp
View File

@ -89,11 +89,12 @@ void createProbingPT(const char * phrasetable_path, const char * target_path){
//Create an entry for the previous source phrase:
Entry pesho;
pesho.value = entrystartidx;
//The key is the sum of hashes of individual words. Probably not entirerly correct, but fast
//The key is the sum of hashes of individual words bitshifted by their position in the phrase.
//Probably not entirerly correct, but fast and seems to work fine in practise.
pesho.key = 0;
std::vector<uint64_t> vocabid_source = getVocabIDs(prev_line.source_phrase);
for (int i = 0; i < vocabid_source.size(); i++){
pesho.key += vocabid_source[i];
pesho.key += (vocabid_source[i] << i);
}
pesho.bytes_toread = binfile.dist_from_start + binfile.extra_counter - entrystartidx;
@ -127,7 +128,7 @@ void createProbingPT(const char * phrasetable_path, const char * target_path){
pesho.key = 0;
std::vector<uint64_t> vocabid_source = getVocabIDs(prev_line.source_phrase);
for (int i = 0; i < vocabid_source.size(); i++){
pesho.key += vocabid_source[i];
pesho.key += (vocabid_source[i] << i);
}
pesho.bytes_toread = binfile.dist_from_start + binfile.extra_counter - entrystartidx;
//Put into table