mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-24 04:12:47 +03:00
Fix incorrect hashing in ProbingPT
This commit is contained in:
parent
4aa95516bf
commit
d0f4402e86
@ -73,7 +73,7 @@ std::pair<bool, std::vector<target_text> > QueryEngine::query(std::vector<uint64
|
||||
//uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size());
|
||||
uint64_t key = 0;
|
||||
for (int i = 0; i < source_phrase.size(); i++){
|
||||
key += source_phrase[i];
|
||||
key += (source_phrase[i] << i);
|
||||
}
|
||||
|
||||
|
||||
@ -114,7 +114,7 @@ std::pair<bool, std::vector<target_text> > QueryEngine::query(StringPiece source
|
||||
//uint64_t key = util::MurmurHashNative(&source_phrase_vid[0], source_phrase_vid.size());
|
||||
uint64_t key = 0;
|
||||
for (int i = 0; i < source_phrase_vid.size(); i++){
|
||||
key += source_phrase_vid[i];
|
||||
key += (source_phrase_vid[i] << i);
|
||||
}
|
||||
|
||||
found = table.Find(key, entry);
|
||||
|
@ -89,11 +89,12 @@ void createProbingPT(const char * phrasetable_path, const char * target_path){
|
||||
//Create an entry for the previous source phrase:
|
||||
Entry pesho;
|
||||
pesho.value = entrystartidx;
|
||||
//The key is the sum of hashes of individual words. Probably not entirerly correct, but fast
|
||||
//The key is the sum of hashes of individual words bitshifted by their position in the phrase.
|
||||
//Probably not entirerly correct, but fast and seems to work fine in practise.
|
||||
pesho.key = 0;
|
||||
std::vector<uint64_t> vocabid_source = getVocabIDs(prev_line.source_phrase);
|
||||
for (int i = 0; i < vocabid_source.size(); i++){
|
||||
pesho.key += vocabid_source[i];
|
||||
pesho.key += (vocabid_source[i] << i);
|
||||
}
|
||||
pesho.bytes_toread = binfile.dist_from_start + binfile.extra_counter - entrystartidx;
|
||||
|
||||
@ -127,7 +128,7 @@ void createProbingPT(const char * phrasetable_path, const char * target_path){
|
||||
pesho.key = 0;
|
||||
std::vector<uint64_t> vocabid_source = getVocabIDs(prev_line.source_phrase);
|
||||
for (int i = 0; i < vocabid_source.size(); i++){
|
||||
pesho.key += vocabid_source[i];
|
||||
pesho.key += (vocabid_source[i] << i);
|
||||
}
|
||||
pesho.bytes_toread = binfile.dist_from_start + binfile.extra_counter - entrystartidx;
|
||||
//Put into table
|
||||
|
Loading…
Reference in New Issue
Block a user