apply Nikolay's changes to moses2 fork

This commit is contained in:
Hieu Hoang 2015-12-23 13:31:23 +00:00
parent 2951d3496e
commit 50a75aef1e
6 changed files with 31 additions and 14 deletions

View File

@ -171,6 +171,15 @@ TargetPhrase *ProbingPT::CreateTargetPhrase(MemPool &pool, const System &system,
// }
// */
// properties
if (probingTargetPhrase.property.size()) {
//cerr << "probingTargetPhrase.property.size()=" << probingTargetPhrase.property.size() << endl;
size_t size = probingTargetPhrase.property.size();
tp->properties = (char*) system.systemPool.Allocate(size + 1);
memcpy(tp->properties, probingTargetPhrase.property.data(), size);
tp->properties[size] = '0x0';
}
// score of all other ff when this rule is being loaded
const FeatureFunctions &ffs = system.featureFunctions;
ffs.EvaluateInIsolation(pool, system, sourcePhrase, *tp);

View File

@ -229,7 +229,7 @@ std::vector<target_text*> HuffmanDecoder::full_decode_line (unsigned char lines[
std::vector<unsigned int>::iterator it = decoded_lines.begin(); //Iterator for them
std::vector<unsigned int> current_target_phrase; //Current target phrase decoded
short zero_count = 0; //Count home many zeroes we have met. so far. Every 3 zeroes mean a new target phrase.
short zero_count = 0; //Count how many zeroes we have met. so far. Every 3 zeroes mean a new target phrase.
while(it != decoded_lines.end()) {
if (zero_count == 1) {
//We are extracting scores. we know how many scores there are so we can push them
@ -241,7 +241,7 @@ std::vector<target_text*> HuffmanDecoder::full_decode_line (unsigned char lines[
}
}
if (zero_count == 3) {
if (zero_count == 6) {
//We have finished with this entry, decode it, and add it to the retvector.
retvector.push_back(decode_line(current_target_phrase, num_scores));
current_target_phrase.clear(); //Clear the current target phrase and the zero_count
@ -255,7 +255,7 @@ std::vector<target_text*> HuffmanDecoder::full_decode_line (unsigned char lines[
it++; //Go to the next word/symbol
}
//Don't forget the last remaining line!
if (zero_count == 3) {
if (zero_count == 6) {
//We have finished with this entry, decode it, and add it to the retvector.
retvector.push_back(decode_line(current_target_phrase, num_scores));
current_target_phrase.clear(); //Clear the current target phrase and the zero_count
@ -277,7 +277,7 @@ target_text *HuffmanDecoder::decode_line (const std::vector<unsigned int> &input
//Split the line into the proper arrays
short num_zeroes = 0;
int counter = 0;
while (num_zeroes < 3) {
while (num_zeroes < 6) {
unsigned int num = input[counter];
if (num == 0) {
num_zeroes++;
@ -295,11 +295,18 @@ target_text *HuffmanDecoder::decode_line (const std::vector<unsigned int> &input
continue;
} else if (num_zeroes == 2) {
wAll = num;
} else if (num_zeroes == 3) {
ret->counts.push_back(static_cast<char>(input[counter]));
} else if (num_zeroes == 4) {
ret->sparse_score.push_back(static_cast<char>(input[counter]));
} else if (num_zeroes == 5) {
ret->property.push_back(static_cast<char>(input[counter]));
}
counter++;
}
ret->word_all1 = &lookup_word_all1.find(wAll)->second;
ret->word_all1 = lookup_word_all1.find(wAll)->second;
return ret;

View File

@ -23,9 +23,12 @@ struct line_text {
//Struct for holding processed line
struct target_text {
std::vector<unsigned int> target_phrase;
std::vector<float> prob;
std::vector<unsigned char> *word_all1;
std::vector<unsigned int> target_phrase;
std::vector<float> prob;
std::vector<unsigned char> word_all1;
std::vector<char> counts;
std::vector<char> sparse_score;
std::vector<char> property;
};
//Ask if it's better to have it receive a pointer to a line_text struct

View File

@ -169,11 +169,11 @@ void QueryEngine::printTargetInfo(const std::vector<target_text> &target_phrases
std::cout << "\t";
//Print word_all1
for (int j = 0; j<target_phrases[i].word_all1->size(); j++) {
for (int j = 0; j<target_phrases[i].word_all1.size(); j++) {
if (j%2 == 0) {
std::cout << (short)(*target_phrases[i].word_all1)[j] << "-";
std::cout << (short)(target_phrases[i].word_all1)[j] << "-";
} else {
std::cout << (short)(*target_phrases[i].word_all1)[j] << " ";
std::cout << (short)(target_phrases[i].word_all1)[j] << " ";
}
}
std::cout << std::endl;

View File

@ -8,7 +8,7 @@
#include "hash.hh" //Includes line splitter
#include "../../Vector.h"
#define API_VERSION 3
#define API_VERSION 4
namespace Moses2
{

View File

@ -76,8 +76,6 @@ GetScore(const Phrase& f, const Phrase& e, const Phrase& c)
key = MakeKey(f,e,sub_c);
}
std::cerr << "key=" << key << std::endl;
size_t index = m_hash[key];
if(m_hash.GetSize() != index) {
std::string scoresString;