mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
- block recombination when two hypotheses have different source word range and at least one is following a gap in the source sentence
- change maxent scoring: subtract optimistic score for left movements when actual score is available git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/eva_maxent@1928 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
42eb8992fc
commit
d594392e3d
@ -99,6 +99,33 @@ bool HypothesisStackNormal::AddPrune(Hypothesis *hypo)
|
|||||||
iterator &iterExisting = addRet.first;
|
iterator &iterExisting = addRet.first;
|
||||||
Hypothesis *hypoExisting = *iterExisting;
|
Hypothesis *hypoExisting = *iterExisting;
|
||||||
assert(iterExisting != m_hypos.end());
|
assert(iterExisting != m_hypos.end());
|
||||||
|
|
||||||
|
|
||||||
|
const WordsRange currSourceWordsRange = hypo->GetCurrSourceWordsRange();
|
||||||
|
const WordsRange existSourceWordsRange = hypoExisting->GetCurrSourceWordsRange();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* if the two hypotheses differ in range and one of them follows a gap, do not recombine! May be needed for maxent scoring
|
||||||
|
*/
|
||||||
|
|
||||||
|
if( StaticData::Instance().UseMaxentReordering()){
|
||||||
|
if( existSourceWordsRange.GetStartPos() != 0 && !(hypoExisting->GetWordsBitmap().GetValue( existSourceWordsRange.GetStartPos()-1)) ){
|
||||||
|
if(currSourceWordsRange.GetNumWordsCovered() != existSourceWordsRange.GetNumWordsCovered()){
|
||||||
|
IFVERBOSE(2){
|
||||||
|
std::cerr << "Words ranges are different and existing hypo is following a gap --> do not recombine hypotheses..\n";
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if( currSourceWordsRange.GetStartPos() != 0 && !(hypo->GetWordsBitmap().GetValue( currSourceWordsRange.GetStartPos()-1)) ){
|
||||||
|
if(currSourceWordsRange.GetNumWordsCovered() != existSourceWordsRange.GetNumWordsCovered()){
|
||||||
|
IFVERBOSE(2){
|
||||||
|
std::cerr << "Words ranges are different and new hypo is following a gap --> do not recombine hypotheses..\n";
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
|
StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
|
||||||
|
|
||||||
|
@ -79,12 +79,8 @@ std::vector<float> MaxentReordering::CalcScore(Hypothesis* hypothesis) const {
|
|||||||
OrientationType orientation_curr = orientations[0];
|
OrientationType orientation_curr = orientations[0];
|
||||||
OrientationType orientation_next = orientations[1];
|
OrientationType orientation_next = orientations[1];
|
||||||
|
|
||||||
assert(orientation_curr != orientation_next);
|
assert(orientation_curr == 0 || orientation_curr == 1 || orientation_curr == 4);
|
||||||
assert(orientation_curr <= 5 && (orientation_next == 2 || orientation_next == 3 || orientation_next == 5) );
|
assert(orientation_next == 2 || orientation_next == 3 || orientation_next == 5);
|
||||||
IFVERBOSE(2){
|
|
||||||
std::cerr << "Maxent orientation type (curr): " << orientation_curr << "\n";
|
|
||||||
std::cerr << "Maxent orientation type (next): " << orientation_next << "\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
// grab data for current hypothesis
|
// grab data for current hypothesis
|
||||||
const ScoreComponentCollection &reorderingScoreColl_curr =
|
const ScoreComponentCollection &reorderingScoreColl_curr =
|
||||||
@ -104,17 +100,22 @@ std::vector<float> MaxentReordering::CalcScore(Hypothesis* hypothesis) const {
|
|||||||
assert(values_next.size() == GetNumOrientationTypes());
|
assert(values_next.size() == GetNumOrientationTypes());
|
||||||
nextHypoExists = true;
|
nextHypoExists = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
IFVERBOSE(2){
|
||||||
|
std::cerr << "Maxent orientation type (curr, ID = " << hypothesis->GetId() << "): " << orientation_curr << "\n";
|
||||||
|
if(nextHypoExists)
|
||||||
|
std::cerr << "Maxent orientation type (next, ID = " << nextHypo->GetId() << "): " << orientation_next << "\n";
|
||||||
|
else
|
||||||
|
std::cerr << "Maxent orientation type (next, no ID): " << orientation_next << "\n";
|
||||||
|
}
|
||||||
|
|
||||||
//add score
|
//add score
|
||||||
float value_curr = 0.0, value_next = 0.0;
|
float value_curr = 0.0, value_next = 0.0;
|
||||||
IFVERBOSE(2){
|
IFVERBOSE(2){
|
||||||
std::cerr << "Curr scores: " << values_curr[0] << " " << values_curr[1] << " " << values_curr[2] << " " << values_curr[3] << "\n";
|
std::cerr << "Curr scores: " << values_curr[0] << " " << values_curr[1] << " " << values_curr[2] << " " << values_curr[3] << "\n";
|
||||||
}
|
}
|
||||||
if(orientation_curr < 4){
|
if(orientation_curr == 4){
|
||||||
value_curr = values_curr[orientation_curr];
|
// optimistic guess: use better one of the values for LEFT and LEFT_PLUS
|
||||||
}
|
|
||||||
else if(orientation_curr == 4){
|
|
||||||
// optimistic guess: use better one of the values for LEFT and LEFT_PLUS
|
|
||||||
if(values_curr[2] >= values_curr[3]){
|
if(values_curr[2] >= values_curr[3]){
|
||||||
value_curr = values_curr[2];
|
value_curr = values_curr[2];
|
||||||
orientation_curr = 2;
|
orientation_curr = 2;
|
||||||
@ -123,23 +124,54 @@ std::vector<float> MaxentReordering::CalcScore(Hypothesis* hypothesis) const {
|
|||||||
value_curr = values_curr[3];
|
value_curr = values_curr[3];
|
||||||
orientation_curr = 3;
|
orientation_curr = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
value_curr = values_curr[orientation_curr];
|
||||||
}
|
}
|
||||||
IFVERBOSE(2){
|
IFVERBOSE(2){
|
||||||
std::cerr << "Maxent value (curr): " << value_curr << "\n";
|
std::cerr << "Maxent value (curr): " << value_curr << "\n";
|
||||||
}
|
}
|
||||||
if(orientation_curr < 5)
|
// assign score of current hypothesis
|
||||||
score[orientation_curr] = value_curr;
|
score[orientation_curr] = value_curr;
|
||||||
|
|
||||||
if(nextHypoExists){
|
if(nextHypoExists){
|
||||||
if(orientation_next != 5){
|
if(orientation_next != 5){
|
||||||
value_next = values_next[orientation_next];
|
// float diff = 0.0;
|
||||||
|
// If the actual score is the worse of the LEFT and LEFT_PLUS scores, add difference of scores.
|
||||||
|
// In this case the optimistic guess has added the better score already.
|
||||||
|
// Otherwise do nothing, because the optimistic score was correct
|
||||||
|
|
||||||
|
// // TODO: only problem: If the optimistic guess was better than the actual score, it was added
|
||||||
|
// // with a different weight because of the different position (e.g. position 2 instead of 3).
|
||||||
|
if(orientation_next == 2)
|
||||||
|
if(values_next[2] < values_next[3]){
|
||||||
|
// // add difference of score, optimistic guess has added the better score already
|
||||||
|
// diff = values_next[2] - values_next[3];
|
||||||
|
// subtract optimistic score values_next[3] and add actual score values_next[2]
|
||||||
|
score[3] -= values_next[3];
|
||||||
|
// add score instead of assigning in case the current hypothesis uses the same slot
|
||||||
|
score[2] += values_next[2];
|
||||||
|
}
|
||||||
|
if(orientation_next == 3)
|
||||||
|
if(values_next[3] < values_next[2]){
|
||||||
|
// // add difference of score, optimistic guess has added the better score already
|
||||||
|
// diff = values_next[3] - values_next[2];
|
||||||
|
// subtract optimistic score values_next[2] and add actual score values_next[3]
|
||||||
|
score[2] -= values_next[2];
|
||||||
|
// add score instead of assigning in case the current hypothesis uses the same slot
|
||||||
|
score[3] += values_next[3];
|
||||||
|
}
|
||||||
|
|
||||||
|
// value_next = diff;
|
||||||
|
// // add score instead of assigning in case the current hypothesis uses the same slot
|
||||||
|
// score[orientation_next] += value_next;
|
||||||
}
|
}
|
||||||
IFVERBOSE(2){
|
IFVERBOSE(2){
|
||||||
std::cerr << "Next scores: " << values_next[0] << " " << values_next[1] << " " << values_next[2] << " " << values_next[3] << "\n";
|
std::cerr << "Next scores: " << values_next[0] << " " << values_next[1] << " " << values_next[2] << " " << values_next[3] << "\n";
|
||||||
std::cerr << "Maxent value (next): " << value_next << "\n";
|
std::cerr << "Maxent value (next): " << value_next << "\n";
|
||||||
}
|
}
|
||||||
if(orientation_next < 5)
|
|
||||||
score[orientation_next] = value_next;
|
|
||||||
}
|
}
|
||||||
IFVERBOSE(2){
|
IFVERBOSE(2){
|
||||||
for(int i=0; i< score.size(); i++){
|
for(int i=0; i< score.size(); i++){
|
||||||
@ -310,22 +342,15 @@ std::vector< MaxentReordering::OrientationType> MaxentOrientationReordering::Get
|
|||||||
// CASE 4: Previous source word is not yet translated -> undefined left movement -> LEFT_undef
|
// CASE 4: Previous source word is not yet translated -> undefined left movement -> LEFT_undef
|
||||||
IFVERBOSE(2){
|
IFVERBOSE(2){
|
||||||
std::cerr << "Previous word is not translated yet.. \n";
|
std::cerr << "Previous word is not translated yet.. \n";
|
||||||
}
|
std::cerr << "fallback: jump LEFT_undef\n";
|
||||||
// if next word is already translated, use ONLY the probability of this well-defined left jump
|
}
|
||||||
// otherwise fallback to LEFT_undef and NONE
|
orientations.push_back(LEFT_undef);
|
||||||
|
|
||||||
if( (currHypothesis->GetWordsBitmap().GetSize() > currSourceWordsRange.GetEndPos()+1) &&
|
if( (currHypothesis->GetWordsBitmap().GetSize() > currSourceWordsRange.GetEndPos()+1) &&
|
||||||
(currHypothesis->GetWordsBitmap().GetValue( currSourceWordsRange.GetEndPos()+1 )) ){
|
(currHypothesis->GetWordsBitmap().GetValue( currSourceWordsRange.GetEndPos()+1 )) ){
|
||||||
orientations.push_back(NONE);
|
|
||||||
IFVERBOSE(2){
|
|
||||||
std::cerr << "no jump w.r.t. previous words..\n";
|
|
||||||
}
|
|
||||||
orientations.push_back( ReEvaluateWithNextPhraseInSource(currHypothesis, currSourceWordsRange) );
|
orientations.push_back( ReEvaluateWithNextPhraseInSource(currHypothesis, currSourceWordsRange) );
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
IFVERBOSE(2){
|
|
||||||
std::cerr << "fallback: jump LEFT_undef\n";
|
|
||||||
}
|
|
||||||
orientations.push_back(LEFT_undef);
|
|
||||||
orientations.push_back(NONE);
|
orientations.push_back(NONE);
|
||||||
}
|
}
|
||||||
return orientations;
|
return orientations;
|
||||||
@ -336,7 +361,7 @@ std::vector< MaxentReordering::OrientationType> MaxentOrientationReordering::Get
|
|||||||
MaxentReordering::OrientationType MaxentOrientationReordering::ReEvaluateWithNextPhraseInSource(Hypothesis* currHypothesis, const WordsRange currSourceWordsRange) const {
|
MaxentReordering::OrientationType MaxentOrientationReordering::ReEvaluateWithNextPhraseInSource(Hypothesis* currHypothesis, const WordsRange currSourceWordsRange) const {
|
||||||
// Word to the right of current phrase is already translated, TODO: what to do?
|
// Word to the right of current phrase is already translated, TODO: what to do?
|
||||||
IFVERBOSE(2){
|
IFVERBOSE(2){
|
||||||
std::cerr << "Following word is already translated: backwards evaluation?\n";
|
std::cerr << "Following word is already translated: backwards evaluation\n";
|
||||||
}
|
}
|
||||||
const Hypothesis *next = currHypothesis->GetHypoContainingPosition( currSourceWordsRange.GetEndPos()+1 );
|
const Hypothesis *next = currHypothesis->GetHypoContainingPosition( currSourceWordsRange.GetEndPos()+1 );
|
||||||
assert( next != NULL);
|
assert( next != NULL);
|
||||||
|
@ -118,7 +118,7 @@ public:
|
|||||||
|
|
||||||
void Assign(const ScoreProducer* sp, const std::vector<float>& scores)
|
void Assign(const ScoreProducer* sp, const std::vector<float>& scores)
|
||||||
{
|
{
|
||||||
assert(scores.size() == sp->GetNumScoreComponents());
|
assert(scores.size() == sp->GetNumScoreComponents());
|
||||||
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
|
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
|
||||||
for (std::vector<float>::const_iterator vi = scores.begin();
|
for (std::vector<float>::const_iterator vi = scores.begin();
|
||||||
vi != scores.end(); ++vi)
|
vi != scores.end(); ++vi)
|
||||||
|
@ -603,8 +603,7 @@ bool StaticData::LoadMaxentReorderingModel()
|
|||||||
//spec[1] = name // lexical reordering
|
//spec[1] = name // lexical reordering
|
||||||
//spec[2] = num weights // lexical reordering
|
//spec[2] = num weights // lexical reordering
|
||||||
//spec[3] = fileName
|
//spec[3] = fileName
|
||||||
//spec[4] = condition for maxent (a or b)
|
//spec[4] = num weights maxent
|
||||||
//spec[5] = number of maxent outcomes
|
|
||||||
|
|
||||||
//decode data into these
|
//decode data into these
|
||||||
vector<FactorType> input,output;
|
vector<FactorType> input,output;
|
||||||
|
Loading…
Reference in New Issue
Block a user