mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Sparse distortion features
in the manner of: Spence Green, Michel Galley, Christopher D. Manning. 2010. Improved Models of Distortion Cost for Statistical Machine Translation. In NAACL 2010.
This commit is contained in:
parent
1ac4ca5735
commit
bbf8a615f2
@ -1,26 +1,31 @@
|
|||||||
|
|
||||||
#include "DistortionScoreProducer.h"
|
#include "DistortionScoreProducer.h"
|
||||||
#include "FFState.h"
|
#include "FFState.h"
|
||||||
|
#include "moses/InputPath.h"
|
||||||
#include "moses/Range.h"
|
#include "moses/Range.h"
|
||||||
#include "moses/StaticData.h"
|
#include "moses/StaticData.h"
|
||||||
#include "moses/Hypothesis.h"
|
#include "moses/Hypothesis.h"
|
||||||
#include "moses/Manager.h"
|
#include "moses/Manager.h"
|
||||||
|
#include "moses/FactorCollection.h"
|
||||||
|
#include <cmath>
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
struct DistortionState_traditional : public FFState {
|
struct DistortionState : public FFState {
|
||||||
Range range;
|
Range range;
|
||||||
int first_gap;
|
int first_gap;
|
||||||
DistortionState_traditional(const Range& wr, int fg) : range(wr), first_gap(fg) {}
|
bool inSubordinateConjunction;
|
||||||
|
DistortionState(const Range& wr, int fg, bool subord=false) : range(wr), first_gap(fg), inSubordinateConjunction(subord) {}
|
||||||
|
|
||||||
size_t hash() const {
|
size_t hash() const {
|
||||||
return range.GetEndPos();
|
return range.GetEndPos();
|
||||||
}
|
}
|
||||||
virtual bool operator==(const FFState& other) const {
|
virtual bool operator==(const FFState& other) const {
|
||||||
const DistortionState_traditional& o =
|
const DistortionState& o =
|
||||||
static_cast<const DistortionState_traditional&>(other);
|
static_cast<const DistortionState&>(other);
|
||||||
return range.GetEndPos() == o.range.GetEndPos();
|
return ( (range.GetEndPos() == o.range.GetEndPos()) && (inSubordinateConjunction == o.inSubordinateConjunction) );
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
@ -29,11 +34,36 @@ std::vector<const DistortionScoreProducer*> DistortionScoreProducer::s_staticCol
|
|||||||
|
|
||||||
DistortionScoreProducer::DistortionScoreProducer(const std::string &line)
|
DistortionScoreProducer::DistortionScoreProducer(const std::string &line)
|
||||||
: StatefulFeatureFunction(1, line)
|
: StatefulFeatureFunction(1, line)
|
||||||
|
, m_useSparse(false)
|
||||||
|
, m_sparseDistance(false)
|
||||||
|
, m_sparseSubordinate(false)
|
||||||
{
|
{
|
||||||
s_staticColl.push_back(this);
|
s_staticColl.push_back(this);
|
||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void DistortionScoreProducer::SetParameter(const std::string& key, const std::string& value)
|
||||||
|
{
|
||||||
|
if (key == "sparse") {
|
||||||
|
m_useSparse = Scan<bool>(value);
|
||||||
|
} else if (key == "sparse-distance") {
|
||||||
|
m_sparseDistance = Scan<bool>(value);
|
||||||
|
} else if (key == "sparse-input-factor") {
|
||||||
|
m_sparseFactorTypeSource = Scan<FactorType>(value);
|
||||||
|
} else if (key == "sparse-output-factor") {
|
||||||
|
m_sparseFactorTypeTarget = Scan<FactorType>(value);
|
||||||
|
} else if (key == "sparse-subordinate") {
|
||||||
|
std::string subordinateConjunctionTag = Scan<std::string>(value);
|
||||||
|
FactorCollection &factorCollection = FactorCollection::Instance();
|
||||||
|
m_subordinateConjunctionTagFactor = factorCollection.AddFactor(subordinateConjunctionTag,false);
|
||||||
|
m_sparseSubordinate = true;
|
||||||
|
} else if (key == "sparse-subordinate-output-factor") {
|
||||||
|
m_sparseFactorTypeTargetSubordinate = Scan<FactorType>(value);
|
||||||
|
} else {
|
||||||
|
StatefulFeatureFunction::SetParameter(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &input) const
|
const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &input) const
|
||||||
{
|
{
|
||||||
// fake previous translated phrase start and end
|
// fake previous translated phrase start and end
|
||||||
@ -44,7 +74,7 @@ const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &in
|
|||||||
start = 0;
|
start = 0;
|
||||||
end = input.m_frontSpanCoveredLength -1;
|
end = input.m_frontSpanCoveredLength -1;
|
||||||
}
|
}
|
||||||
return new DistortionState_traditional(
|
return new DistortionState(
|
||||||
Range(start, end),
|
Range(start, end),
|
||||||
NOT_FOUND);
|
NOT_FOUND);
|
||||||
}
|
}
|
||||||
@ -101,17 +131,184 @@ FFState* DistortionScoreProducer::EvaluateWhenApplied(
|
|||||||
const FFState* prev_state,
|
const FFState* prev_state,
|
||||||
ScoreComponentCollection* out) const
|
ScoreComponentCollection* out) const
|
||||||
{
|
{
|
||||||
const DistortionState_traditional* prev = static_cast<const DistortionState_traditional*>(prev_state);
|
const DistortionState* prev = static_cast<const DistortionState*>(prev_state);
|
||||||
|
bool subordinateConjunction = prev->inSubordinateConjunction;
|
||||||
|
|
||||||
|
if (m_useSparse) {
|
||||||
|
int jumpFromPos = prev->range.GetEndPos()+1;
|
||||||
|
int jumpToPos = hypo.GetCurrSourceWordsRange().GetStartPos();
|
||||||
|
size_t distance = std::abs( jumpFromPos - jumpToPos );
|
||||||
|
|
||||||
|
const Sentence& sentence = static_cast<const Sentence&>(hypo.GetInput());
|
||||||
|
|
||||||
|
StringPiece jumpFromSourceFactorPrev;
|
||||||
|
StringPiece jumpFromSourceFactor;
|
||||||
|
StringPiece jumpToSourceFactor;
|
||||||
|
if (jumpFromPos < (int)sentence.GetSize()) {
|
||||||
|
jumpFromSourceFactor = sentence.GetWord(jumpFromPos).GetFactor(m_sparseFactorTypeSource)->GetString();
|
||||||
|
} else {
|
||||||
|
jumpFromSourceFactor = "</s>";
|
||||||
|
}
|
||||||
|
if (jumpFromPos > 0) {
|
||||||
|
jumpFromSourceFactorPrev = sentence.GetWord(jumpFromPos-1).GetFactor(m_sparseFactorTypeSource)->GetString();
|
||||||
|
} else {
|
||||||
|
jumpFromSourceFactorPrev = "<s>";
|
||||||
|
}
|
||||||
|
jumpToSourceFactor = sentence.GetWord(jumpToPos).GetFactor(m_sparseFactorTypeSource)->GetString();
|
||||||
|
|
||||||
|
const TargetPhrase& currTargetPhrase = hypo.GetCurrTargetPhrase();
|
||||||
|
StringPiece jumpToTargetFactor = currTargetPhrase.GetWord(0).GetFactor(m_sparseFactorTypeTarget)->GetString();
|
||||||
|
|
||||||
|
util::StringStream featureName;
|
||||||
|
|
||||||
|
// source factor (start position)
|
||||||
|
featureName = util::StringStream();
|
||||||
|
featureName << m_description << "_";
|
||||||
|
if ( jumpToPos > jumpFromPos ) {
|
||||||
|
featureName << "R";
|
||||||
|
} else if ( jumpToPos < jumpFromPos ) {
|
||||||
|
featureName << "L";
|
||||||
|
} else {
|
||||||
|
featureName << "M";
|
||||||
|
}
|
||||||
|
if (m_sparseDistance) {
|
||||||
|
featureName << distance;
|
||||||
|
}
|
||||||
|
featureName << "_SFS_" << jumpFromSourceFactor;
|
||||||
|
if (m_sparseSubordinate && subordinateConjunction) {
|
||||||
|
featureName << "_SUBORD";
|
||||||
|
}
|
||||||
|
out->SparsePlusEquals(featureName.str(), 1);
|
||||||
|
|
||||||
|
// source factor (start position minus 1)
|
||||||
|
featureName = util::StringStream();
|
||||||
|
featureName << m_description << "_";
|
||||||
|
if ( jumpToPos > jumpFromPos ) {
|
||||||
|
featureName << "R";
|
||||||
|
} else if ( jumpToPos < jumpFromPos ) {
|
||||||
|
featureName << "L";
|
||||||
|
} else {
|
||||||
|
featureName << "M";
|
||||||
|
}
|
||||||
|
if (m_sparseDistance) {
|
||||||
|
featureName << distance;
|
||||||
|
}
|
||||||
|
featureName << "_SFP_" << jumpFromSourceFactorPrev;
|
||||||
|
if (m_sparseSubordinate && subordinateConjunction) {
|
||||||
|
featureName << "_SUBORD";
|
||||||
|
}
|
||||||
|
out->SparsePlusEquals(featureName.str(), 1);
|
||||||
|
|
||||||
|
// source factor (end position)
|
||||||
|
featureName = util::StringStream();
|
||||||
|
featureName << m_description << "_";
|
||||||
|
if ( jumpToPos > jumpFromPos ) {
|
||||||
|
featureName << "R";
|
||||||
|
} else if ( jumpToPos < jumpFromPos ) {
|
||||||
|
featureName << "L";
|
||||||
|
} else {
|
||||||
|
featureName << "M";
|
||||||
|
}
|
||||||
|
if (m_sparseDistance) {
|
||||||
|
featureName << distance;
|
||||||
|
}
|
||||||
|
featureName << "_SFE_" << jumpToSourceFactor;
|
||||||
|
if (m_sparseSubordinate && subordinateConjunction) {
|
||||||
|
featureName << "_SUBORD";
|
||||||
|
}
|
||||||
|
out->SparsePlusEquals(featureName.str(), 1);
|
||||||
|
|
||||||
|
// target factor (end position)
|
||||||
|
featureName = util::StringStream();
|
||||||
|
featureName << m_description << "_";
|
||||||
|
if ( jumpToPos > jumpFromPos ) {
|
||||||
|
featureName << "R";
|
||||||
|
} else if ( jumpToPos < jumpFromPos ) {
|
||||||
|
featureName << "L";
|
||||||
|
} else {
|
||||||
|
featureName << "M";
|
||||||
|
}
|
||||||
|
if (m_sparseDistance) {
|
||||||
|
featureName << distance;
|
||||||
|
}
|
||||||
|
featureName << "_TFE_" << jumpToTargetFactor;
|
||||||
|
if (m_sparseSubordinate && subordinateConjunction) {
|
||||||
|
featureName << "_SUBORD";
|
||||||
|
}
|
||||||
|
out->SparsePlusEquals(featureName.str(), 1);
|
||||||
|
|
||||||
|
// relative source sentence position
|
||||||
|
featureName = util::StringStream();
|
||||||
|
featureName << m_description << "_";
|
||||||
|
if ( jumpToPos > jumpFromPos ) {
|
||||||
|
featureName << "R";
|
||||||
|
} else if ( jumpToPos < jumpFromPos ) {
|
||||||
|
featureName << "L";
|
||||||
|
} else {
|
||||||
|
featureName << "M";
|
||||||
|
}
|
||||||
|
if (m_sparseDistance) {
|
||||||
|
featureName << distance;
|
||||||
|
}
|
||||||
|
size_t relativeSourceSentencePosBin = std::floor( 5 * (float)jumpFromPos / (sentence.GetSize()+1) );
|
||||||
|
featureName << "_P_" << relativeSourceSentencePosBin;
|
||||||
|
if (m_sparseSubordinate && subordinateConjunction) {
|
||||||
|
featureName << "_SUBORD";
|
||||||
|
}
|
||||||
|
out->SparsePlusEquals(featureName.str(), 1);
|
||||||
|
|
||||||
|
// source sentence length bin
|
||||||
|
featureName = util::StringStream();
|
||||||
|
featureName << m_description << "_";
|
||||||
|
if ( jumpToPos > jumpFromPos ) {
|
||||||
|
featureName << "R";
|
||||||
|
} else if ( jumpToPos < jumpFromPos ) {
|
||||||
|
featureName << "L";
|
||||||
|
} else {
|
||||||
|
featureName << "M";
|
||||||
|
}
|
||||||
|
if (m_sparseDistance) {
|
||||||
|
featureName << distance;
|
||||||
|
}
|
||||||
|
size_t sourceSentenceLengthBin = 3;
|
||||||
|
if (sentence.GetSize() < 15) {
|
||||||
|
sourceSentenceLengthBin = 0;
|
||||||
|
} else if (sentence.GetSize() < 23) {
|
||||||
|
sourceSentenceLengthBin = 1;
|
||||||
|
} else if (sentence.GetSize() < 33) {
|
||||||
|
sourceSentenceLengthBin = 2;
|
||||||
|
}
|
||||||
|
featureName << "_SL_" << sourceSentenceLengthBin;
|
||||||
|
if (m_sparseSubordinate && subordinateConjunction) {
|
||||||
|
featureName << "_SUBORD";
|
||||||
|
}
|
||||||
|
out->SparsePlusEquals(featureName.str(), 1);
|
||||||
|
|
||||||
|
if (m_sparseSubordinate) {
|
||||||
|
for (size_t posT=0; posT<currTargetPhrase.GetSize(); ++posT) {
|
||||||
|
const Word &wordT = currTargetPhrase.GetWord(posT);
|
||||||
|
if (wordT[m_sparseFactorTypeTargetSubordinate] == m_subordinateConjunctionTagFactor) {
|
||||||
|
subordinateConjunction = true;
|
||||||
|
} else if (wordT[m_sparseFactorTypeTargetSubordinate]->GetString()[0] == 'V') {
|
||||||
|
subordinateConjunction = false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const float distortionScore = CalculateDistortionScore(
|
const float distortionScore = CalculateDistortionScore(
|
||||||
hypo,
|
hypo,
|
||||||
prev->range,
|
prev->range,
|
||||||
hypo.GetCurrSourceWordsRange(),
|
hypo.GetCurrSourceWordsRange(),
|
||||||
prev->first_gap);
|
prev->first_gap);
|
||||||
out->PlusEquals(this, distortionScore);
|
out->PlusEquals(this, distortionScore);
|
||||||
DistortionState_traditional* res = new DistortionState_traditional(
|
|
||||||
|
DistortionState* state = new DistortionState(
|
||||||
hypo.GetCurrSourceWordsRange(),
|
hypo.GetCurrSourceWordsRange(),
|
||||||
hypo.GetWordsBitmap().GetFirstGapPos());
|
hypo.GetWordsBitmap().GetFirstGapPos(),
|
||||||
return res;
|
subordinateConjunction);
|
||||||
|
|
||||||
|
return state;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,16 +1,11 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <stdexcept>
|
|
||||||
#include <string>
|
#include <string>
|
||||||
#include "StatefulFeatureFunction.h"
|
#include "StatefulFeatureFunction.h"
|
||||||
|
#include "moses/Range.h"
|
||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
class FFState;
|
|
||||||
class ScoreComponentCollection;
|
|
||||||
class Hypothesis;
|
|
||||||
class ChartHypothesis;
|
|
||||||
class Range;
|
|
||||||
|
|
||||||
/** Calculates Distortion scores
|
/** Calculates Distortion scores
|
||||||
*/
|
*/
|
||||||
@ -19,6 +14,14 @@ class DistortionScoreProducer : public StatefulFeatureFunction
|
|||||||
protected:
|
protected:
|
||||||
static std::vector<const DistortionScoreProducer*> s_staticColl;
|
static std::vector<const DistortionScoreProducer*> s_staticColl;
|
||||||
|
|
||||||
|
FactorType m_sparseFactorTypeSource;
|
||||||
|
FactorType m_sparseFactorTypeTarget;
|
||||||
|
bool m_useSparse;
|
||||||
|
bool m_sparseDistance;
|
||||||
|
bool m_sparseSubordinate;
|
||||||
|
FactorType m_sparseFactorTypeTargetSubordinate;
|
||||||
|
const Factor* m_subordinateConjunctionTagFactor;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
static const std::vector<const DistortionScoreProducer*>& GetDistortionFeatureFunctions() {
|
static const std::vector<const DistortionScoreProducer*>& GetDistortionFeatureFunctions() {
|
||||||
return s_staticColl;
|
return s_staticColl;
|
||||||
@ -26,6 +29,8 @@ public:
|
|||||||
|
|
||||||
DistortionScoreProducer(const std::string &line);
|
DistortionScoreProducer(const std::string &line);
|
||||||
|
|
||||||
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const {
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -44,7 +49,7 @@ public:
|
|||||||
const ChartHypothesis& /* cur_hypo */,
|
const ChartHypothesis& /* cur_hypo */,
|
||||||
int /* featureID - used to index the state in the previous hypotheses */,
|
int /* featureID - used to index the state in the previous hypotheses */,
|
||||||
ScoreComponentCollection*) const {
|
ScoreComponentCollection*) const {
|
||||||
throw std::logic_error("DistortionScoreProducer not supported in chart decoder, yet");
|
UTIL_THROW(util::Exception, "DIstortion not implemented in chart decoder");
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
Loading…
Reference in New Issue
Block a user