moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-05 15:58:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Hiero phrase orientation

Browse Source
This commit is contained in:
Matthias Huck 2016-01-06 18:52:14 +00:00
parent f6b0d988d9
commit dda3f1867c
4 changed files with 53 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
moses/FF/PhraseOrientationFeature.cpp
View File

@ -84,6 +84,8 @@ PhraseOrientationFeature::PhraseOrientationFeature(const std::string &line)
: StatefulFeatureFunction(6, line)
, m_glueLabelStr("Q")
, m_distinguishStates(false)
, m_lookaheadScore(true)
, m_heuristicScoreUseWeights(true)
, m_useSparseWord(false)
, m_useSparseNT(false)
, m_offsetR2LScores(m_numScoreComponents/2)
@ -104,6 +106,10 @@ void PhraseOrientationFeature::SetParameter(const std::string& key, const std::s
m_glueLabelStr = value;
} else if (key == "distinguish-states") {
m_distinguishStates = Scan<bool>(value);
} else if (key == "lookahead-score") {
m_lookaheadScore = Scan<bool>(value);
} else if (key == "heuristic-score-use-weights") {
m_heuristicScoreUseWeights = Scan<bool>(value);
} else if (key == "sparse-word") {
m_useSparseWord = Scan<bool>(value);
} else if (key == "sparse-nt") {
@ -156,10 +162,13 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source,
ScoreComponentCollection &estimatedScores) const
{
targetPhrase.SetRuleSource(source);
const Factor* targetPhraseLHS = targetPhrase.GetTargetLHS()[0];
if (const PhraseProperty *property = targetPhrase.GetProperty("Orientation")) {
const OrientationPhraseProperty *orientationPhraseProperty = static_cast<const OrientationPhraseProperty*>(property);
LookaheadScore(orientationPhraseProperty, scoreBreakdown);
if (m_lookaheadScore) {
LookaheadScore(orientationPhraseProperty, scoreBreakdown, targetPhraseLHS);
}
} else {
// abort with error message if the phrase does not translate an unknown word
UTIL_THROW_IF2(!targetPhrase.GetWord(0).IsOOV(), GetScoreProducerDescription()
@ -188,14 +197,12 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source,
// Initialize phrase orientation scoring object
MosesTraining::PhraseOrientation phraseOrientation(source.GetSize(), targetPhrase.GetSize(),
targetPhrase.GetAlignTerm(), targetPhrase.GetAlignNonTerm());
targetPhrase.GetAlignTerm(), targetPhrase.GetAlignNonTerm());
PhraseOrientationFeature::ReoClassData* reoClassData = new PhraseOrientationFeature::ReoClassData();
// Determine orientation classes of non-terminals
const Factor* targetPhraseLHS = targetPhrase.GetTargetLHS()[0];
for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
it!=targetPhrase.GetAlignNonTerm().end(); ++it) {
size_t sourceIndex = it->first;
@ -272,6 +279,7 @@ void PhraseOrientationFeature::EvaluateInIsolation(const Phrase &source,
void PhraseOrientationFeature::LookaheadScore(const OrientationPhraseProperty *orientationPhraseProperty,
ScoreComponentCollection &scoreBreakdown,
const Factor* targetPhraseLHS,
bool subtract) const
{
size_t ffScoreIndex = m_index;
@ -280,7 +288,10 @@ void PhraseOrientationFeature::LookaheadScore(const OrientationPhraseProperty *o
scoresL2R.push_back( TransformScore(orientationPhraseProperty->GetLeftToRightProbabilityMono()) );
scoresL2R.push_back( TransformScore(orientationPhraseProperty->GetLeftToRightProbabilitySwap()) );
scoresL2R.push_back( TransformScore(orientationPhraseProperty->GetLeftToRightProbabilityDiscontinuous()) );
size_t heuristicScoreIndexL2R = GetHeuristicScoreIndex(scoresL2R, 0);
size_t heuristicScoreIndexL2R = 0;
if (targetPhraseLHS != m_glueLabel) {
heuristicScoreIndexL2R = GetHeuristicScoreIndex(scoresL2R, 0);
}
if (subtract) {
scoreBreakdown.PlusEquals(ffScoreIndex+heuristicScoreIndexL2R,
@ -294,7 +305,10 @@ void PhraseOrientationFeature::LookaheadScore(const OrientationPhraseProperty *o
scoresR2L.push_back( TransformScore(orientationPhraseProperty->GetRightToLeftProbabilityMono()) );
scoresR2L.push_back( TransformScore(orientationPhraseProperty->GetRightToLeftProbabilitySwap()) );
scoresR2L.push_back( TransformScore(orientationPhraseProperty->GetRightToLeftProbabilityDiscontinuous()) );
size_t heuristicScoreIndexR2L = GetHeuristicScoreIndex(scoresR2L, m_offsetR2LScores);
size_t heuristicScoreIndexR2L = 0;
if (targetPhraseLHS != m_glueLabel) {
heuristicScoreIndexR2L = GetHeuristicScoreIndex(scoresR2L, m_offsetR2LScores);
}
if (subtract) {
scoreBreakdown.PlusEquals(ffScoreIndex+m_offsetR2LScores+heuristicScoreIndexR2L,
@ -380,7 +394,9 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
<< " R2L_Dleft " << orientationPhraseProperty->GetRightToLeftProbabilityDleft()
<< std::endl);
LookaheadScore(orientationPhraseProperty, *accumulator, true);
if (m_lookaheadScore) {
LookaheadScore(orientationPhraseProperty, *accumulator, prevTarPhrLHS, true);
}
const PhraseOrientationFeatureState* prevState =
static_cast<const PhraseOrientationFeatureState*>(prevHypo->GetFFState(featureID));
@ -609,19 +625,27 @@ size_t PhraseOrientationFeature::GetHeuristicScoreIndex(const std::vector<float>
size_t weightsVectorOffset,
const std::bitset<3> possibleFutureOrientations) const
{
if (m_weightsVector.empty()) {
m_weightsVector = StaticData::Instance().GetAllWeights().GetScoresForProducer(this);
}
std::vector<float> weightedScores;
for ( size_t i=0; i<3; ++i ) {
weightedScores.push_back( m_weightsVector[weightsVectorOffset+i] * scores[i] );
if (m_heuristicScoreUseWeights) {
if (m_weightsVector.empty()) {
m_weightsVector = StaticData::Instance().GetAllWeights().GetScoresForProducer(this);
}
for ( size_t i=0; i<3; ++i ) {
weightedScores.push_back( m_weightsVector[weightsVectorOffset+i] * scores[i] );
}
}
size_t heuristicScoreIndex = 0;
for (size_t i=1; i<3; ++i) {
if (possibleFutureOrientations[i]) {
if (weightedScores[i] > weightedScores[heuristicScoreIndex]) {
heuristicScoreIndex = i;
if (m_heuristicScoreUseWeights) {
if (weightedScores[i] > weightedScores[heuristicScoreIndex]) {
heuristicScoreIndex = i;
}
} else {
if (scores[i] > scores[heuristicScoreIndex]) {
heuristicScoreIndex = i;
}
}
}
}
@ -629,12 +653,14 @@ size_t PhraseOrientationFeature::GetHeuristicScoreIndex(const std::vector<float>
IFFEATUREVERBOSE(5) {
FEATUREVERBOSE(5, "Heuristic score computation: "
<< "heuristicScoreIndex== " << heuristicScoreIndex);
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " m_weightsVector[" << weightsVectorOffset+i << "]== " << m_weightsVector[weightsVectorOffset+i]);
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " scores[" << i << "]== " << scores[i]);
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " weightedScores[" << i << "]== " << weightedScores[i]);
if (m_heuristicScoreUseWeights) {
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " m_weightsVector[" << weightsVectorOffset+i << "]== " << m_weightsVector[weightsVectorOffset+i]);
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " weightedScores[" << i << "]== " << weightedScores[i]);
}
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " possibleFutureOrientations[" << i << "]== " << possibleFutureOrientations[i]);
if ( possibleFutureOrientations == 0x7 ) {

3
moses/FF/PhraseOrientationFeature.h
View File

@ -365,6 +365,7 @@ protected:
void LookaheadScore(const OrientationPhraseProperty *orientationPhraseProperty,
ScoreComponentCollection &scoreBreakdown,
const Factor* targetPhraseLHS,
bool subtract=false) const;
size_t GetHeuristicScoreIndex(const std::vector<float>& scores,
@ -408,6 +409,8 @@ protected:
std::string m_glueLabelStr;
const Factor* m_glueLabel;
bool m_distinguishStates;
bool m_lookaheadScore;
bool m_heuristicScoreUseWeights;
bool m_useSparseWord;
bool m_useSparseNT;
size_t m_offsetR2LScores;

10
phrase-extract/extract-ghkm/ExtractGHKM.cpp
View File

@ -685,7 +685,7 @@ void ExtractGHKM::WriteGlueGrammar(
out << " {{SourceLabels 2 1 " << sourceLabelSentenceStart << " 1 1 " << sourceLabelGlueTop << " 1}}";
}
if (options.phraseOrientation) {
out << " {{Orientation 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25}}";
out << " {{Orientation 1 1 0.5 0.5 1 1 0.5 0.5}}";
}
out << std::endl;
@ -700,7 +700,7 @@ void ExtractGHKM::WriteGlueGrammar(
out << " {{SourceLabels 4 1 " << sourceLabelSentenceStart << " " << sourceLabelGlueTop << " " << sourceLabelSentenceEnd << " 1 1 " << sourceLabelGlueTop << " 1}}";
}
if (options.phraseOrientation) {
out << " {{Orientation 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25}}";
out << " {{Orientation 1 1 0.5 0.5 1 1 0.5 0.5}}";
}
out << std::endl;
@ -718,7 +718,7 @@ void ExtractGHKM::WriteGlueGrammar(
out << " {{SourceLabels 4 1 " << sourceLabelSentenceStart << " " << sourceLabelGlueX << " " << sourceLabelSentenceEnd << " 1 1 " << sourceLabelGlueTop << " 1}}";
}
if (options.phraseOrientation) {
out << " {{Orientation 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25}}";
out << " {{Orientation 1 1 0.5 0.5 1 1 0.5 0.5}}";
}
out << std::endl;
}
@ -734,7 +734,7 @@ void ExtractGHKM::WriteGlueGrammar(
out << " {{SourceLabels 3 1 " << sourceLabelGlueTop << " " << sourceLabelGlueX << " 1 1 " << sourceLabelGlueTop << " 1}}";
}
if (options.phraseOrientation) {
out << " {{Orientation 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25}}";
out << " {{Orientation 1 1 0.5 0.5 1 1 0.5 0.5}}";
}
out << std::endl;
}
@ -748,7 +748,7 @@ void ExtractGHKM::WriteGlueGrammar(
out << " {{SourceLabels 3 1 " << sourceLabelGlueTop << " " << sourceLabelGlueX << " 1 1 " << sourceLabelGlueTop << " 1}}";
}
if (options.phraseOrientation) {
out << " {{Orientation 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25}}";
out << " {{Orientation 1 1 0.5 0.5 1 1 0.5 0.5}}";
}
out << std::endl;
}

2
phrase-extract/extract-rules-main.cpp
View File

@ -1165,7 +1165,7 @@ void writeGlueGrammar( const string & fileName, RuleExtractionOptions &options,
grammarFile.open(fileName.c_str());
std::string glueRulesPhraseProperty = "";
if (options.phraseOrientation) {
glueRulesPhraseProperty.append(" ||| ||| {{Orientation 0.25 0.25 0.25 0.25 0.25 0.25 0.25 0.25}}");
glueRulesPhraseProperty.append(" ||| ||| {{Orientation 1 1 0.5 0.5 1 1 0.5 0.5}}");
}
if (!options.targetSyntax) {
grammarFile << "<s> [X] ||| <s> [S] ||| 1 ||| 0-0 ||| 0" << glueRulesPhraseProperty << endl