mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
TargetPreferencesFeature
This commit is contained in:
parent
7764d86fcb
commit
3d051db850
@ -42,6 +42,7 @@
|
|||||||
#include "moses/FF/ControlRecombination.h"
|
#include "moses/FF/ControlRecombination.h"
|
||||||
#include "moses/FF/ConstrainedDecoding.h"
|
#include "moses/FF/ConstrainedDecoding.h"
|
||||||
#include "moses/FF/SoftSourceSyntacticConstraintsFeature.h"
|
#include "moses/FF/SoftSourceSyntacticConstraintsFeature.h"
|
||||||
|
#include "moses/FF/TargetPreferencesFeature.h"
|
||||||
#include "moses/FF/CoveredReferenceFeature.h"
|
#include "moses/FF/CoveredReferenceFeature.h"
|
||||||
#include "moses/FF/TreeStructureFeature.h"
|
#include "moses/FF/TreeStructureFeature.h"
|
||||||
#include "moses/FF/SoftMatchingFeature.h"
|
#include "moses/FF/SoftMatchingFeature.h"
|
||||||
@ -254,6 +255,7 @@ FeatureRegistry::FeatureRegistry()
|
|||||||
MOSES_FNAME(CoveredReferenceFeature);
|
MOSES_FNAME(CoveredReferenceFeature);
|
||||||
MOSES_FNAME(SourceGHKMTreeInputMatchFeature);
|
MOSES_FNAME(SourceGHKMTreeInputMatchFeature);
|
||||||
MOSES_FNAME(SoftSourceSyntacticConstraintsFeature);
|
MOSES_FNAME(SoftSourceSyntacticConstraintsFeature);
|
||||||
|
MOSES_FNAME(TargetPreferencesFeature);
|
||||||
MOSES_FNAME(TreeStructureFeature);
|
MOSES_FNAME(TreeStructureFeature);
|
||||||
MOSES_FNAME(SoftMatchingFeature);
|
MOSES_FNAME(SoftMatchingFeature);
|
||||||
MOSES_FNAME(DynamicCacheBasedLanguageModel);
|
MOSES_FNAME(DynamicCacheBasedLanguageModel);
|
||||||
|
403
moses/FF/TargetPreferencesFeature.cpp
Normal file
403
moses/FF/TargetPreferencesFeature.cpp
Normal file
@ -0,0 +1,403 @@
|
|||||||
|
#include <vector>
|
||||||
|
#include <limits>
|
||||||
|
#include <boost/math/special_functions/fpclassify.hpp>
|
||||||
|
#include <assert.h>
|
||||||
|
#include "TargetPreferencesFeature.h"
|
||||||
|
#include "moses/StaticData.h"
|
||||||
|
#include "moses/InputFileStream.h"
|
||||||
|
#include "moses/ScoreComponentCollection.h"
|
||||||
|
#include "moses/Hypothesis.h"
|
||||||
|
#include "moses/ChartHypothesis.h"
|
||||||
|
#include "moses/ChartManager.h"
|
||||||
|
#include "moses/FactorCollection.h"
|
||||||
|
#include "moses/TreeInput.h"
|
||||||
|
#include "moses/PP/TargetPreferencesPhraseProperty.h"
|
||||||
|
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
namespace Moses
|
||||||
|
{
|
||||||
|
|
||||||
|
void TargetPreferencesFeatureState::AddProbabilityForLHSLabel(size_t label, double cost) {
|
||||||
|
std::pair< std::map<size_t,double>::iterator, bool > inserted =
|
||||||
|
m_probabilitiesForLHSLabels.insert(std::pair<size_t,double>(label,cost));
|
||||||
|
if ( !inserted.second ) {
|
||||||
|
(inserted.first)->second += cost;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void TargetPreferencesFeatureState::NormalizeProbabilitiesForLHSLabels(double denominator) {
|
||||||
|
for ( std::map<size_t,double>::iterator iter=m_probabilitiesForLHSLabels.begin();
|
||||||
|
iter!=m_probabilitiesForLHSLabels.end(); ++iter ) {
|
||||||
|
(iter->second) /= denominator;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
double TargetPreferencesFeatureState::GetProbabilityForLHSLabel(size_t label, bool &isMatch) const {
|
||||||
|
std::map<size_t,double>::const_iterator iter = m_probabilitiesForLHSLabels.find(label);
|
||||||
|
if ( iter != m_probabilitiesForLHSLabels.end() ) {
|
||||||
|
isMatch = true;
|
||||||
|
return iter->second;
|
||||||
|
}
|
||||||
|
isMatch = false;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t TargetPreferencesFeatureState::hash() const {
|
||||||
|
if (!m_distinguishStates) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
size_t ret = 0;
|
||||||
|
boost::hash_combine(ret, m_probabilitiesForLHSLabels.size());
|
||||||
|
for (std::map<size_t,double>::const_iterator it=m_probabilitiesForLHSLabels.begin();
|
||||||
|
it!=m_probabilitiesForLHSLabels.end(); ++it) {
|
||||||
|
boost::hash_combine(ret, it->first);
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
};
|
||||||
|
|
||||||
|
bool TargetPreferencesFeatureState::operator==(const FFState& other) const {
|
||||||
|
if (!m_distinguishStates) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this == &other) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const TargetPreferencesFeatureState* otherState =
|
||||||
|
dynamic_cast<const TargetPreferencesFeatureState*>(&other);
|
||||||
|
UTIL_THROW_IF2(otherState == NULL, "Wrong state type");
|
||||||
|
|
||||||
|
if (m_probabilitiesForLHSLabels.size() != (otherState->m_probabilitiesForLHSLabels).size()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
std::map<size_t,double>::const_iterator thisIt, otherIt;
|
||||||
|
for (thisIt=m_probabilitiesForLHSLabels.begin(), otherIt=(otherState->m_probabilitiesForLHSLabels).begin();
|
||||||
|
thisIt!=m_probabilitiesForLHSLabels.end(); ++thisIt, ++otherIt) {
|
||||||
|
if (thisIt->first != otherIt->first) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
TargetPreferencesFeature::TargetPreferencesFeature(const std::string &line)
|
||||||
|
: StatefulFeatureFunction(2, line)
|
||||||
|
, m_featureVariant(0)
|
||||||
|
, m_distinguishStates(false)
|
||||||
|
, m_noMismatches(false)
|
||||||
|
{
|
||||||
|
VERBOSE(1, "Initializing feature " << GetScoreProducerDescription() << " ...");
|
||||||
|
ReadParameters();
|
||||||
|
VERBOSE(1, " Done." << std::endl);
|
||||||
|
VERBOSE(1, " Feature variant: " << m_featureVariant << "." << std::endl);
|
||||||
|
}
|
||||||
|
|
||||||
|
TargetPreferencesFeature::~TargetPreferencesFeature()
|
||||||
|
{}
|
||||||
|
|
||||||
|
void TargetPreferencesFeature::SetParameter(const std::string& key, const std::string& value)
|
||||||
|
{
|
||||||
|
if (key == "label-set-file") {
|
||||||
|
m_labelSetFile = value;
|
||||||
|
} else if (key == "unknown-word-labels-file") {
|
||||||
|
m_unknownLeftHandSideFile = value;
|
||||||
|
} else if (key == "variant") {
|
||||||
|
m_featureVariant = Scan<size_t>(value);
|
||||||
|
} else if (key == "distinguish-states") {
|
||||||
|
m_distinguishStates = Scan<bool>(value);
|
||||||
|
} else if (key == "no-mismatches") {
|
||||||
|
m_noMismatches = Scan<bool>(value);
|
||||||
|
} else {
|
||||||
|
StatefulFeatureFunction::SetParameter(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void TargetPreferencesFeature::Load(AllOptions::ptr const& opts)
|
||||||
|
{
|
||||||
|
// don't change the loading order!
|
||||||
|
LoadLabelSet();
|
||||||
|
LoadUnknownLeftHandSideFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
void TargetPreferencesFeature::LoadLabelSet()
|
||||||
|
{
|
||||||
|
FEATUREVERBOSE(2, "Loading label set from file " << m_labelSetFile << " ...");
|
||||||
|
InputFileStream inFile(m_labelSetFile);
|
||||||
|
|
||||||
|
// read label set
|
||||||
|
std::string line;
|
||||||
|
m_labels.clear();
|
||||||
|
m_labelsByIndex.clear();
|
||||||
|
while (getline(inFile, line)) {
|
||||||
|
std::istringstream tokenizer(line);
|
||||||
|
std::string label;
|
||||||
|
size_t index;
|
||||||
|
try {
|
||||||
|
tokenizer >> label >> index;
|
||||||
|
} catch (const std::exception &e) {
|
||||||
|
UTIL_THROW2(GetScoreProducerDescription()
|
||||||
|
<< ": Error reading label set file " << m_labelSetFile << " .");
|
||||||
|
}
|
||||||
|
std::pair< boost::unordered_map<std::string,size_t>::iterator, bool > inserted = m_labels.insert( std::pair<std::string,size_t>(label,index) );
|
||||||
|
UTIL_THROW_IF2(!inserted.second, GetScoreProducerDescription()
|
||||||
|
<< ": Label set file " << m_labelSetFile << " should contain each label only once.");
|
||||||
|
|
||||||
|
if (index >= m_labelsByIndex.size()) {
|
||||||
|
m_labelsByIndex.resize(index+1);
|
||||||
|
}
|
||||||
|
m_labelsByIndex[index] = label;
|
||||||
|
}
|
||||||
|
|
||||||
|
inFile.Close();
|
||||||
|
|
||||||
|
std::list<std::string> specialLabels;
|
||||||
|
specialLabels.push_back("GlueTop");
|
||||||
|
for (std::list<std::string>::const_iterator iter=specialLabels.begin();
|
||||||
|
iter!=specialLabels.end(); ++iter) {
|
||||||
|
boost::unordered_map<std::string,size_t>::iterator found = m_labels.find(*iter);
|
||||||
|
UTIL_THROW_IF2(found == m_labels.end(), GetScoreProducerDescription()
|
||||||
|
<< ": Label set file " << m_labelSetFile << " should contain an entry for the special label \"" << *iter << "\".");
|
||||||
|
if (!(found->first).compare("GlueTop")) {
|
||||||
|
m_GlueTopLabel = found->second;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FEATUREVERBOSE2(2, " Done." << std::endl);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make sure to call this method _after_ LoadLabelSet()
|
||||||
|
void TargetPreferencesFeature::LoadUnknownLeftHandSideFile()
|
||||||
|
{
|
||||||
|
FEATUREVERBOSE(2, "Loading left-hand side labels for unknowns from file " << m_unknownLeftHandSideFile << std::endl);
|
||||||
|
InputFileStream inFile(m_unknownLeftHandSideFile);
|
||||||
|
|
||||||
|
// read left-hand side labels for unknowns
|
||||||
|
std::string line;
|
||||||
|
m_unknownLHSProbabilities.clear();
|
||||||
|
double countsSum = 0.0;
|
||||||
|
while (getline(inFile, line)) {
|
||||||
|
istringstream tokenizer(line);
|
||||||
|
std::string label;
|
||||||
|
double count;
|
||||||
|
tokenizer >> label;
|
||||||
|
tokenizer >> count;
|
||||||
|
boost::unordered_map<std::string,size_t>::iterator found = m_labels.find( label );
|
||||||
|
if ( found != m_labels.end() ) {
|
||||||
|
std::pair< std::map<size_t,double>::iterator, bool > inserted =
|
||||||
|
m_unknownLHSProbabilities.insert( std::pair<size_t,double>(found->second,count) );
|
||||||
|
if ( !inserted.second ) {
|
||||||
|
(inserted.first)->second += count;
|
||||||
|
}
|
||||||
|
countsSum += count;
|
||||||
|
} else {
|
||||||
|
FEATUREVERBOSE(1, "WARNING: undefined label \"" << label << "\" in file " << m_unknownLeftHandSideFile << std::endl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// compute probabilities from counts
|
||||||
|
countsSum += (double)m_labels.size();
|
||||||
|
for (std::map<size_t,double>::iterator iter=m_unknownLHSProbabilities.begin();
|
||||||
|
iter!=m_unknownLHSProbabilities.end(); ++iter) {
|
||||||
|
iter->second /= countsSum;
|
||||||
|
}
|
||||||
|
|
||||||
|
IFFEATUREVERBOSE(3) {
|
||||||
|
for (std::map<size_t,double>::iterator iter=m_unknownLHSProbabilities.begin();
|
||||||
|
iter!=m_unknownLHSProbabilities.end(); ++iter) {
|
||||||
|
FEATUREVERBOSE(3, GetScoreProducerDescription() << "::LoadUnknownLeftHandSideFile(): " << iter->first << " " << iter->second << std::endl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
inFile.Close();
|
||||||
|
}
|
||||||
|
|
||||||
|
FFState* TargetPreferencesFeature::EvaluateWhenApplied(
|
||||||
|
const ChartHypothesis& hypo,
|
||||||
|
int featureID, // used to index the state in the previous hypotheses
|
||||||
|
ScoreComponentCollection* accumulator) const
|
||||||
|
{
|
||||||
|
streamsize cerr_precision = std::cerr.precision();
|
||||||
|
std::cerr.precision(20); // TODO: remove. just for debug purposes.
|
||||||
|
|
||||||
|
// dense scores
|
||||||
|
std::vector<float> newScores(m_numScoreComponents,0); // m_numScoreComponents == 2
|
||||||
|
|
||||||
|
// state: used to store tree probabilities of partial hypotheses
|
||||||
|
// and access the respective tree probabilities of subderivations
|
||||||
|
TargetPreferencesFeatureState *state = new TargetPreferencesFeatureState(m_distinguishStates);
|
||||||
|
|
||||||
|
size_t nNTs = 1;
|
||||||
|
double overallTreeProbability = 0.0;
|
||||||
|
bool isGlueGrammarRule = false;
|
||||||
|
|
||||||
|
// read TargetPreferences property
|
||||||
|
const TargetPhrase &currTarPhr = hypo.GetCurrTargetPhrase();
|
||||||
|
|
||||||
|
FEATUREVERBOSE(2, "Phrase: " << currTarPhr << std::endl);
|
||||||
|
|
||||||
|
if (const PhraseProperty *property = currTarPhr.GetProperty("TargetPreferences")) {
|
||||||
|
|
||||||
|
const TargetPreferencesPhraseProperty *targetPreferencesPhraseProperty = static_cast<const TargetPreferencesPhraseProperty*>(property);
|
||||||
|
|
||||||
|
// IFFEATUREVERBOSE(2) {
|
||||||
|
// const std::string *targetPreferencesPhrasePropertyValueString = targetPreferencesPhraseProperty->GetValueString();
|
||||||
|
// if (targetPreferencesPhrasePropertyValueString) {
|
||||||
|
// FEATUREVERBOSE(2, "PreferencesPhraseProperty " << *targetPreferencesPhrasePropertyValueString << std::endl);
|
||||||
|
// } else {
|
||||||
|
// FEATUREVERBOSE(2, "PreferencesPhraseProperty NULL" << std::endl);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
nNTs = targetPreferencesPhraseProperty->GetNumberOfNonTerminals();
|
||||||
|
double totalCount = targetPreferencesPhraseProperty->GetTotalCount();
|
||||||
|
|
||||||
|
// get index map for underlying hypotheses
|
||||||
|
const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
|
||||||
|
currTarPhr.GetAlignNonTerm().GetNonTermIndexMap();
|
||||||
|
|
||||||
|
// retrieve states from previous hypotheses, if any
|
||||||
|
std::vector< const TargetPreferencesFeatureState* > prevStatesByNonTerminal(nNTs-1);
|
||||||
|
|
||||||
|
if (nNTs > 1) { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
|
||||||
|
size_t nonTerminalNumber = 0;
|
||||||
|
|
||||||
|
for (size_t phrasePos=0; phrasePos<currTarPhr.GetSize(); ++phrasePos) {
|
||||||
|
// consult rule for either word or non-terminal
|
||||||
|
const Word &word = currTarPhr.GetWord(phrasePos);
|
||||||
|
if ( word.IsNonTerminal() ) {
|
||||||
|
// non-terminal: consult subderivation
|
||||||
|
size_t nonTermIndex = nonTermIndexMap[phrasePos];
|
||||||
|
const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndex);
|
||||||
|
const TargetPreferencesFeatureState* prevState =
|
||||||
|
static_cast<const TargetPreferencesFeatureState*>(prevHypo->GetFFState(featureID));
|
||||||
|
prevStatesByNonTerminal[nonTerminalNumber] = prevState;
|
||||||
|
|
||||||
|
IFFEATUREVERBOSE(2) {
|
||||||
|
// some log output that is not required in any way for the functionality
|
||||||
|
const std::map<size_t,double> &prevHypoTreeProbabilities =
|
||||||
|
prevStatesByNonTerminal[nonTerminalNumber]->GetProbabilitiesForLHSLabels();
|
||||||
|
FEATUREVERBOSE(2, "Previous tree probs:");
|
||||||
|
for (std::map<size_t,double>::const_iterator iter=prevHypoTreeProbabilities.begin();
|
||||||
|
iter!=prevHypoTreeProbabilities.end(); ++iter) {
|
||||||
|
FEATUREVERBOSE2(2, " " << m_labelsByIndex[iter->first] << " " << iter->second);
|
||||||
|
}
|
||||||
|
FEATUREVERBOSE2(2, std::endl);
|
||||||
|
}
|
||||||
|
|
||||||
|
++nonTerminalNumber;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// inspect labelled rule items
|
||||||
|
|
||||||
|
overallTreeProbability = 0.0;
|
||||||
|
|
||||||
|
const std::list<TargetPreferencesPhrasePropertyItem> &targetPreferencesItems = targetPreferencesPhraseProperty->GetTargetPreferencesItems();
|
||||||
|
|
||||||
|
for (std::list<TargetPreferencesPhrasePropertyItem>::const_iterator targetPreferencesItem = targetPreferencesItems.begin();
|
||||||
|
targetPreferencesItem != targetPreferencesItems.end(); ++targetPreferencesItem) {
|
||||||
|
|
||||||
|
const std::list<size_t> &targetPreferencesRHS = targetPreferencesItem->GetTargetPreferencesRHS();
|
||||||
|
const std::list< std::pair<size_t,float> > &targetPreferencesLHSList = targetPreferencesItem->GetTargetPreferencesLHSList();
|
||||||
|
|
||||||
|
assert(targetPreferencesRHS.size() == nNTs-1);
|
||||||
|
|
||||||
|
size_t currentTargetLabelsMismatches = nNTs - 1;
|
||||||
|
double matchingLabelsProbabilityProduct = 1.0;
|
||||||
|
|
||||||
|
size_t nonTerminalNumber=0;
|
||||||
|
for (std::list<size_t>::const_iterator targetPreferencesRHSIt = targetPreferencesRHS.begin();
|
||||||
|
targetPreferencesRHSIt != targetPreferencesRHS.end(); ++targetPreferencesRHSIt, ++nonTerminalNumber) {
|
||||||
|
|
||||||
|
bool isLabelMatch = false;
|
||||||
|
double matchingLabelsProbability =
|
||||||
|
prevStatesByNonTerminal[nonTerminalNumber]->GetProbabilityForLHSLabel(*targetPreferencesRHSIt,
|
||||||
|
isLabelMatch);
|
||||||
|
matchingLabelsProbabilityProduct *= matchingLabelsProbability;
|
||||||
|
|
||||||
|
if ( isLabelMatch ) {
|
||||||
|
currentTargetLabelsMismatches -= 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FEATUREVERBOSE(2, "matchingLabelsProbabilityProduct = " << matchingLabelsProbabilityProduct << std::endl);
|
||||||
|
|
||||||
|
// LHS labels seen with this RHS
|
||||||
|
for (std::list< std::pair<size_t,float> >::const_iterator targetPreferencesLHSIt = targetPreferencesLHSList.begin();
|
||||||
|
targetPreferencesLHSIt != targetPreferencesLHSList.end(); ++targetPreferencesLHSIt) {
|
||||||
|
|
||||||
|
size_t targetPreferenceLHS = targetPreferencesLHSIt->first;
|
||||||
|
|
||||||
|
if ( targetPreferenceLHS == m_GlueTopLabel ) {
|
||||||
|
isGlueGrammarRule = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// proceed with the actual probability computations
|
||||||
|
double ruleTargetPreferenceCount = targetPreferencesLHSIt->second;
|
||||||
|
double ruleTargetPreferenceProbability = ruleTargetPreferenceCount / totalCount;
|
||||||
|
|
||||||
|
FEATUREVERBOSE(2, " ruleTargetPreferenceProbability = " << ruleTargetPreferenceProbability << std::endl);
|
||||||
|
|
||||||
|
double weightedTargetPreferenceRuleProbability = ruleTargetPreferenceProbability * matchingLabelsProbabilityProduct;
|
||||||
|
if ( weightedTargetPreferenceRuleProbability != 0 ) {
|
||||||
|
state->AddProbabilityForLHSLabel(targetPreferenceLHS, weightedTargetPreferenceRuleProbability);
|
||||||
|
}
|
||||||
|
overallTreeProbability += weightedTargetPreferenceRuleProbability;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
IFFEATUREVERBOSE(2) {
|
||||||
|
FEATUREVERBOSE(2, "overallTreeProbability = " << overallTreeProbability);
|
||||||
|
if ( overallTreeProbability > 1.0001 ) { // account for some rounding error
|
||||||
|
FEATUREVERBOSE2(2, " -- WARNING: overallTreeProbability > 1");
|
||||||
|
}
|
||||||
|
FEATUREVERBOSE2(2, std::endl);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ( overallTreeProbability != 0 ) {
|
||||||
|
UTIL_THROW_IF2(!boost::math::isnormal(overallTreeProbability), GetScoreProducerDescription()
|
||||||
|
<< ": Oops. Numerical precision issues.");
|
||||||
|
state->NormalizeProbabilitiesForLHSLabels(overallTreeProbability);
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
|
||||||
|
// abort with error message if the phrase does not translate an unknown word
|
||||||
|
UTIL_THROW_IF2(!currTarPhr.GetWord(0).IsOOV(), GetScoreProducerDescription()
|
||||||
|
<< ": Missing TargetPreferences property. Please check phrase table and glue rules.");
|
||||||
|
|
||||||
|
// unknown word
|
||||||
|
overallTreeProbability = 1.0;
|
||||||
|
|
||||||
|
for (std::map<size_t,double>::const_iterator iter=m_unknownLHSProbabilities.begin();
|
||||||
|
iter!=m_unknownLHSProbabilities.end(); ++iter) {
|
||||||
|
// update state
|
||||||
|
state->AddProbabilityForLHSLabel(iter->first, iter->second);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FEATUREVERBOSE(2, "-> OVERALLTREEPROB = " << overallTreeProbability << std::endl);
|
||||||
|
|
||||||
|
// add scores
|
||||||
|
|
||||||
|
// tree probability (preference grammar style)
|
||||||
|
newScores[0] = (overallTreeProbability == 0 ? 0 : std::log(overallTreeProbability) );
|
||||||
|
if ( m_noMismatches && (overallTreeProbability == 0) && !isGlueGrammarRule ) {
|
||||||
|
newScores[0] = -std::numeric_limits<float>::infinity();
|
||||||
|
}
|
||||||
|
// tree mismatch penalty
|
||||||
|
// TODO: deactivate the tree mismatch penalty score component automatically if feature configuration parameter no-mismatches=true
|
||||||
|
newScores[1] = (overallTreeProbability == 0 ? 1 : 0 );
|
||||||
|
|
||||||
|
accumulator->PlusEquals(this, newScores);
|
||||||
|
|
||||||
|
std::cerr.precision(cerr_precision);
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
121
moses/FF/TargetPreferencesFeature.h
Normal file
121
moses/FF/TargetPreferencesFeature.h
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <map>
|
||||||
|
#include <iostream>
|
||||||
|
#include <boost/unordered_map.hpp>
|
||||||
|
#include "StatefulFeatureFunction.h"
|
||||||
|
#include "FFState.h"
|
||||||
|
#include "util/exception.hh"
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
namespace Moses
|
||||||
|
{
|
||||||
|
|
||||||
|
class TargetPreferencesFeatureState : public FFState
|
||||||
|
{
|
||||||
|
|
||||||
|
public:
|
||||||
|
|
||||||
|
TargetPreferencesFeatureState(bool distinguishStates)
|
||||||
|
: m_distinguishStates(distinguishStates)
|
||||||
|
{}
|
||||||
|
|
||||||
|
void AddProbabilityForLHSLabel(size_t label, double cost);
|
||||||
|
|
||||||
|
void NormalizeProbabilitiesForLHSLabels(double denominator);
|
||||||
|
|
||||||
|
const std::map<size_t,double> &GetProbabilitiesForLHSLabels() const {
|
||||||
|
return m_probabilitiesForLHSLabels;
|
||||||
|
}
|
||||||
|
|
||||||
|
double GetProbabilityForLHSLabel(size_t label, bool &isMatch) const;
|
||||||
|
|
||||||
|
size_t hash() const;
|
||||||
|
|
||||||
|
virtual bool operator==(const FFState& other) const;
|
||||||
|
|
||||||
|
|
||||||
|
private:
|
||||||
|
|
||||||
|
const bool m_distinguishStates;
|
||||||
|
std::map<size_t,double> m_probabilitiesForLHSLabels;
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
class TargetPreferencesFeature : public StatefulFeatureFunction
|
||||||
|
{
|
||||||
|
|
||||||
|
public:
|
||||||
|
|
||||||
|
TargetPreferencesFeature(const std::string &line);
|
||||||
|
|
||||||
|
~TargetPreferencesFeature();
|
||||||
|
|
||||||
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
|
||||||
|
return new TargetPreferencesFeatureState(m_distinguishStates);
|
||||||
|
}
|
||||||
|
|
||||||
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
|
void EvaluateInIsolation(const Phrase &source
|
||||||
|
, const TargetPhrase &targetPhrase
|
||||||
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
|
, ScoreComponentCollection &estimatedFutureScore) const
|
||||||
|
{};
|
||||||
|
|
||||||
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
|
, const InputPath &inputPath
|
||||||
|
, const TargetPhrase &targetPhrase
|
||||||
|
, const StackVec *stackVec
|
||||||
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||||
|
{};
|
||||||
|
|
||||||
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
|
, const TranslationOptionList &translationOptionList) const
|
||||||
|
{}
|
||||||
|
|
||||||
|
FFState* EvaluateWhenApplied(
|
||||||
|
const Hypothesis& cur_hypo,
|
||||||
|
const FFState* prev_state,
|
||||||
|
ScoreComponentCollection* accumulator) const {
|
||||||
|
UTIL_THROW2(GetScoreProducerDescription() << ": feature currently not implemented for phrase-based decoding.");
|
||||||
|
return new TargetPreferencesFeatureState(m_distinguishStates);
|
||||||
|
};
|
||||||
|
|
||||||
|
FFState* EvaluateWhenApplied(
|
||||||
|
const ChartHypothesis& cur_hypo,
|
||||||
|
int featureID, // used to index the state in the previous hypotheses
|
||||||
|
ScoreComponentCollection* accumulator) const;
|
||||||
|
|
||||||
|
|
||||||
|
private:
|
||||||
|
|
||||||
|
std::string m_labelSetFile;
|
||||||
|
std::string m_unknownLeftHandSideFile;
|
||||||
|
size_t m_featureVariant;
|
||||||
|
bool m_distinguishStates;
|
||||||
|
bool m_noMismatches;
|
||||||
|
|
||||||
|
mutable boost::unordered_map<std::string,size_t> m_labels;
|
||||||
|
mutable std::vector<std::string> m_labelsByIndex;
|
||||||
|
mutable size_t m_XRHSLabel;
|
||||||
|
mutable size_t m_XLHSLabel;
|
||||||
|
mutable size_t m_GlueTopLabel;
|
||||||
|
std::map<size_t,double> m_unknownLHSProbabilities;
|
||||||
|
|
||||||
|
void LoadLabelSet();
|
||||||
|
void LoadUnknownLeftHandSideFile();
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user