2015-11-01 17:51:33 +03:00
|
|
|
// -*- mode: c++; indent-tabs-mode: nil; tab-width:2 -*-
|
2008-06-11 14:52:57 +04:00
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2006 University of Edinburgh
|
|
|
|
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
|
|
|
|
2010-02-24 14:15:44 +03:00
|
|
|
#ifndef moses_ScoreComponentCollection_h
|
|
|
|
#define moses_ScoreComponentCollection_h
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
#include <numeric>
|
2011-08-07 04:58:56 +04:00
|
|
|
#include <sstream>
|
2010-08-10 17:12:00 +04:00
|
|
|
|
2010-09-28 19:13:50 +04:00
|
|
|
#ifdef MPI_ENABLE
|
|
|
|
#include <boost/serialization/access.hpp>
|
|
|
|
#include <boost/serialization/split_member.hpp>
|
|
|
|
#endif
|
|
|
|
|
2013-05-24 21:02:49 +04:00
|
|
|
#include "moses/FF/FeatureFunction.h"
|
2010-09-15 15:30:25 +04:00
|
|
|
#include "FeatureVector.h"
|
2008-06-11 14:52:57 +04:00
|
|
|
#include "TypeDef.h"
|
|
|
|
#include "Util.h"
|
2014-01-13 18:37:05 +04:00
|
|
|
#include "util/exception.hh"
|
2010-09-28 19:13:50 +04:00
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
|
|
|
|
2013-09-08 17:57:31 +04:00
|
|
|
/**
|
|
|
|
* Smaller version for just 1 FF.
|
|
|
|
*/
|
2013-09-27 12:35:24 +04:00
|
|
|
struct ScorePair {
|
|
|
|
friend std::ostream& operator<<(std::ostream& os, const ScorePair& rhs);
|
|
|
|
|
|
|
|
std::vector<float> denseScores;
|
|
|
|
std::map<StringPiece, float> sparseScores;
|
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
ScorePair() {
|
|
|
|
}
|
2013-09-27 12:35:24 +04:00
|
|
|
ScorePair(const std::vector<float> &other)
|
2015-01-14 14:07:42 +03:00
|
|
|
:denseScores(other) {
|
|
|
|
}
|
2013-09-27 12:35:24 +04:00
|
|
|
|
|
|
|
void PlusEquals(const ScorePair &other);
|
|
|
|
void PlusEquals(const StringPiece &key, float value);
|
|
|
|
|
|
|
|
void PlusEquals(const std::vector<float> &other) {
|
2013-11-21 21:51:01 +04:00
|
|
|
UTIL_THROW_IF2(denseScores.size() != other.size(), "Number of scores incorrect");
|
2013-09-27 12:35:24 +04:00
|
|
|
std::transform(denseScores.begin(),
|
|
|
|
denseScores.end(),
|
|
|
|
other.begin(),
|
|
|
|
denseScores.begin(),
|
|
|
|
std::plus<float>());
|
|
|
|
}
|
2013-09-08 17:57:31 +04:00
|
|
|
};
|
2010-09-28 19:13:50 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
/*** An unweighted collection of scores for a translation or step in a translation.
|
|
|
|
*
|
|
|
|
* In the factored phrase-based models that are implemented by moses, there are a set of
|
|
|
|
* scores that come from a variety of sources (translation probabilities, language model
|
|
|
|
* probablilities, distortion probabilities, generation probabilities). Furthermore, while
|
|
|
|
* some of these scores may be 0, this number is fixed (and generally quite small, ie, less
|
|
|
|
* than 15), for a given model.
|
|
|
|
*
|
|
|
|
* The values contained in ScoreComponentCollection objects are unweighted scores (log-probs).
|
2011-02-24 16:14:42 +03:00
|
|
|
*
|
2008-06-11 14:52:57 +04:00
|
|
|
* ScoreComponentCollection objects can be added and subtracted, which makes them appropriate
|
|
|
|
* to be the datatype used to return the result of a score computations (in this case they will
|
|
|
|
* have most values set to zero, except for the ones that are results of the indivudal computation
|
|
|
|
* this will then be added into the "running total" in the Hypothesis. In fact, for a score
|
|
|
|
* to be tracked in the hypothesis (and thus to participate in the decoding process), a class
|
|
|
|
* representing that score must extend the ScoreProducer abstract base class. For an example
|
|
|
|
* refer to the DistortionScoreProducer class.
|
|
|
|
*/
|
2011-02-24 16:14:42 +03:00
|
|
|
class ScoreComponentCollection
|
|
|
|
{
|
2008-06-11 14:52:57 +04:00
|
|
|
friend std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs);
|
2013-08-22 13:16:24 +04:00
|
|
|
friend void swap(ScoreComponentCollection &first, ScoreComponentCollection &second);
|
2015-11-02 03:00:37 +03:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
private:
|
2013-05-29 21:16:15 +04:00
|
|
|
FVector m_scores;
|
2013-08-22 13:16:24 +04:00
|
|
|
|
2014-06-12 21:31:56 +04:00
|
|
|
public:
|
2015-04-26 18:46:36 +03:00
|
|
|
// typedef std::pair<size_t,size_t> IndexPair;
|
2014-06-12 21:31:56 +04:00
|
|
|
private:
|
2015-04-26 18:46:36 +03:00
|
|
|
// typedef std::map<const FeatureFunction*,IndexPair> ScoreIndexMap;
|
|
|
|
// static ScoreIndexMap s_scoreIndexes;
|
2011-11-09 21:16:02 +04:00
|
|
|
static size_t s_denseVectorSize;
|
2014-06-12 21:31:56 +04:00
|
|
|
public:
|
2015-04-26 18:46:36 +03:00
|
|
|
// static IndexPair GetIndexes(const FeatureFunction* sp) {
|
|
|
|
// ScoreIndexMap::const_iterator indexIter = s_scoreIndexes.find(sp);
|
|
|
|
// if (indexIter == s_scoreIndexes.end()) {
|
|
|
|
// std::stringstream strme;
|
|
|
|
// strme << "ERROR: FeatureFunction: " << sp->GetScoreProducerDescription() <<
|
|
|
|
// " not registered with ScoreIndexMap" << std::endl;
|
|
|
|
// strme << "You must call ScoreComponentCollection.RegisterScoreProducer() " <<
|
|
|
|
// " for every FeatureFunction" << std::endl;
|
|
|
|
// UTIL_THROW2(strme.str());
|
|
|
|
// }
|
|
|
|
// return indexIter->second;
|
|
|
|
// }
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
public:
|
2012-04-29 08:37:48 +04:00
|
|
|
static void ResetCounter() {
|
|
|
|
s_denseVectorSize = 0;
|
|
|
|
}
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
//! Create a new score collection with all values set to 0.0
|
2011-02-24 16:14:42 +03:00
|
|
|
ScoreComponentCollection();
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
//! Clone a score collection
|
2013-05-29 21:16:15 +04:00
|
|
|
ScoreComponentCollection(const ScoreComponentCollection& rhs)
|
2013-06-10 21:11:55 +04:00
|
|
|
: m_scores(rhs.m_scores) {
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2012-10-16 20:50:00 +04:00
|
|
|
ScoreComponentCollection& operator=( const ScoreComponentCollection& rhs ) {
|
|
|
|
m_scores = rhs.m_scores;
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
2011-11-09 21:16:02 +04:00
|
|
|
/**
|
2013-05-29 21:16:15 +04:00
|
|
|
* Register a ScoreProducer with a fixed number of scores, so that it can
|
2011-11-09 21:16:02 +04:00
|
|
|
* be allocated space in the dense part of the feature vector.
|
|
|
|
**/
|
2015-04-26 18:46:36 +03:00
|
|
|
static void RegisterScoreProducer(FeatureFunction* scoreProducer);
|
2011-11-09 21:16:02 +04:00
|
|
|
|
2010-10-15 19:19:17 +04:00
|
|
|
/** Load from file */
|
2013-05-29 21:16:15 +04:00
|
|
|
bool Load(const std::string& filename) {
|
|
|
|
return m_scores.load(filename);
|
2010-10-15 19:19:17 +04:00
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
const FVector& GetScoresVector() const {
|
|
|
|
return m_scores;
|
2010-12-06 18:28:51 +03:00
|
|
|
}
|
2008-09-24 20:48:23 +04:00
|
|
|
|
2012-10-22 17:41:23 +04:00
|
|
|
const std::valarray<FValue> &getCoreFeatures() const {
|
|
|
|
return m_scores.getCoreFeatures();
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
size_t Size() const {
|
|
|
|
return m_scores.size();
|
2011-02-02 21:03:08 +03:00
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
void Resize() {
|
2011-11-09 21:16:02 +04:00
|
|
|
if (m_scores.coreSize() != s_denseVectorSize) {
|
|
|
|
m_scores.resize(s_denseVectorSize);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Create and FVector with the right number of core features */
|
2013-05-29 21:16:15 +04:00
|
|
|
static FVector CreateFVector() {
|
2011-11-09 21:16:02 +04:00
|
|
|
return FVector(s_denseVectorSize);
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
void SetToBinaryOf(const ScoreComponentCollection& rhs) {
|
|
|
|
m_scores.setToBinaryOf(rhs.m_scores);
|
2012-03-13 20:34:48 +04:00
|
|
|
}
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
//! Set all values to 0.0
|
2013-05-29 21:16:15 +04:00
|
|
|
void ZeroAll() {
|
|
|
|
m_scores.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
void MultiplyEquals(float scalar);
|
|
|
|
void DivideEquals(float scalar);
|
|
|
|
void CoreDivideEquals(float scalar);
|
|
|
|
void DivideEquals(const ScoreComponentCollection& rhs);
|
|
|
|
void MultiplyEquals(const ScoreComponentCollection& rhs);
|
|
|
|
void MultiplyEqualsBackoff(const ScoreComponentCollection& rhs, float backoff);
|
|
|
|
void MultiplyEquals(float core_r0, float sparse_r0);
|
|
|
|
void MultiplyEquals(const FeatureFunction* sp, float scalar);
|
|
|
|
|
|
|
|
size_t GetNumberWeights(const FeatureFunction* sp);
|
|
|
|
|
|
|
|
void CoreAssign(const ScoreComponentCollection& rhs) {
|
|
|
|
m_scores.coreAssign(rhs.m_scores);
|
|
|
|
}
|
|
|
|
|
|
|
|
//! add the score in rhs
|
|
|
|
void PlusEquals(const ScoreComponentCollection& rhs) {
|
|
|
|
m_scores += rhs.m_scores;
|
|
|
|
}
|
|
|
|
|
|
|
|
// add only sparse features
|
|
|
|
void SparsePlusEquals(const ScoreComponentCollection& rhs) {
|
|
|
|
m_scores.sparsePlusEquals(rhs.m_scores);
|
|
|
|
}
|
|
|
|
|
2015-01-07 17:25:43 +03:00
|
|
|
// add only core features
|
|
|
|
void CorePlusEquals(const ScoreComponentCollection& rhs) {
|
|
|
|
m_scores.corePlusEquals(rhs.m_scores);
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
void PlusEquals(const FVector& scores) {
|
|
|
|
m_scores += scores;
|
|
|
|
}
|
|
|
|
|
|
|
|
//! subtract the score in rhs
|
|
|
|
void MinusEquals(const ScoreComponentCollection& rhs) {
|
|
|
|
m_scores -= rhs.m_scores;
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-12-07 01:33:23 +04:00
|
|
|
//For features which have an unbounded number of components
|
2013-05-29 21:16:15 +04:00
|
|
|
void MinusEquals(const FeatureFunction*sp, const std::string& name, float score) {
|
2011-12-07 01:33:23 +04:00
|
|
|
FName fname(sp->GetScoreProducerDescription(),name);
|
|
|
|
m_scores[fname] -= score;
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-12-16 23:53:46 +04:00
|
|
|
//For features which have an unbounded number of components
|
2013-05-29 21:16:15 +04:00
|
|
|
void SparseMinusEquals(const std::string& full_name, float score) {
|
2011-12-16 23:53:46 +04:00
|
|
|
FName fname(full_name);
|
|
|
|
m_scores[fname] -= score;
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
//! Add scores from a single ScoreProducer only
|
|
|
|
//! The length of scores must be equal to the number of score components
|
|
|
|
//! produced by sp
|
2015-04-30 08:05:11 +03:00
|
|
|
void
|
|
|
|
PlusEquals(const FeatureFunction* sp,
|
2015-05-02 13:45:24 +03:00
|
|
|
const ScoreComponentCollection& scores) {
|
2015-04-26 18:46:36 +03:00
|
|
|
size_t i = sp->GetIndex();
|
|
|
|
size_t stop = i + sp->GetNumScoreComponents();
|
2015-05-02 13:45:24 +03:00
|
|
|
for (; i < stop; ++i) m_scores[i] += scores.m_scores[i];
|
2013-05-29 21:16:15 +04:00
|
|
|
}
|
2009-02-06 18:43:06 +03:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
//! Add scores from a single FeatureFunction only
|
|
|
|
//! The length of scores must be equal to the number of score components
|
|
|
|
//! produced by sp
|
|
|
|
void PlusEquals(const FeatureFunction* sp, const std::vector<float>& scores) {
|
2015-04-26 18:46:36 +03:00
|
|
|
UTIL_THROW_IF2(scores.size() != sp->GetNumScoreComponents(),
|
2014-01-15 19:42:02 +04:00
|
|
|
"Number of scores is incorrect");
|
2015-04-26 18:46:36 +03:00
|
|
|
size_t offset = sp->GetIndex();
|
2010-10-07 02:06:49 +04:00
|
|
|
for (size_t i = 0; i < scores.size(); ++i) {
|
2015-04-26 18:46:36 +03:00
|
|
|
m_scores[i + offset] += scores[i];
|
2010-10-07 02:06:49 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-10-06 17:14:56 +03:00
|
|
|
void PlusEquals(const FeatureFunction* sp, float scores[]) {
|
2016-10-05 17:15:47 +03:00
|
|
|
size_t numScores = sp->GetNumScoreComponents();
|
|
|
|
size_t offset = sp->GetIndex();
|
|
|
|
for (size_t i = 0; i < numScores; ++i) {
|
|
|
|
m_scores[i + offset] += scores[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
//! Special version PlusEquals(ScoreProducer, vector<float>)
|
|
|
|
//! to add the score from a single ScoreProducer that produces
|
|
|
|
//! a single value
|
|
|
|
void PlusEquals(const FeatureFunction* sp, float score) {
|
2015-04-26 18:46:36 +03:00
|
|
|
UTIL_THROW_IF2(sp->GetNumScoreComponents() != 1,
|
2014-01-15 19:42:02 +04:00
|
|
|
"Number of scores is incorrect");
|
2015-04-26 18:46:36 +03:00
|
|
|
m_scores[sp->GetIndex()] += score;
|
2013-05-29 21:16:15 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2013-04-29 21:46:48 +04:00
|
|
|
//For features which have an unbounded number of components
|
2013-05-29 21:16:15 +04:00
|
|
|
void PlusEquals(const FeatureFunction*sp, const StringPiece& name, float score) {
|
2010-10-15 01:52:35 +04:00
|
|
|
FName fname(sp->GetScoreProducerDescription(),name);
|
|
|
|
m_scores[fname] += score;
|
|
|
|
}
|
|
|
|
|
2013-09-08 17:57:31 +04:00
|
|
|
void PlusEquals(const FeatureFunction* sp, const ScorePair &scorePair);
|
|
|
|
|
2014-08-08 00:42:30 +04:00
|
|
|
// Add score by index
|
|
|
|
void PlusEquals(size_t index, float score) {
|
|
|
|
m_scores[index] += score;
|
|
|
|
}
|
|
|
|
|
2011-12-16 23:53:46 +04:00
|
|
|
//For features which have an unbounded number of components
|
2013-05-29 21:16:15 +04:00
|
|
|
void SparsePlusEquals(const std::string& full_name, float score) {
|
|
|
|
FName fname(full_name);
|
2011-12-16 23:53:46 +04:00
|
|
|
m_scores[fname] += score;
|
|
|
|
}
|
|
|
|
|
2014-07-25 00:23:08 +04:00
|
|
|
void SparsePlusEquals(const FName& fname, float score) {
|
|
|
|
m_scores[fname] += score;
|
|
|
|
}
|
|
|
|
|
2013-09-10 17:36:21 +04:00
|
|
|
void Assign(const FeatureFunction* sp, const std::vector<float>& scores);
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2016-05-09 18:57:00 +03:00
|
|
|
void Assign(const FeatureFunction* sp, size_t idx, float sc);
|
|
|
|
|
2012-06-01 04:49:42 +04:00
|
|
|
//! Special version Assign(ScoreProducer, vector<float>)
|
|
|
|
//! to add the score from a single ScoreProducer that produces
|
|
|
|
//! a single value
|
2013-05-29 21:16:15 +04:00
|
|
|
void Assign(const FeatureFunction* sp, float score) {
|
2015-04-30 08:05:11 +03:00
|
|
|
|
2015-04-26 18:46:36 +03:00
|
|
|
UTIL_THROW_IF2(sp->GetNumScoreComponents() != 1,
|
2014-01-15 19:42:02 +04:00
|
|
|
"Feature function must must only contain 1 score");
|
2015-04-26 18:46:36 +03:00
|
|
|
m_scores[sp->GetIndex()] = score;
|
2012-06-01 04:49:42 +04:00
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2014-08-08 00:42:30 +04:00
|
|
|
// Assign score by index
|
2012-06-01 04:49:42 +04:00
|
|
|
void Assign(size_t index, float score) {
|
|
|
|
m_scores[index] = score;
|
|
|
|
}
|
2011-12-09 13:30:48 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
void Assign(const FeatureFunction*sp, const StringPiece &name, float score) {
|
2010-10-15 01:52:35 +04:00
|
|
|
FName fname(sp->GetScoreProducerDescription(),name);
|
|
|
|
m_scores[fname] = score;
|
|
|
|
}
|
|
|
|
|
2011-12-09 13:30:48 +04:00
|
|
|
|
2011-09-20 19:32:26 +04:00
|
|
|
//Read sparse features from string
|
2014-06-06 23:35:58 +04:00
|
|
|
void Assign(const FeatureFunction* sp, const std::string &line);
|
2011-09-20 19:32:26 +04:00
|
|
|
|
2011-06-01 21:26:41 +04:00
|
|
|
// shortcut: setting the value directly using the feature name
|
2013-05-29 21:16:15 +04:00
|
|
|
void Assign(const std::string name, float score) {
|
|
|
|
FName fname(name);
|
|
|
|
m_scores[fname] = score;
|
|
|
|
}
|
|
|
|
|
|
|
|
float InnerProduct(const ScoreComponentCollection& rhs) const {
|
|
|
|
return m_scores.inner_product(rhs.m_scores);
|
|
|
|
}
|
|
|
|
|
|
|
|
float PartialInnerProduct(const FeatureFunction* sp, const std::vector<float>& rhs) const {
|
|
|
|
std::vector<float> lhs = GetScoresForProducer(sp);
|
2013-11-21 21:51:01 +04:00
|
|
|
UTIL_THROW_IF2(lhs.size() != rhs.size(),
|
2014-01-15 19:42:02 +04:00
|
|
|
"Number of weights must match number of scores");
|
2013-05-29 21:16:15 +04:00
|
|
|
return std::inner_product(lhs.begin(), lhs.end(), rhs.begin(), 0.0f);
|
|
|
|
}
|
|
|
|
|
|
|
|
//! return a vector of all the scores associated with a certain FeatureFunction
|
|
|
|
std::vector<float> GetScoresForProducer(const FeatureFunction* sp) const {
|
2011-03-18 18:37:15 +03:00
|
|
|
size_t components = sp->GetNumScoreComponents();
|
2013-05-08 18:34:56 +04:00
|
|
|
|
2011-03-18 18:37:15 +03:00
|
|
|
std::vector<float> res(components);
|
2015-04-26 18:46:36 +03:00
|
|
|
size_t offset = sp->GetIndex();
|
2011-11-09 21:16:02 +04:00
|
|
|
for (size_t i = 0; i < res.size(); ++i) {
|
2015-04-26 18:46:36 +03:00
|
|
|
res[i] = m_scores[i + offset];
|
2010-10-07 02:06:49 +04:00
|
|
|
}
|
|
|
|
return res;
|
2013-05-29 21:16:15 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-08-07 04:58:56 +04:00
|
|
|
//! get subset of scores that belong to a certain sparse ScoreProducer
|
2013-05-15 14:37:21 +04:00
|
|
|
FVector GetVectorForProducer(const FeatureFunction* sp) const;
|
2011-08-07 04:58:56 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
float GetSparseWeight(const FName& featureName) const {
|
2012-03-11 20:26:25 +04:00
|
|
|
return m_scores[featureName];
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-04-29 08:37:48 +04:00
|
|
|
void PrintCoreFeatures() {
|
|
|
|
m_scores.printCoreFeatures();
|
|
|
|
}
|
2010-12-16 23:01:36 +03:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
void ThresholdScaling(float maxValue) {
|
|
|
|
// find (smallest) factor for which all weights are <= maxValue
|
|
|
|
// 0.1 / 0.14 = 0.714285714
|
|
|
|
// 0.1 / 0.17 = 0.588235294
|
2011-08-16 15:10:54 +04:00
|
|
|
m_scores.thresholdScale(maxValue);
|
2013-05-29 21:16:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void CapMax(float maxValue) {
|
|
|
|
// cap all sparse features to maxValue
|
|
|
|
m_scores.capMax(maxValue);
|
|
|
|
}
|
|
|
|
|
|
|
|
void CapMin(float minValue) {
|
|
|
|
// cap all sparse features to minValue
|
|
|
|
m_scores.capMin(minValue);
|
|
|
|
}
|
|
|
|
|
2015-04-26 18:46:36 +03:00
|
|
|
// std::pair<size_t,size_t> GetIndexesForProducer(const FeatureFunction* sp) const {
|
|
|
|
// IndexPair indexPair = GetIndexes(sp);
|
|
|
|
// return indexPair;
|
|
|
|
// }
|
2014-08-08 00:42:30 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
//! if a FeatureFunction produces a single score (for example, a language model score)
|
|
|
|
//! this will return it. If not, this method will throw
|
|
|
|
float GetScoreForProducer(const FeatureFunction* sp) const {
|
2015-04-26 18:46:36 +03:00
|
|
|
UTIL_THROW_IF2(sp->GetNumScoreComponents() != 1,
|
2014-01-15 19:42:02 +04:00
|
|
|
"Feature function must must only contain 1 score");
|
2015-04-26 18:46:36 +03:00
|
|
|
return m_scores[sp->GetIndex()];
|
2013-05-29 21:16:15 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2010-10-15 01:52:35 +04:00
|
|
|
//For features which have an unbounded number of components
|
|
|
|
float GetScoreForProducer
|
2013-05-29 21:16:15 +04:00
|
|
|
(const FeatureFunction* sp, const std::string& name) const {
|
2010-10-15 01:52:35 +04:00
|
|
|
FName fname(sp->GetScoreProducerDescription(),name);
|
|
|
|
return m_scores[fname];
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
float GetWeightedScore() const;
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2013-05-13 16:19:25 +04:00
|
|
|
void ZeroDenseFeatures(const FeatureFunction* sp);
|
2013-07-31 15:32:53 +04:00
|
|
|
void InvertDenseFeatures(const FeatureFunction* sp);
|
2010-09-28 19:13:50 +04:00
|
|
|
void L1Normalise();
|
2011-11-16 15:49:31 +04:00
|
|
|
float GetL1Norm() const;
|
|
|
|
float GetL2Norm() const;
|
2011-12-23 14:55:54 +04:00
|
|
|
float GetLInfNorm() const;
|
2012-07-26 20:32:50 +04:00
|
|
|
size_t L1Regularize(float lambda);
|
|
|
|
void L2Regularize(float lambda);
|
2012-06-01 04:49:42 +04:00
|
|
|
size_t SparseL1Regularize(float lambda);
|
|
|
|
void SparseL2Regularize(float lambda);
|
2011-11-16 15:49:31 +04:00
|
|
|
void Save(const std::string& filename) const;
|
2014-08-06 14:38:51 +04:00
|
|
|
void Save(std::ostream&, bool multiline=true) const;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
void IncrementSparseHopeFeatures() {
|
|
|
|
m_scores.incrementSparseHopeFeatures();
|
|
|
|
}
|
|
|
|
void IncrementSparseFearFeatures() {
|
|
|
|
m_scores.incrementSparseFearFeatures();
|
|
|
|
}
|
|
|
|
void PrintSparseHopeFeatureCounts(std::ofstream& out) {
|
|
|
|
m_scores.printSparseHopeFeatureCounts(out);
|
|
|
|
}
|
|
|
|
void PrintSparseFearFeatureCounts(std::ofstream& out) {
|
|
|
|
m_scores.printSparseFearFeatureCounts(out);
|
|
|
|
}
|
|
|
|
void PrintSparseHopeFeatureCounts() {
|
|
|
|
m_scores.printSparseHopeFeatureCounts();
|
|
|
|
}
|
|
|
|
void PrintSparseFearFeatureCounts() {
|
|
|
|
m_scores.printSparseFearFeatureCounts();
|
|
|
|
}
|
|
|
|
size_t PruneSparseFeatures(size_t threshold) {
|
|
|
|
return m_scores.pruneSparseFeatures(threshold);
|
|
|
|
}
|
|
|
|
size_t PruneZeroWeightFeatures() {
|
|
|
|
return m_scores.pruneZeroWeightFeatures();
|
|
|
|
}
|
|
|
|
void UpdateConfidenceCounts(ScoreComponentCollection &weightUpdate, bool signedCounts) {
|
|
|
|
m_scores.updateConfidenceCounts(weightUpdate.m_scores, signedCounts);
|
|
|
|
}
|
|
|
|
void UpdateLearningRates(float decay_core, float decay_sparse, ScoreComponentCollection &confidenceCounts, float core_r0, float sparse_r0) {
|
|
|
|
m_scores.updateLearningRates(decay_core, decay_sparse, confidenceCounts.m_scores, core_r0, sparse_r0);
|
|
|
|
}
|
2013-05-30 22:20:54 +04:00
|
|
|
void Merge(const ScoreComponentCollection &other) {
|
|
|
|
m_scores.merge(other.m_scores);
|
|
|
|
}
|
2010-09-28 19:13:50 +04:00
|
|
|
|
2015-10-31 15:50:52 +03:00
|
|
|
void OutputAllFeatureScores(std::ostream &out, bool with_labels) const;
|
|
|
|
void OutputFeatureScores(std::ostream& out, Moses::FeatureFunction const* ff,
|
2015-11-02 03:00:37 +03:00
|
|
|
std::string &lastName, bool with_labels) const;
|
2014-12-30 16:23:30 +03:00
|
|
|
|
2010-09-28 19:13:50 +04:00
|
|
|
#ifdef MPI_ENABLE
|
2013-05-29 21:16:15 +04:00
|
|
|
public:
|
|
|
|
friend class boost::serialization::access;
|
2010-09-28 19:13:50 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
private:
|
|
|
|
//serialization
|
|
|
|
template<class Archive>
|
|
|
|
void save(Archive &ar, const unsigned int version) const {
|
|
|
|
ar << m_scores;
|
|
|
|
}
|
|
|
|
|
|
|
|
template<class Archive>
|
|
|
|
void load(Archive &ar, const unsigned int version) {
|
|
|
|
ar >> m_scores;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
BOOST_SERIALIZATION_SPLIT_MEMBER()
|
2010-09-28 19:13:50 +04:00
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
};
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2010-09-28 19:13:50 +04:00
|
|
|
struct SCCPlus {
|
|
|
|
ScoreComponentCollection operator()
|
2013-05-29 21:16:15 +04:00
|
|
|
(const ScoreComponentCollection& lhs,
|
|
|
|
const ScoreComponentCollection& rhs) {
|
2010-09-28 19:13:50 +04:00
|
|
|
ScoreComponentCollection sum(lhs);
|
|
|
|
sum.PlusEquals(rhs);
|
|
|
|
return sum;
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
};
|
|
|
|
|
2013-08-23 16:53:30 +04:00
|
|
|
inline void swap(ScoreComponentCollection &first, ScoreComponentCollection &second)
|
|
|
|
{
|
2013-08-22 13:16:24 +04:00
|
|
|
swap(first.m_scores, second.m_scores);
|
|
|
|
}
|
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
}
|
2010-02-24 14:15:44 +03:00
|
|
|
#endif
|