mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
I observed a bug that ScoreComponentCollection core entries would retain their score even after ZeroAll(). This may have affected the Moses implementation of MIRA. * std::valarray::resize(0) means "resize to 0" [1] * subsequent accesses using operator[] result in undefined behavior [2] FeatureVector::clear() is used by ScoreComponentCollection::ZeroAll(), which in turn was used in these places: ./contrib/mira/Main.cpp:665: cumulativeWeights.ZeroAll(); ./contrib/mira/Main.cpp:666: cumulativeWeightsBinary.ZeroAll(); ./moses/Incremental.cpp:580: features.ZeroAll(); It seems to me that the Moses implementation of MIRA may have been affected? [1] http://www.cplusplus.com/reference/valarray/valarray/resize/ [2] http://www.cplusplus.com/reference/valarray/valarray/operator%5B%5D/
880 lines
22 KiB
C++
880 lines
22 KiB
C++
/*
|
|
Moses - statistical machine translation system
|
|
Copyright (C) 2005-2015 University of Edinburgh
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with this library; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <fstream>
|
|
#include <sstream>
|
|
#include <stdexcept>
|
|
|
|
#if defined __MINGW32__ && defined WITH_THREADS
|
|
#include <boost/thread/locks.hpp>
|
|
#endif // WITH_THREADS
|
|
|
|
#include "FeatureVector.h"
|
|
#include "util/string_piece_hash.hh"
|
|
#include "util/string_stream.hh"
|
|
|
|
using namespace std;
|
|
|
|
|
|
namespace Moses
|
|
{
|
|
|
|
const string FName::SEP = "_";
|
|
FName::Name2Id FName::name2id;
|
|
vector<string> FName::id2name;
|
|
FName::Id2Count FName::id2hopeCount;
|
|
FName::Id2Count FName::id2fearCount;
|
|
#ifdef WITH_THREADS
|
|
boost::shared_mutex FName::m_idLock;
|
|
#endif
|
|
|
|
void FName::init(const StringPiece &name)
|
|
{
|
|
#ifdef WITH_THREADS
|
|
//reader lock
|
|
boost::shared_lock<boost::shared_mutex> lock(m_idLock);
|
|
#endif
|
|
Name2Id::iterator i = FindStringPiece(name2id, name);
|
|
if (i != name2id.end()) {
|
|
m_id = i->second;
|
|
} else {
|
|
#ifdef WITH_THREADS
|
|
//release the reader lock, and upgrade to writer lock
|
|
lock.unlock();
|
|
boost::unique_lock<boost::shared_mutex> write_lock(m_idLock);
|
|
#endif
|
|
std::pair<std::string, size_t> to_ins;
|
|
to_ins.first.assign(name.data(), name.size());
|
|
to_ins.second = name2id.size();
|
|
std::pair<Name2Id::iterator, bool> res(name2id.insert(to_ins));
|
|
if (res.second) {
|
|
// TODO this should be string pointers backed by the hash table.
|
|
id2name.push_back(to_ins.first);
|
|
}
|
|
m_id = res.first->second;
|
|
}
|
|
}
|
|
|
|
size_t FName::getId(const string& name)
|
|
{
|
|
Name2Id::iterator i = name2id.find(name);
|
|
assert (i != name2id.end());
|
|
return i->second;
|
|
}
|
|
|
|
size_t FName::getHopeIdCount(const string& name)
|
|
{
|
|
Name2Id::iterator i = name2id.find(name);
|
|
if (i != name2id.end()) {
|
|
float id = i->second;
|
|
return id2hopeCount[id];
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
size_t FName::getFearIdCount(const string& name)
|
|
{
|
|
Name2Id::iterator i = name2id.find(name);
|
|
if (i != name2id.end()) {
|
|
float id = i->second;
|
|
return id2fearCount[id];
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
void FName::incrementHopeId(const string& name)
|
|
{
|
|
Name2Id::iterator i = name2id.find(name);
|
|
assert(i != name2id.end());
|
|
#ifdef WITH_THREADS
|
|
// get upgradable lock and upgrade to writer lock
|
|
boost::upgrade_lock<boost::shared_mutex> upgradeLock(m_idLock);
|
|
boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(upgradeLock);
|
|
#endif
|
|
id2hopeCount[i->second] += 1;
|
|
}
|
|
|
|
void FName::incrementFearId(const string& name)
|
|
{
|
|
Name2Id::iterator i = name2id.find(name);
|
|
assert(i != name2id.end());
|
|
#ifdef WITH_THREADS
|
|
// get upgradable lock and upgrade to writer lock
|
|
boost::upgrade_lock<boost::shared_mutex> upgradeLock(m_idLock);
|
|
boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(upgradeLock);
|
|
#endif
|
|
id2fearCount[i->second] += 1;
|
|
}
|
|
|
|
void FName::eraseId(size_t id)
|
|
{
|
|
#ifdef WITH_THREADS
|
|
// get upgradable lock and upgrade to writer lock
|
|
boost::upgrade_lock<boost::shared_mutex> upgradeLock(m_idLock);
|
|
boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(upgradeLock);
|
|
#endif
|
|
id2hopeCount.erase(id);
|
|
id2fearCount.erase(id);
|
|
}
|
|
|
|
std::ostream& operator<<( std::ostream& out, const FName& name)
|
|
{
|
|
out << name.name();
|
|
return out;
|
|
}
|
|
|
|
size_t FName::hash() const
|
|
{
|
|
return boost::hash_value(m_id);
|
|
}
|
|
|
|
const std::string& FName::name() const
|
|
{
|
|
return id2name[m_id];
|
|
}
|
|
|
|
|
|
bool FName::operator==(const FName& rhs) const
|
|
{
|
|
return m_id == rhs.m_id;
|
|
}
|
|
|
|
bool FName::operator!=(const FName& rhs) const
|
|
{
|
|
return ! (*this == rhs);
|
|
}
|
|
|
|
FVector::FVector(size_t coreFeatures) : m_coreFeatures(coreFeatures) {}
|
|
|
|
void FVector::resize(size_t newsize)
|
|
{
|
|
valarray<FValue> oldValues(m_coreFeatures);
|
|
m_coreFeatures.resize(newsize);
|
|
for (size_t i = 0; i < min(m_coreFeatures.size(), oldValues.size()); ++i) {
|
|
m_coreFeatures[i] = oldValues[i];
|
|
}
|
|
}
|
|
|
|
void FVector::clear()
|
|
{
|
|
m_coreFeatures.resize(m_coreFeatures.size(), 0);
|
|
m_features.clear();
|
|
}
|
|
|
|
bool FVector::load(const std::string& filename)
|
|
{
|
|
clear();
|
|
ifstream in (filename.c_str());
|
|
if (!in) {
|
|
return false;
|
|
}
|
|
string line;
|
|
while(getline(in,line)) {
|
|
if (line[0] == '#') continue;
|
|
istringstream linestream(line);
|
|
string namestring;
|
|
FValue value;
|
|
linestream >> namestring;
|
|
linestream >> value;
|
|
FName fname(namestring);
|
|
//cerr << "Setting sparse weight " << fname << " to value " << value << "." << endl;
|
|
set(fname,value);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void FVector::save(const string& filename) const
|
|
{
|
|
ofstream out(filename.c_str());
|
|
if (!out) {
|
|
util::StringStream msg;
|
|
msg << "Unable to open " << filename;
|
|
throw runtime_error(msg.str());
|
|
}
|
|
write(out);
|
|
out.close();
|
|
}
|
|
|
|
void FVector::write(ostream& out,const string& sep, const string& linesep) const
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
out << i->first << sep << i->second << linesep;
|
|
}
|
|
}
|
|
|
|
static bool equalsTolerance(FValue lhs, FValue rhs)
|
|
{
|
|
if (lhs == rhs) return true;
|
|
static const FValue TOLERANCE = 1e-4;
|
|
FValue diff = abs(lhs-rhs);
|
|
FValue mean = (abs(lhs)+abs(rhs))/2;
|
|
//cerr << "ET " << lhs << " " << rhs << " " << diff << " " << mean << " " << endl;
|
|
return diff/mean < TOLERANCE ;
|
|
}
|
|
|
|
bool FVector::operator== (const FVector& rhs) const
|
|
{
|
|
if (this == &rhs) {
|
|
return true;
|
|
}
|
|
if (m_coreFeatures.size() != rhs.m_coreFeatures.size()) {
|
|
return false;
|
|
}
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
if (!equalsTolerance(m_coreFeatures[i], rhs.m_coreFeatures[i])) return false;
|
|
}
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
if (!equalsTolerance(i->second,rhs.get(i->first))) return false;
|
|
}
|
|
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) {
|
|
if (!equalsTolerance(i->second, get(i->first))) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool FVector::operator!= (const FVector& rhs) const
|
|
{
|
|
return ! (*this == rhs);
|
|
}
|
|
|
|
ProxyFVector FVector::operator[](const FName& name)
|
|
{
|
|
// At this point, we don't know whether operator[] was called, so we return
|
|
// a proxy object and defer the decision until later
|
|
return ProxyFVector(this, name);
|
|
}
|
|
|
|
/** Equivalent for core features. */
|
|
FValue& FVector::operator[](size_t index)
|
|
{
|
|
return m_coreFeatures[index];
|
|
}
|
|
|
|
|
|
FValue FVector::operator[](const FName& name) const
|
|
{
|
|
return get(name);
|
|
}
|
|
|
|
FValue FVector::operator[](size_t index) const
|
|
{
|
|
return m_coreFeatures[index];
|
|
}
|
|
|
|
ostream& FVector::print(ostream& out) const
|
|
{
|
|
out << "core=(";
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
out << m_coreFeatures[i];
|
|
if (i + 1 < m_coreFeatures.size()) {
|
|
out << ",";
|
|
}
|
|
}
|
|
out << ") ";
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
if (i != cbegin())
|
|
out << " ";
|
|
out << i->first << "=" << i->second;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
ostream& operator<<(ostream& out, const FVector& fv)
|
|
{
|
|
return fv.print(out);
|
|
}
|
|
|
|
const FValue& FVector::get(const FName& name) const
|
|
{
|
|
static const FValue DEFAULT = 0;
|
|
const_iterator fi = m_features.find(name);
|
|
if (fi == m_features.end()) {
|
|
return DEFAULT;
|
|
} else {
|
|
return fi->second;
|
|
}
|
|
}
|
|
|
|
FValue FVector::getBackoff(const FName& name, float backoff) const
|
|
{
|
|
const_iterator fi = m_features.find(name);
|
|
if (fi == m_features.end()) {
|
|
return backoff;
|
|
} else {
|
|
return fi->second;
|
|
}
|
|
}
|
|
|
|
void FVector::thresholdScale(FValue maxValue )
|
|
{
|
|
FValue factor = 1.0;
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
FValue value = i->second;
|
|
if (abs(value)*factor > maxValue) {
|
|
factor = abs(value) / maxValue;
|
|
}
|
|
}
|
|
operator*=(factor);
|
|
}
|
|
|
|
void FVector::capMax(FValue maxValue)
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
if (i->second > maxValue)
|
|
set(i->first, maxValue);
|
|
}
|
|
|
|
void FVector::capMin(FValue minValue)
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
if (i->second < minValue)
|
|
set(i->first, minValue);
|
|
}
|
|
|
|
void FVector::set(const FName& name, const FValue& value)
|
|
{
|
|
m_features[name] = value;
|
|
}
|
|
|
|
void FVector::printCoreFeatures()
|
|
{
|
|
cerr << "core=(";
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
cerr << m_coreFeatures[i];
|
|
if (i + 1 < m_coreFeatures.size()) {
|
|
cerr << ",";
|
|
}
|
|
}
|
|
cerr << ") ";
|
|
}
|
|
|
|
FVector& FVector::operator+= (const FVector& rhs)
|
|
{
|
|
if (rhs.m_coreFeatures.size() > m_coreFeatures.size())
|
|
resize(rhs.m_coreFeatures.size());
|
|
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i)
|
|
set(i->first, get(i->first) + i->second);
|
|
for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i)
|
|
m_coreFeatures[i] += rhs.m_coreFeatures[i];
|
|
return *this;
|
|
}
|
|
|
|
// add only sparse features
|
|
void FVector::sparsePlusEquals(const FVector& rhs)
|
|
{
|
|
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i)
|
|
set(i->first, get(i->first) + i->second);
|
|
}
|
|
|
|
// add only core features
|
|
void FVector::corePlusEquals(const FVector& rhs)
|
|
{
|
|
if (rhs.m_coreFeatures.size() > m_coreFeatures.size())
|
|
resize(rhs.m_coreFeatures.size());
|
|
for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i)
|
|
m_coreFeatures[i] += rhs.m_coreFeatures[i];
|
|
}
|
|
|
|
// assign only core features
|
|
void FVector::coreAssign(const FVector& rhs)
|
|
{
|
|
for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i)
|
|
m_coreFeatures[i] = rhs.m_coreFeatures[i];
|
|
}
|
|
|
|
void FVector::incrementSparseHopeFeatures()
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
FName::incrementHopeId((i->first).name());
|
|
}
|
|
|
|
void FVector::incrementSparseFearFeatures()
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
FName::incrementFearId((i->first).name());
|
|
}
|
|
|
|
void FVector::printSparseHopeFeatureCounts(std::ofstream& out)
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
out << (i->first).name() << ": " << FName::getHopeIdCount((i->first).name()) << std::endl;
|
|
}
|
|
|
|
void FVector::printSparseFearFeatureCounts(std::ofstream& out)
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
out << (i->first).name() << ": " << FName::getFearIdCount((i->first).name()) << std::endl;
|
|
}
|
|
|
|
void FVector::printSparseHopeFeatureCounts()
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
std::cerr << (i->first).name() << ": " << FName::getHopeIdCount((i->first).name()) << std::endl;
|
|
}
|
|
|
|
void FVector::printSparseFearFeatureCounts()
|
|
{
|
|
for (const_iterator i = cbegin(); i != cend(); ++i)
|
|
std::cerr << (i->first).name() << ": " << FName::getFearIdCount((i->first).name()) << std::endl;
|
|
}
|
|
|
|
size_t FVector::pruneSparseFeatures(size_t threshold)
|
|
{
|
|
size_t count = 0;
|
|
vector<FName> toErase;
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
const std::string& fname = (i->first).name();
|
|
if (FName::getHopeIdCount(fname) < threshold && FName::getFearIdCount(fname) < threshold) {
|
|
toErase.push_back(i->first);
|
|
std::cerr << "pruning: " << fname << " (" << FName::getHopeIdCount(fname) << ", " << FName::getFearIdCount(fname) << ")" << std::endl;
|
|
FName::eraseId(FName::getId(fname));
|
|
++count;
|
|
}
|
|
}
|
|
|
|
for (size_t i = 0; i < toErase.size(); ++i)
|
|
m_features.erase(toErase[i]);
|
|
|
|
return count;
|
|
}
|
|
|
|
size_t FVector::pruneZeroWeightFeatures()
|
|
{
|
|
size_t count = 0;
|
|
vector<FName> toErase;
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
const std::string& fname = (i->first).name();
|
|
if (i->second == 0) {
|
|
toErase.push_back(i->first);
|
|
//std::cerr << "prune: " << fname << std::endl;
|
|
FName::eraseId(FName::getId(fname));
|
|
++count;
|
|
}
|
|
}
|
|
|
|
for (size_t i = 0; i < toErase.size(); ++i)
|
|
m_features.erase(toErase[i]);
|
|
|
|
return count;
|
|
}
|
|
|
|
void FVector::updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts)
|
|
{
|
|
for (size_t i = 0; i < weightUpdate.m_coreFeatures.size(); ++i) {
|
|
if (signedCounts) {
|
|
//int sign = weightUpdate.m_coreFeatures[i] >= 0 ? 1 : -1;
|
|
//m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]) * sign;
|
|
m_coreFeatures[i] += weightUpdate.m_coreFeatures[i];
|
|
} else
|
|
//m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]);
|
|
m_coreFeatures[i] += abs(weightUpdate.m_coreFeatures[i]);
|
|
}
|
|
|
|
for (const_iterator i = weightUpdate.cbegin(); i != weightUpdate.cend(); ++i) {
|
|
if (weightUpdate[i->first] == 0)
|
|
continue;
|
|
float value = get(i->first);
|
|
if (signedCounts) {
|
|
//int sign = weightUpdate[i->first] >= 0 ? 1 : -1;
|
|
//value += (weightUpdate[i->first] * weightUpdate[i->first]) * sign;
|
|
value += weightUpdate[i->first];
|
|
} else
|
|
//value += (weightUpdate[i->first] * weightUpdate[i->first]);
|
|
value += abs(weightUpdate[i->first]);
|
|
set(i->first, value);
|
|
}
|
|
}
|
|
|
|
void FVector::updateLearningRates(float decay_core, float decay_sparse, const FVector &confidenceCounts, float core_r0, float sparse_r0)
|
|
{
|
|
for (size_t i = 0; i < confidenceCounts.m_coreFeatures.size(); ++i) {
|
|
m_coreFeatures[i] = 1.0/(1.0/core_r0 + decay_core * abs(confidenceCounts.m_coreFeatures[i]));
|
|
}
|
|
|
|
for (const_iterator i = confidenceCounts.cbegin(); i != confidenceCounts.cend(); ++i) {
|
|
float value = 1.0/(1.0/sparse_r0 + decay_sparse * abs(i->second));
|
|
set(i->first, value);
|
|
}
|
|
}
|
|
|
|
// count non-zero occurrences for all sparse features
|
|
void FVector::setToBinaryOf(const FVector& rhs)
|
|
{
|
|
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i)
|
|
if (rhs.get(i->first) != 0)
|
|
set(i->first, 1);
|
|
for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i)
|
|
m_coreFeatures[i] = 1;
|
|
}
|
|
|
|
// divide only core features by scalar
|
|
FVector& FVector::coreDivideEquals(float scalar)
|
|
{
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i)
|
|
m_coreFeatures[i] /= scalar;
|
|
return *this;
|
|
}
|
|
|
|
// lhs vector is a sum of vectors, rhs vector holds number of non-zero summands
|
|
FVector& FVector::divideEquals(const FVector& rhs)
|
|
{
|
|
assert(m_coreFeatures.size() == rhs.m_coreFeatures.size());
|
|
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i)
|
|
set(i->first, get(i->first)/rhs.get(i->first)); // divide by number of summands
|
|
for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i)
|
|
m_coreFeatures[i] /= rhs.m_coreFeatures[i]; // divide by number of summands
|
|
return *this;
|
|
}
|
|
|
|
FVector& FVector::operator-= (const FVector& rhs)
|
|
{
|
|
if (rhs.m_coreFeatures.size() > m_coreFeatures.size())
|
|
resize(rhs.m_coreFeatures.size());
|
|
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i)
|
|
set(i->first, get(i->first) -(i->second));
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
if (i < rhs.m_coreFeatures.size()) {
|
|
m_coreFeatures[i] -= rhs.m_coreFeatures[i];
|
|
}
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
FVector& FVector::operator*= (const FVector& rhs)
|
|
{
|
|
if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) {
|
|
resize(rhs.m_coreFeatures.size());
|
|
}
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
FValue lhsValue = i->second;
|
|
FValue rhsValue = rhs.get(i->first);
|
|
set(i->first,lhsValue*rhsValue);
|
|
}
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
if (i < rhs.m_coreFeatures.size()) {
|
|
m_coreFeatures[i] *= rhs.m_coreFeatures[i];
|
|
} else {
|
|
m_coreFeatures[i] = 0;
|
|
}
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
FVector& FVector::operator/= (const FVector& rhs)
|
|
{
|
|
if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) {
|
|
resize(rhs.m_coreFeatures.size());
|
|
}
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
FValue lhsValue = i->second;
|
|
FValue rhsValue = rhs.get(i->first);
|
|
set(i->first, lhsValue / rhsValue) ;
|
|
}
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
if (i < rhs.m_coreFeatures.size()) {
|
|
m_coreFeatures[i] /= rhs.m_coreFeatures[i];
|
|
} else {
|
|
if (m_coreFeatures[i] < 0) {
|
|
m_coreFeatures[i] = -numeric_limits<FValue>::infinity();
|
|
} else if (m_coreFeatures[i] > 0) {
|
|
m_coreFeatures[i] = numeric_limits<FValue>::infinity();
|
|
}
|
|
}
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
FVector& FVector::operator*= (const FValue& rhs)
|
|
{
|
|
//NB Could do this with boost::bind ?
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
i->second *= rhs;
|
|
}
|
|
m_coreFeatures *= rhs;
|
|
return *this;
|
|
}
|
|
|
|
FVector& FVector::operator/= (const FValue& rhs)
|
|
{
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
i->second /= rhs;
|
|
}
|
|
m_coreFeatures /= rhs;
|
|
return *this;
|
|
}
|
|
|
|
FVector& FVector::multiplyEqualsBackoff(const FVector& rhs, float backoff)
|
|
{
|
|
if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) {
|
|
resize(rhs.m_coreFeatures.size());
|
|
}
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
FValue lhsValue = i->second;
|
|
FValue rhsValue = rhs.getBackoff(i->first, backoff);
|
|
set(i->first,lhsValue*rhsValue);
|
|
}
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
if (i < rhs.m_coreFeatures.size()) {
|
|
m_coreFeatures[i] *= rhs.m_coreFeatures[i];
|
|
} else {
|
|
m_coreFeatures[i] = 0;
|
|
}
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
FVector& FVector::multiplyEquals(float core_r0, float sparse_r0)
|
|
{
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
m_coreFeatures[i] *= core_r0;
|
|
}
|
|
for (iterator i = begin(); i != end(); ++i)
|
|
set(i->first,(i->second)*sparse_r0);
|
|
return *this;
|
|
}
|
|
|
|
FValue FVector::l1norm() const
|
|
{
|
|
FValue norm = 0;
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
norm += abs(i->second);
|
|
}
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
norm += abs(m_coreFeatures[i]);
|
|
}
|
|
return norm;
|
|
}
|
|
|
|
FValue FVector::l1norm_coreFeatures() const
|
|
{
|
|
FValue norm = 0;
|
|
// ignore Bleu score feature (last feature)
|
|
for (size_t i = 0; i < m_coreFeatures.size()-1; ++i)
|
|
norm += abs(m_coreFeatures[i]);
|
|
return norm;
|
|
}
|
|
|
|
FValue FVector::l2norm() const
|
|
{
|
|
return sqrt(inner_product(*this));
|
|
}
|
|
|
|
FValue FVector::linfnorm() const
|
|
{
|
|
FValue norm = 0;
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
float absValue = abs(i->second);
|
|
if (absValue > norm)
|
|
norm = absValue;
|
|
}
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
float absValue = abs(m_coreFeatures[i]);
|
|
if (absValue > norm)
|
|
norm = absValue;
|
|
}
|
|
return norm;
|
|
}
|
|
|
|
size_t FVector::l1regularize(float lambda)
|
|
{
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
float value = m_coreFeatures[i];
|
|
if (value > 0) {
|
|
m_coreFeatures[i] = max(0.0f, value - lambda);
|
|
} else {
|
|
m_coreFeatures[i] = min(0.0f, value + lambda);
|
|
}
|
|
}
|
|
|
|
size_t numberPruned = size();
|
|
vector<FName> toErase;
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
float value = i->second;
|
|
if (value != 0.0f) {
|
|
if (value > 0)
|
|
value = max(0.0f, value - lambda);
|
|
else
|
|
value = min(0.0f, value + lambda);
|
|
|
|
if (value != 0.0f)
|
|
i->second = value;
|
|
else {
|
|
toErase.push_back(i->first);
|
|
const std::string& fname = (i->first).name();
|
|
FName::eraseId(FName::getId(fname));
|
|
}
|
|
}
|
|
}
|
|
|
|
// erase features that have become zero
|
|
for (size_t i = 0; i < toErase.size(); ++i)
|
|
m_features.erase(toErase[i]);
|
|
numberPruned -= size();
|
|
return numberPruned;
|
|
}
|
|
|
|
void FVector::l2regularize(float lambda)
|
|
{
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
m_coreFeatures[i] *= (1 - lambda);
|
|
}
|
|
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
i->second *= (1 - lambda);
|
|
}
|
|
}
|
|
|
|
size_t FVector::sparseL1regularize(float lambda)
|
|
{
|
|
/*for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
float value = m_coreFeatures[i];
|
|
if (value > 0) {
|
|
m_coreFeatures[i] = max(0.0f, value - lambda);
|
|
}
|
|
else {
|
|
m_coreFeatures[i] = min(0.0f, value + lambda);
|
|
}
|
|
}*/
|
|
|
|
size_t numberPruned = size();
|
|
vector<FName> toErase;
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
float value = i->second;
|
|
if (value != 0.0f) {
|
|
if (value > 0)
|
|
value = max(0.0f, value - lambda);
|
|
else
|
|
value = min(0.0f, value + lambda);
|
|
|
|
if (value != 0.0f)
|
|
i->second = value;
|
|
else {
|
|
toErase.push_back(i->first);
|
|
const std::string& fname = (i->first).name();
|
|
FName::eraseId(FName::getId(fname));
|
|
}
|
|
}
|
|
}
|
|
|
|
// erase features that have become zero
|
|
for (size_t i = 0; i < toErase.size(); ++i)
|
|
m_features.erase(toErase[i]);
|
|
numberPruned -= size();
|
|
return numberPruned;
|
|
}
|
|
|
|
void FVector::sparseL2regularize(float lambda)
|
|
{
|
|
/*for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
m_coreFeatures[i] *= (1 - lambda);
|
|
}*/
|
|
|
|
for (iterator i = begin(); i != end(); ++i) {
|
|
i->second *= (1 - lambda);
|
|
}
|
|
}
|
|
|
|
FValue FVector::sum() const
|
|
{
|
|
FValue sum = 0;
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
sum += i->second;
|
|
}
|
|
sum += m_coreFeatures.sum();
|
|
return sum;
|
|
}
|
|
|
|
FValue FVector::inner_product(const FVector& rhs) const
|
|
{
|
|
assert(m_coreFeatures.size() == rhs.m_coreFeatures.size());
|
|
FValue product = 0.0;
|
|
for (const_iterator i = cbegin(); i != cend(); ++i) {
|
|
product += ((i->second)*(rhs.get(i->first)));
|
|
}
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
product += m_coreFeatures[i]*rhs.m_coreFeatures[i];
|
|
}
|
|
return product;
|
|
}
|
|
|
|
void FVector::merge(const FVector &other)
|
|
{
|
|
// dense
|
|
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
|
|
FValue &thisVal = m_coreFeatures[i];
|
|
const FValue otherVal = other.m_coreFeatures[i];
|
|
|
|
if (otherVal) {
|
|
assert(thisVal == 0 || thisVal == otherVal);
|
|
thisVal = otherVal;
|
|
}
|
|
}
|
|
|
|
// sparse
|
|
FNVmap::const_iterator iter;
|
|
for (iter = other.m_features.begin(); iter != other.m_features.end(); ++iter) {
|
|
const FName &otherKey = iter->first;
|
|
const FValue otherVal = iter->second;
|
|
m_features[otherKey] = otherVal;
|
|
}
|
|
}
|
|
|
|
const FVector operator+(const FVector& lhs, const FVector& rhs)
|
|
{
|
|
return FVector(lhs) += rhs;
|
|
}
|
|
|
|
const FVector operator-(const FVector& lhs, const FVector& rhs)
|
|
{
|
|
return FVector(lhs) -= rhs;
|
|
}
|
|
|
|
const FVector operator*(const FVector& lhs, const FVector& rhs)
|
|
{
|
|
return FVector(lhs) *= rhs;
|
|
}
|
|
|
|
const FVector operator/(const FVector& lhs, const FVector& rhs)
|
|
{
|
|
return FVector(lhs) /= rhs;
|
|
}
|
|
|
|
|
|
const FVector operator*(const FVector& lhs, const FValue& rhs)
|
|
{
|
|
return FVector(lhs) *= rhs;
|
|
}
|
|
|
|
const FVector operator/(const FVector& lhs, const FValue& rhs)
|
|
{
|
|
return FVector(lhs) /= rhs;
|
|
}
|
|
|
|
FValue inner_product(const FVector& lhs, const FVector& rhs)
|
|
{
|
|
if (lhs.size() >= rhs.size()) {
|
|
return rhs.inner_product(lhs);
|
|
} else {
|
|
return lhs.inner_product(rhs);
|
|
}
|
|
}
|
|
}
|