mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
sparse vec
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mira-mtm5@3429 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
hieuhoang1972
parent
c905b49ac0
commit
356c6b1868
@ -11,6 +11,8 @@
|
||||
1E5D8E0511F25F03000F027F /* PhraseDictionaryNodeSCFG.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E5D8E0311F25F03000F027F /* PhraseDictionaryNodeSCFG.h */; };
|
||||
1E5D8E0811F25F2F000F027F /* PhraseDictionarySCFG.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E5D8E0611F25F2F000F027F /* PhraseDictionarySCFG.cpp */; };
|
||||
1E5D8E0911F25F2F000F027F /* PhraseDictionarySCFGChart.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E5D8E0711F25F2F000F027F /* PhraseDictionarySCFGChart.cpp */; };
|
||||
1E7739C2123F619500B88EB7 /* FeatureVector.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E7739C0123F619500B88EB7 /* FeatureVector.h */; };
|
||||
1E7739C3123F619500B88EB7 /* FeatureVector.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E7739C1123F619500B88EB7 /* FeatureVector.cpp */; };
|
||||
1ED4FD3711BDC0D2004E826A /* AlignmentInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1ED4FC5F11BDC0D2004E826A /* AlignmentInfo.cpp */; };
|
||||
1ED4FD3811BDC0D2004E826A /* AlignmentInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 1ED4FC6011BDC0D2004E826A /* AlignmentInfo.h */; };
|
||||
1ED4FD3911BDC0D2004E826A /* BilingualDynSuffixArray.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1ED4FC6111BDC0D2004E826A /* BilingualDynSuffixArray.cpp */; };
|
||||
@ -226,6 +228,8 @@
|
||||
1E5D8E0311F25F03000F027F /* PhraseDictionaryNodeSCFG.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryNodeSCFG.h; path = src/PhraseDictionaryNodeSCFG.h; sourceTree = "<group>"; };
|
||||
1E5D8E0611F25F2F000F027F /* PhraseDictionarySCFG.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDictionarySCFG.cpp; path = src/PhraseDictionarySCFG.cpp; sourceTree = "<group>"; };
|
||||
1E5D8E0711F25F2F000F027F /* PhraseDictionarySCFGChart.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDictionarySCFGChart.cpp; path = src/PhraseDictionarySCFGChart.cpp; sourceTree = "<group>"; };
|
||||
1E7739C0123F619500B88EB7 /* FeatureVector.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FeatureVector.h; sourceTree = "<group>"; };
|
||||
1E7739C1123F619500B88EB7 /* FeatureVector.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FeatureVector.cpp; sourceTree = "<group>"; };
|
||||
1ED4FC5F11BDC0D2004E826A /* AlignmentInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = AlignmentInfo.cpp; path = src/AlignmentInfo.cpp; sourceTree = "<group>"; };
|
||||
1ED4FC6011BDC0D2004E826A /* AlignmentInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AlignmentInfo.h; path = src/AlignmentInfo.h; sourceTree = "<group>"; };
|
||||
1ED4FC6111BDC0D2004E826A /* BilingualDynSuffixArray.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = BilingualDynSuffixArray.cpp; path = src/BilingualDynSuffixArray.cpp; sourceTree = "<group>"; };
|
||||
@ -671,6 +675,8 @@
|
||||
1ED4FD3411BDC0D2004E826A /* WordsRange.h */,
|
||||
1ED4FD3511BDC0D2004E826A /* XmlOption.cpp */,
|
||||
1ED4FD3611BDC0D2004E826A /* XmlOption.h */,
|
||||
1E7739C0123F619500B88EB7 /* FeatureVector.h */,
|
||||
1E7739C1123F619500B88EB7 /* FeatureVector.cpp */,
|
||||
);
|
||||
name = Source;
|
||||
sourceTree = "<group>";
|
||||
@ -823,6 +829,7 @@
|
||||
1EF549B212118A0C00C481EB /* DecodeFeature.h in Headers */,
|
||||
1EF549B612118A4A00C481EB /* LanguageModelDelegate.h in Headers */,
|
||||
1EF549BA12118A5D00C481EB /* TranslationSystem.h in Headers */,
|
||||
1E7739C2123F619500B88EB7 /* FeatureVector.h in Headers */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
@ -969,6 +976,7 @@
|
||||
1E5D8E0911F25F2F000F027F /* PhraseDictionarySCFGChart.cpp in Sources */,
|
||||
1EF549B112118A0C00C481EB /* DecodeFeature.cpp in Sources */,
|
||||
1EF549B912118A5D00C481EB /* TranslationSystem.cpp in Sources */,
|
||||
1E7739C3123F619500B88EB7 /* FeatureVector.cpp in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
|
||||
427
moses/src/FeatureVector.cpp
Normal file
427
moses/src/FeatureVector.cpp
Normal file
@ -0,0 +1,427 @@
|
||||
/*
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 University of Edinburgh
|
||||
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
|
||||
|
||||
#include "FeatureVector.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
|
||||
namespace Moses {
|
||||
|
||||
const string FName::SEP = "_";
|
||||
FName::Name2Id FName::name2id;
|
||||
vector<string> FName::id2name;
|
||||
|
||||
void FName::init(const string& name) {
|
||||
Name2Id::iterator i = name2id.find(name);
|
||||
if (i != name2id.end()) {
|
||||
m_id = i->second;
|
||||
} else {
|
||||
m_id = name2id.size();
|
||||
name2id[name] = m_id;
|
||||
id2name.push_back(name);
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream& operator<<( std::ostream& out, const FName& name) {
|
||||
out << name.name();
|
||||
return out;
|
||||
}
|
||||
|
||||
size_t FName::hash() const {
|
||||
/*std::size_t seed = 0;
|
||||
boost::hash_combine(seed, m_root);
|
||||
boost::hash_combine(seed, m_name);
|
||||
return seed;*/
|
||||
return boost::hash_value(m_id);
|
||||
}
|
||||
|
||||
const std::string& FName::name() const {
|
||||
return id2name[m_id];
|
||||
}
|
||||
|
||||
|
||||
bool FName::operator==(const FName& rhs) const {
|
||||
return m_id == rhs.m_id;
|
||||
}
|
||||
|
||||
bool FName::operator!=(const FName& rhs) const {
|
||||
return ! (*this == rhs);
|
||||
}
|
||||
|
||||
FVector::FVector(FValue defaultValue) {
|
||||
m_features[DEFAULT_NAME] = defaultValue;
|
||||
}
|
||||
|
||||
void FVector::clear() {
|
||||
m_features.clear();
|
||||
m_features[DEFAULT_NAME] = DEFAULT;
|
||||
}
|
||||
|
||||
void FVector::load(const std::string& filename) {
|
||||
ifstream in (filename.c_str());
|
||||
if (!in) {
|
||||
ostringstream msg;
|
||||
msg << "Unable to open " << filename;
|
||||
throw runtime_error(msg.str());
|
||||
}
|
||||
string line;
|
||||
while(getline(in,line)) {
|
||||
if (line[0] == '#') continue;
|
||||
istringstream linestream(line);
|
||||
string namestring;
|
||||
FValue value;
|
||||
linestream >> namestring;
|
||||
linestream >> value;
|
||||
FName fname(namestring);
|
||||
set(fname,value);
|
||||
}
|
||||
}
|
||||
|
||||
void FVector::save(const string& filename) const {
|
||||
ofstream out(filename.c_str());
|
||||
if (!out) {
|
||||
ostringstream msg;
|
||||
msg << "Unable to open " << filename;
|
||||
throw runtime_error(msg.str());
|
||||
}
|
||||
write(out);
|
||||
out.close();
|
||||
}
|
||||
|
||||
void FVector::write(ostream& out) const {
|
||||
for (const_iterator i = begin(); i != end(); ++i) {
|
||||
out << i->first << " " << i->second << endl;
|
||||
}
|
||||
}
|
||||
|
||||
FName FVector::DEFAULT_NAME("DEFAULT","");
|
||||
const FValue FVector::DEFAULT = 0;
|
||||
|
||||
|
||||
static bool equalsTolerance(FValue lhs, FValue rhs) {
|
||||
if (lhs == rhs) return true;
|
||||
static const FValue TOLERANCE = 1e-4;
|
||||
FValue diff = abs(lhs-rhs);
|
||||
FValue mean = (abs(lhs)+abs(rhs))/2;
|
||||
//cerr << "ET " << lhs << " " << rhs << " " << diff << " " << mean << " " << endl;
|
||||
return diff/mean < TOLERANCE ;
|
||||
}
|
||||
|
||||
bool FVector::operator== (const FVector& rhs) const {
|
||||
if (this == &rhs) {
|
||||
return true;
|
||||
}
|
||||
if (get(DEFAULT_NAME) != rhs.get(DEFAULT_NAME)) return false;
|
||||
for (const_iterator i = begin(); i != end(); ++i) {
|
||||
if (!equalsTolerance(i->second,rhs.get(i->first))) return false;
|
||||
}
|
||||
for (const_iterator i = rhs.begin(); i != rhs.end(); ++i) {
|
||||
if (!equalsTolerance(i->second, get(i->first))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool FVector::operator!= (const FVector& rhs) const {
|
||||
return ! (*this == rhs);
|
||||
}
|
||||
|
||||
|
||||
|
||||
ProxyFVector FVector::operator[](const FName& name) {
|
||||
// At this point, we don't know whether operator[] was called, so we return
|
||||
// a proxy object and defer the decision until later
|
||||
return ProxyFVector(this, name);
|
||||
|
||||
}
|
||||
|
||||
FValue FVector::operator[](const FName& name) const {
|
||||
return get(name) + get(DEFAULT_NAME);
|
||||
}
|
||||
|
||||
|
||||
|
||||
ostream& FVector::print(ostream& out) const {
|
||||
out << "{";
|
||||
for (const_iterator i = begin(); i != end(); ++i) {
|
||||
FValue value = i->second;
|
||||
if (i->first != DEFAULT_NAME) {
|
||||
value += get(DEFAULT_NAME);
|
||||
}
|
||||
if (i->first != DEFAULT_NAME && i->second != 0.0) {
|
||||
out << i->first << "=" << value << ", ";
|
||||
}
|
||||
}
|
||||
out << "}";
|
||||
return out;
|
||||
}
|
||||
|
||||
ostream& operator<<(ostream& out, const FVector& fv) {
|
||||
return fv.print(out);
|
||||
}
|
||||
|
||||
const FValue& FVector::get(const FName& name) const {
|
||||
const_iterator fi = m_features.find(name);
|
||||
if (fi == m_features.end()) {
|
||||
return DEFAULT;
|
||||
} else {
|
||||
return fi->second;
|
||||
}
|
||||
}
|
||||
|
||||
void FVector::set(const FName& name, const FValue& value) {
|
||||
m_features[name] = value;
|
||||
}
|
||||
|
||||
FVector& FVector::operator+= (const FVector& rhs) {
|
||||
//default value will take care of itself here.
|
||||
for (iterator i = begin(); i != end(); ++i) {
|
||||
set(i->first,i->second + rhs.get(i->first));
|
||||
}
|
||||
for (const_iterator i = rhs.begin(); i != rhs.end(); ++i) {
|
||||
if (!hasNonDefaultValue(i->first)) {
|
||||
set(i->first,i->second);
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
FVector& FVector::operator-= (const FVector& rhs) {
|
||||
for (iterator i = begin(); i != end(); ++i) {
|
||||
set(i->first,i->second - rhs.get(i->first));
|
||||
}
|
||||
for (const_iterator i = rhs.begin(); i != rhs.end(); ++i) {
|
||||
if (!hasNonDefaultValue(i->first)) {
|
||||
set(i->first,-(i->second));
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
FVector& FVector::operator*= (const FVector& rhs) {
|
||||
FValue lhsDefault = get(DEFAULT_NAME);
|
||||
FValue rhsDefault = rhs.get(DEFAULT_NAME);
|
||||
for (iterator i = begin(); i != end(); ++i) {
|
||||
if (i->first == DEFAULT_NAME) {
|
||||
set(i->first,lhsDefault*rhsDefault);
|
||||
} else {
|
||||
FValue lhsValue = i->second;
|
||||
FValue rhsValue = rhs.get(i->first);
|
||||
set(i->first, lhsValue*rhsDefault + rhsValue*lhsDefault + lhsValue*rhsValue);
|
||||
}
|
||||
}
|
||||
if (lhsDefault) {
|
||||
//Features that have the default value in the lhs
|
||||
for (const_iterator i = rhs.begin(); i != rhs.end(); ++i) {
|
||||
if (!hasNonDefaultValue(i->first)) {
|
||||
set(i->first, lhsDefault*i->second);
|
||||
}
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
FVector& FVector::operator/= (const FVector& rhs) {
|
||||
FValue lhsDefault = get(DEFAULT_NAME);
|
||||
FValue rhsDefault = rhs.get(DEFAULT_NAME);
|
||||
if (lhsDefault && !rhsDefault) {
|
||||
throw runtime_error("Attempt to divide feature vectors where lhs has default and rhs does not");
|
||||
}
|
||||
FValue quotientDefault = 0;
|
||||
if (rhsDefault) {
|
||||
quotientDefault = lhsDefault / rhsDefault;
|
||||
}
|
||||
for (iterator i = begin(); i != end(); ++i) {
|
||||
if (i->first == DEFAULT_NAME) {
|
||||
set(i->first, quotientDefault);
|
||||
} else {
|
||||
FValue lhsValue = i->second;
|
||||
FValue rhsValue = rhs.get(i->first);
|
||||
set(i->first, (lhsValue + lhsDefault) / (rhsValue + rhsDefault) - quotientDefault);
|
||||
}
|
||||
}
|
||||
if (lhsDefault) {
|
||||
//Features that have the default value in the lhs
|
||||
for (const_iterator i = rhs.begin(); i != rhs.end(); ++i) {
|
||||
if (!hasNonDefaultValue(i->first)) {
|
||||
set(i->first, lhsDefault / (i->second + rhsDefault) - quotientDefault);
|
||||
}
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
FVector& FVector::max_equals(const FVector& rhs) {
|
||||
FValue lhsDefault = get(DEFAULT_NAME);
|
||||
FValue rhsDefault = rhs.get(DEFAULT_NAME);
|
||||
FValue maxDefault = max(lhsDefault,rhsDefault);
|
||||
for (iterator i = begin(); i != end(); ++i) {
|
||||
if (i->first == DEFAULT_NAME) {
|
||||
set(i->first, maxDefault);
|
||||
} else {
|
||||
set(i->first, max(i->second + lhsDefault, rhs.get(i->first) + rhsDefault) - maxDefault);
|
||||
}
|
||||
}
|
||||
for (const_iterator i = rhs.begin(); i != rhs.end(); ++i) {
|
||||
if (!hasNonDefaultValue(i->first)) {
|
||||
set(i->first, max(lhsDefault, (i->second + rhsDefault)) - maxDefault);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
FVector& FVector::operator+= (const FValue& rhs) {
|
||||
set(DEFAULT_NAME, get(DEFAULT_NAME) + rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
FVector& FVector::operator-= (const FValue& rhs) {
|
||||
set(DEFAULT_NAME, get(DEFAULT_NAME) - rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
FVector& FVector::operator*= (const FValue& rhs) {
|
||||
//NB Could do this with boost::bind ?
|
||||
//This multiplies the default value, which is what we want
|
||||
for (iterator i = begin(); i != end(); ++i) {
|
||||
i->second *= rhs;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
||||
FVector& FVector::operator/= (const FValue& rhs) {
|
||||
//This dividess the default value, which is what we want
|
||||
for (iterator i = begin(); i != end(); ++i) {
|
||||
i->second /= rhs;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
FValue FVector::l1norm() const {
|
||||
FValue norm = 0;
|
||||
for (const_iterator i = begin(); i != end(); ++i) {
|
||||
if (!(i->second) && i->first == DEFAULT_NAME) {
|
||||
throw runtime_error("Cannot take l1norm with non-zero default values");
|
||||
}
|
||||
norm += abs(i->second);
|
||||
}
|
||||
return norm;
|
||||
}
|
||||
|
||||
FValue FVector::sum() const {
|
||||
FValue sum = 0;
|
||||
for (const_iterator i = begin(); i != end(); ++i) {
|
||||
if (!(i->second) && i->first == DEFAULT_NAME) {
|
||||
throw runtime_error("Cannot take sum with non-zero default values");
|
||||
}
|
||||
sum += i->second;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
FValue FVector::l2norm() const {
|
||||
return sqrt(inner_product(*this));
|
||||
}
|
||||
|
||||
|
||||
|
||||
FValue FVector::inner_product(const FVector& rhs) const {
|
||||
FValue lhsDefault = get(DEFAULT_NAME);
|
||||
FValue rhsDefault = rhs.get(DEFAULT_NAME);
|
||||
if (lhsDefault && rhsDefault) {
|
||||
throw runtime_error("Cannot take inner product if both lhs and rhs have non-zero default values");
|
||||
}
|
||||
FValue product = 0.0;
|
||||
for (const_iterator i = begin(); i != end(); ++i) {
|
||||
if (i->first != DEFAULT_NAME) {
|
||||
product += ((i->second + lhsDefault)*(rhs.get(i->first) + rhsDefault));
|
||||
}
|
||||
}
|
||||
|
||||
if (lhsDefault) {
|
||||
//Features that have the default value in the rhs
|
||||
for (const_iterator i = rhs.begin(); i != rhs.end(); ++i) {
|
||||
if (!hasNonDefaultValue(i->first)) {
|
||||
product += (i->second + rhsDefault)*lhsDefault;
|
||||
}
|
||||
}
|
||||
}
|
||||
return product;
|
||||
}
|
||||
|
||||
|
||||
const FVector operator+(const FVector& lhs, const FVector& rhs) {
|
||||
return FVector(lhs) += rhs;
|
||||
}
|
||||
|
||||
const FVector operator-(const FVector& lhs, const FVector& rhs) {
|
||||
return FVector(lhs) -= rhs;
|
||||
}
|
||||
|
||||
const FVector operator*(const FVector& lhs, const FVector& rhs) {
|
||||
return FVector(lhs) *= rhs;
|
||||
}
|
||||
|
||||
const FVector operator/(const FVector& lhs, const FVector& rhs) {
|
||||
return FVector(lhs) /= rhs;
|
||||
}
|
||||
|
||||
const FVector operator+(const FVector& lhs, const FValue& rhs) {
|
||||
return FVector(lhs) += rhs;
|
||||
}
|
||||
|
||||
const FVector operator-(const FVector& lhs, const FValue& rhs) {
|
||||
return FVector(lhs) -= rhs;
|
||||
}
|
||||
|
||||
const FVector operator*(const FVector& lhs, const FValue& rhs) {
|
||||
return FVector(lhs) *= rhs;
|
||||
}
|
||||
|
||||
const FVector operator/(const FVector& lhs, const FValue& rhs) {
|
||||
return FVector(lhs) /= rhs;
|
||||
}
|
||||
|
||||
const FVector fvmax(const FVector& lhs, const FVector& rhs) {
|
||||
return FVector(lhs).max_equals(rhs);
|
||||
}
|
||||
|
||||
FValue inner_product(const FVector& lhs, const FVector& rhs) {
|
||||
if (lhs.size() >= rhs.size()) {
|
||||
return rhs.inner_product(lhs);
|
||||
} else {
|
||||
return lhs.inner_product(rhs);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
293
moses/src/FeatureVector.h
Normal file
293
moses/src/FeatureVector.h
Normal file
@ -0,0 +1,293 @@
|
||||
/*
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 University of Edinburgh
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
*/
|
||||
|
||||
#ifndef FEATUREVECTOR_H
|
||||
#define FEATUREVECTOR_H
|
||||
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/functional/hash.hpp>
|
||||
#include <boost/unordered_map.hpp>
|
||||
|
||||
#ifdef MPI_ENABLED
|
||||
#include <boost/serialization/access.hpp>
|
||||
#include <boost/serialization/split_member.hpp>
|
||||
#include <boost/serialization/string.hpp>
|
||||
#include <boost/serialization/vector.hpp>
|
||||
#endif
|
||||
|
||||
namespace Moses {
|
||||
|
||||
typedef float FValue;
|
||||
|
||||
/**
|
||||
* Feature name
|
||||
**/
|
||||
struct FName {
|
||||
|
||||
static const std::string SEP;
|
||||
|
||||
typedef boost::unordered_map<std::string,size_t> Name2Id;
|
||||
//typedef std::map<std::string, size_t> Name2Id;
|
||||
static Name2Id name2id;
|
||||
static std::vector<std::string> id2name;
|
||||
|
||||
|
||||
//A feature name can either be initialised as a pair of strings,
|
||||
//which will be concatenated with a SEP between them, or as
|
||||
//a single string, which will be used as-is.
|
||||
FName(const std::string root, const std::string name)
|
||||
{init(root + SEP + name);}
|
||||
FName(const std::string& name)
|
||||
{init(name);}
|
||||
|
||||
const std::string& name() const;
|
||||
//const std::string& root() const {return m_root;}
|
||||
|
||||
size_t hash() const;
|
||||
|
||||
bool operator==(const FName& rhs) const ;
|
||||
bool operator!=(const FName& rhs) const ;
|
||||
|
||||
|
||||
private:
|
||||
void init(const std::string& name);
|
||||
size_t m_id;
|
||||
|
||||
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream& out,const FName& name);
|
||||
|
||||
struct FNameEquals {
|
||||
inline bool operator() (const FName& lhs, const FName& rhs) const {
|
||||
return (lhs == rhs);
|
||||
}
|
||||
};
|
||||
|
||||
struct FNameHash
|
||||
: std::unary_function<FName, std::size_t>
|
||||
{
|
||||
std::size_t operator()(const FName& x) const
|
||||
{
|
||||
return x.hash();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class ProxyFVector;
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* A sparse feature (or weight) vector.
|
||||
**/
|
||||
class FVector
|
||||
{
|
||||
public:
|
||||
/** Empty feature vector, possibly with default value */
|
||||
FVector(FValue defaultValue = DEFAULT);
|
||||
|
||||
//defaults
|
||||
static FName DEFAULT_NAME;
|
||||
static const FValue DEFAULT;
|
||||
|
||||
|
||||
void clear();
|
||||
|
||||
|
||||
bool hasNonDefaultValue(FName name) const { return m_features.find(name) != m_features.end();}
|
||||
|
||||
/** Load from file - each line should be 'root[_name] value' */
|
||||
void load(const std::string& filename);
|
||||
void save(const std::string& filename) const;
|
||||
void write(std::ostream& out) const ;
|
||||
|
||||
/** Element access */
|
||||
ProxyFVector operator[](const FName& name);
|
||||
FValue operator[](const FName& name) const;
|
||||
|
||||
/** Size */
|
||||
size_t size() const {return m_features.size();}
|
||||
|
||||
/** Equality */
|
||||
bool operator== (const FVector& rhs) const;
|
||||
bool operator!= (const FVector& rhs) const;
|
||||
|
||||
|
||||
|
||||
friend class ProxyFVector;
|
||||
|
||||
/**arithmetic */
|
||||
//Element-wise
|
||||
FVector& operator+= (const FVector& rhs);
|
||||
FVector& operator-= (const FVector& rhs);
|
||||
FVector& operator*= (const FVector& rhs);
|
||||
FVector& operator/= (const FVector& rhs);
|
||||
//Scalar
|
||||
FVector& operator+= (const FValue& rhs);
|
||||
FVector& operator-= (const FValue& rhs);
|
||||
FVector& operator*= (const FValue& rhs);
|
||||
FVector& operator/= (const FValue& rhs);
|
||||
|
||||
FValue inner_product(const FVector& rhs) const;
|
||||
|
||||
FVector& max_equals(const FVector& rhs);
|
||||
|
||||
/** norms and sums */
|
||||
FValue l1norm() const;
|
||||
FValue l2norm() const;
|
||||
FValue sum() const;
|
||||
|
||||
/** printing */
|
||||
std::ostream& print(std::ostream& out) const;
|
||||
|
||||
#ifdef MPI_ENABLED
|
||||
friend class boost::serialization::access;
|
||||
#endif
|
||||
|
||||
private:
|
||||
typedef boost::unordered_map<FName,FValue,FNameHash, FNameEquals> FNVmap;
|
||||
|
||||
/** Internal get and set. Note that the get() doesn't include the
|
||||
default value */
|
||||
const FValue& get(const FName& name) const;
|
||||
void set(const FName& name, const FValue& value);
|
||||
|
||||
/** Iterators */
|
||||
typedef FNVmap::iterator iterator;
|
||||
typedef FNVmap::const_iterator const_iterator;
|
||||
iterator begin() {return m_features.begin();}
|
||||
iterator end() {return m_features.end();}
|
||||
const_iterator begin() const {return m_features.begin();}
|
||||
const_iterator end() const {return m_features.end();}
|
||||
|
||||
|
||||
FNVmap m_features;
|
||||
|
||||
#ifdef MPI_ENABLED
|
||||
//serialization
|
||||
template<class Archive>
|
||||
void save(Archive &ar, const unsigned int version) const {
|
||||
std::vector<std::string> names;
|
||||
std::vector<FValue> values;
|
||||
for (const_iterator i = begin(); i != end(); ++i) {
|
||||
std::ostringstream ostr;
|
||||
ostr << i->first;
|
||||
names.push_back(ostr.str());
|
||||
values.push_back(i->second);
|
||||
}
|
||||
ar << names;
|
||||
ar << values;
|
||||
}
|
||||
|
||||
template<class Archive>
|
||||
void load(Archive &ar, const unsigned int version) {
|
||||
clear();
|
||||
std::vector<std::string> names;
|
||||
std::vector<FValue> values;
|
||||
ar >> names;
|
||||
ar >> values;
|
||||
assert (names.size() == values.size());
|
||||
for (size_t i = 0; i < names.size(); ++i) {
|
||||
set(FName(names[i]), values[i]);
|
||||
}
|
||||
}
|
||||
|
||||
BOOST_SERIALIZATION_SPLIT_MEMBER()
|
||||
|
||||
#endif
|
||||
|
||||
};
|
||||
|
||||
std::ostream& operator<<( std::ostream& out, const FVector& fv);
|
||||
//Element-wise operations
|
||||
const FVector operator+(const FVector& lhs, const FVector& rhs);
|
||||
const FVector operator-(const FVector& lhs, const FVector& rhs);
|
||||
const FVector operator*(const FVector& lhs, const FVector& rhs);
|
||||
const FVector operator/(const FVector& lhs, const FVector& rhs);
|
||||
|
||||
//Scalar operations
|
||||
const FVector operator+(const FVector& lhs, const FValue& rhs);
|
||||
const FVector operator-(const FVector& lhs, const FValue& rhs);
|
||||
const FVector operator*(const FVector& lhs, const FValue& rhs);
|
||||
const FVector operator/(const FVector& lhs, const FValue& rhs);
|
||||
|
||||
const FVector fvmax(const FVector& lhs, const FVector& rhs);
|
||||
|
||||
FValue inner_product(const FVector& lhs, const FVector& rhs);
|
||||
|
||||
struct FVectorPlus {
|
||||
FVector operator()(const FVector& lhs, const FVector& rhs) const {
|
||||
return lhs + rhs;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Used to help with subscript operator overloading.
|
||||
* See http://stackoverflow.com/questions/1386075/overloading-operator-for-a-sparse-vector
|
||||
**/
|
||||
class ProxyFVector {
|
||||
public:
|
||||
ProxyFVector(FVector *fv, const FName& name ) : m_fv(fv), m_name(name) {}
|
||||
ProxyFVector &operator=(const FValue& value) {
|
||||
// If we get here, we know that operator[] was called to perform a write access,
|
||||
// so we can insert an item in the vector if needed
|
||||
//std::cerr << "Inserting " << value << " into " << m_name << std::endl;
|
||||
m_fv->set(m_name,value-m_fv->get(FVector::DEFAULT_NAME));
|
||||
return *this;
|
||||
|
||||
}
|
||||
|
||||
operator FValue() {
|
||||
// If we get here, we know that operator[] was called to perform a read access,
|
||||
// so we can simply return the value from the vector
|
||||
return m_fv->get(m_name) + m_fv->get(FVector::DEFAULT_NAME);
|
||||
}
|
||||
|
||||
/*operator FValue&() {
|
||||
return m_fv->m_features[m_name];
|
||||
}*/
|
||||
|
||||
FValue operator++() {
|
||||
return ++m_fv->m_features[m_name] + m_fv->get(FVector::DEFAULT_NAME);
|
||||
}
|
||||
|
||||
FValue operator +=(FValue lhs) {
|
||||
return (m_fv->m_features[m_name] += lhs) + m_fv->get(FVector::DEFAULT_NAME);
|
||||
}
|
||||
|
||||
private:
|
||||
FValue m_tmp;
|
||||
|
||||
private:
|
||||
FVector* m_fv;
|
||||
const FName& m_name;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
Loading…
Reference in New Issue
Block a user