mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-26 21:42:19 +03:00
121 lines
4.0 KiB
C++
121 lines
4.0 KiB
C++
/***********************************************************************
|
|
Moses - factored phrase-based language decoder
|
|
Copyright (C) 2014- University of Edinburgh
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with this library; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
***********************************************************************/
|
|
#ifndef MERT_FOREST_RESCORE_H
|
|
#define MERT_FOREST_RESCORE_H
|
|
|
|
#include <valarray>
|
|
#include <vector>
|
|
|
|
#include <boost/unordered_set.hpp>
|
|
|
|
#include "BleuScorer.h"
|
|
#include "Hypergraph.h"
|
|
|
|
namespace MosesTuning {
|
|
|
|
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec);
|
|
|
|
struct NgramHash : public std::unary_function<const WordVec&, std::size_t> {
|
|
std::size_t operator()(const WordVec& ngram) const {
|
|
return util::MurmurHashNative(&(ngram[0]), ngram.size() * sizeof(WordVec::value_type));
|
|
}
|
|
};
|
|
|
|
struct NgramEquals : public std::binary_function<const WordVec&, const WordVec&, bool> {
|
|
bool operator()(const WordVec& first, const WordVec& second) const {
|
|
if (first.size() != second.size()) return false;
|
|
return memcmp(&(first[0]), &(second[0]), first.size() * sizeof(WordVec::value_type)) == 0;
|
|
}
|
|
};
|
|
|
|
typedef boost::unordered_map<WordVec, size_t, NgramHash, NgramEquals> NgramCounter;
|
|
|
|
|
|
class ReferenceSet {
|
|
|
|
|
|
public:
|
|
|
|
void AddLine(size_t sentenceId, const StringPiece& line, Vocab& vocab);
|
|
|
|
void Load(const std::vector<std::string>& files, Vocab& vocab);
|
|
|
|
size_t NgramMatches(size_t sentenceId, const WordVec&, bool clip) const;
|
|
|
|
size_t Length(size_t sentenceId) const {return lengths_[sentenceId];}
|
|
|
|
private:
|
|
//ngrams to (clipped,unclipped) counts
|
|
typedef boost::unordered_map<WordVec, std::pair<std::size_t,std::size_t>, NgramHash,NgramEquals> NgramMap;
|
|
std::vector<NgramMap> ngramCounts_;
|
|
std::vector<size_t> lengths_;
|
|
|
|
};
|
|
|
|
struct VertexState {
|
|
VertexState();
|
|
|
|
std::vector<FeatureStatsType> bleuStats;
|
|
WordVec leftContext;
|
|
WordVec rightContext;
|
|
size_t targetLength;
|
|
};
|
|
|
|
/**
|
|
* Used to score an rule (ie edge) when we are applying it.
|
|
**/
|
|
class HgBleuScorer {
|
|
public:
|
|
HgBleuScorer(const ReferenceSet& references, const Graph& graph, size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu):
|
|
references_(references), sentenceId_(sentenceId), graph_(graph), backgroundBleu_(backgroundBleu),
|
|
backgroundRefLength_(backgroundBleu[kBleuNgramOrder*2]) {
|
|
vertexStates_.resize(graph.VertexSize());
|
|
totalSourceLength_ = graph.GetVertex(graph.VertexSize()-1).SourceCovered();
|
|
}
|
|
|
|
FeatureStatsType Score(const Edge& edge, const Vertex& head, std::vector<FeatureStatsType>& bleuStats) ;
|
|
|
|
void UpdateState(const Edge& winnerEdge, size_t vertexId, const std::vector<FeatureStatsType>& bleuStats);
|
|
|
|
|
|
private:
|
|
const ReferenceSet& references_;
|
|
std::vector<VertexState> vertexStates_;
|
|
size_t sentenceId_;
|
|
size_t totalSourceLength_;
|
|
const Graph& graph_;
|
|
std::vector<FeatureStatsType> backgroundBleu_;
|
|
FeatureStatsType backgroundRefLength_;
|
|
|
|
void UpdateMatches(const NgramCounter& counter, std::vector<FeatureStatsType>& bleuStats) const;
|
|
size_t GetTargetLength(const Edge& edge) const;
|
|
};
|
|
|
|
struct HgHypothesis {
|
|
SparseVector featureVector;
|
|
WordVec text;
|
|
std::vector<FeatureStatsType> bleuStats;
|
|
};
|
|
|
|
void Viterbi(const Graph& graph, const SparseVector& weights, float bleuWeight, const ReferenceSet& references, size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu, HgHypothesis* bestHypo);
|
|
|
|
};
|
|
|
|
#endif
|