mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-28 06:22:14 +03:00
116 lines
3.5 KiB
C++
116 lines
3.5 KiB
C++
#ifndef LM_BUILDER_PRINT_H
|
|
#define LM_BUILDER_PRINT_H
|
|
|
|
#include "lm/builder/ngram.hh"
|
|
#include "lm/builder/ngram_stream.hh"
|
|
#include "lm/builder/output.hh"
|
|
#include "util/fake_ofstream.hh"
|
|
#include "util/file.hh"
|
|
#include "util/mmap.hh"
|
|
#include "util/string_piece.hh"
|
|
|
|
#include <ostream>
|
|
|
|
#include <assert.h>
|
|
|
|
// Warning: print routines read all unigrams before all bigrams before all
|
|
// trigrams etc. So if other parts of the chain move jointly, you'll have to
|
|
// buffer.
|
|
|
|
namespace lm { namespace builder {
|
|
|
|
class VocabReconstitute {
|
|
public:
|
|
// fd must be alive for life of this object; does not take ownership.
|
|
explicit VocabReconstitute(int fd);
|
|
|
|
const char *Lookup(WordIndex index) const {
|
|
assert(index < map_.size() - 1);
|
|
return map_[index];
|
|
}
|
|
|
|
StringPiece LookupPiece(WordIndex index) const {
|
|
return StringPiece(map_[index], map_[index + 1] - 1 - map_[index]);
|
|
}
|
|
|
|
std::size_t Size() const {
|
|
// There's an extra entry to support StringPiece lengths.
|
|
return map_.size() - 1;
|
|
}
|
|
|
|
private:
|
|
util::scoped_memory memory_;
|
|
std::vector<const char*> map_;
|
|
};
|
|
|
|
// Not defined, only specialized.
|
|
template <class T> void PrintPayload(util::FakeOFStream &to, const Payload &payload);
|
|
template <> inline void PrintPayload<uint64_t>(util::FakeOFStream &to, const Payload &payload) {
|
|
// TODO slow
|
|
to << boost::lexical_cast<std::string>(payload.count);
|
|
}
|
|
template <> inline void PrintPayload<Uninterpolated>(util::FakeOFStream &to, const Payload &payload) {
|
|
to << log10(payload.uninterp.prob) << ' ' << log10(payload.uninterp.gamma);
|
|
}
|
|
template <> inline void PrintPayload<ProbBackoff>(util::FakeOFStream &to, const Payload &payload) {
|
|
to << payload.complete.prob << ' ' << payload.complete.backoff;
|
|
}
|
|
|
|
// template parameter is the type stored.
|
|
template <class V> class Print {
|
|
public:
|
|
static void DumpSeparateFiles(const VocabReconstitute &vocab, const std::string &file_base, util::stream::Chains &chains) {
|
|
for (unsigned int i = 0; i < chains.size(); ++i) {
|
|
std::string file(file_base + boost::lexical_cast<std::string>(i));
|
|
chains[i] >> Print(vocab, util::CreateOrThrow(file.c_str()));
|
|
}
|
|
}
|
|
|
|
explicit Print(const VocabReconstitute &vocab, int fd) : vocab_(vocab), to_(fd) {}
|
|
|
|
void Run(const util::stream::ChainPositions &chains) {
|
|
util::scoped_fd fd(to_);
|
|
util::FakeOFStream out(to_);
|
|
NGramStreams streams(chains);
|
|
for (NGramStream *s = streams.begin(); s != streams.end(); ++s) {
|
|
DumpStream(*s, out);
|
|
}
|
|
}
|
|
|
|
void Run(const util::stream::ChainPosition &position) {
|
|
util::scoped_fd fd(to_);
|
|
util::FakeOFStream out(to_);
|
|
NGramStream stream(position);
|
|
DumpStream(stream, out);
|
|
}
|
|
|
|
private:
|
|
void DumpStream(NGramStream &stream, util::FakeOFStream &to) {
|
|
for (; stream; ++stream) {
|
|
PrintPayload<V>(to, stream->Value());
|
|
for (const WordIndex *w = stream->begin(); w != stream->end(); ++w) {
|
|
to << ' ' << vocab_.Lookup(*w) << '=' << *w;
|
|
}
|
|
to << '\n';
|
|
}
|
|
}
|
|
|
|
const VocabReconstitute &vocab_;
|
|
int to_;
|
|
};
|
|
|
|
class PrintARPA : public OutputHook {
|
|
public:
|
|
explicit PrintARPA(int fd, bool verbose_header)
|
|
: OutputHook(PROB_SEQUENTIAL_HOOK), out_fd_(fd), verbose_header_(verbose_header) {}
|
|
|
|
void Run(const util::stream::ChainPositions &positions);
|
|
|
|
private:
|
|
util::scoped_fd out_fd_;
|
|
bool verbose_header_;
|
|
};
|
|
|
|
}} // namespaces
|
|
#endif // LM_BUILDER_PRINT_H
|