mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-11-10 10:59:21 +03:00
Merge branch 'master' of ssh://github.com/moses-smt/mosesdecoder
This commit is contained in:
phikoehn
commit
14a499d9aa
@ -52,6 +52,7 @@ void Usage(const char *name, const char *default_mem) {
|
||||
"-a compresses pointers using an array of offsets. The parameter is the\n"
|
||||
" maximum number of bits encoded by the array. Memory is minimized subject\n"
|
||||
" to the maximum, so pick 255 to minimize memory.\n\n"
|
||||
"-h print this help message.\n\n"
|
||||
"Get a memory estimate by passing an ARPA file without an output file name.\n";
|
||||
exit(1);
|
||||
}
|
||||
@ -104,12 +105,15 @@ int main(int argc, char *argv[]) {
|
||||
|
||||
const char *default_mem = util::GuessPhysicalMemory() ? "80%" : "1G";
|
||||
|
||||
if (argc == 2 && !strcmp(argv[1], "--help"))
|
||||
Usage(argv[0], default_mem);
|
||||
|
||||
try {
|
||||
bool quantize = false, set_backoff_bits = false, bhiksha = false, set_write_method = false, rest = false;
|
||||
lm::ngram::Config config;
|
||||
config.building_memory = util::ParseSize(default_mem);
|
||||
int opt;
|
||||
while ((opt = getopt(argc, argv, "q:b:a:u:p:t:T:m:S:w:sir:")) != -1) {
|
||||
while ((opt = getopt(argc, argv, "q:b:a:u:p:t:T:m:S:w:sir:h")) != -1) {
|
||||
switch(opt) {
|
||||
case 'q':
|
||||
config.prob_bits = ParseBitCount(optarg);
|
||||
@ -161,6 +165,7 @@ int main(int argc, char *argv[]) {
|
||||
ParseFileList(optarg, config.rest_lower_files);
|
||||
config.rest_function = Config::REST_LOWER;
|
||||
break;
|
||||
case 'h': // help
|
||||
default:
|
||||
Usage(argv[0], default_mem);
|
||||
}
|
||||
|
||||
@ -238,12 +238,14 @@ void CorpusCount::Run(const util::stream::ChainPosition &position) {
|
||||
const WordIndex end_sentence = vocab.Lookup("</s>");
|
||||
Writer writer(NGram::OrderFromSize(position.GetChain().EntrySize()), position, dedupe_mem_.get(), dedupe_mem_size_);
|
||||
uint64_t count = 0;
|
||||
StringPiece delimiters("\0\t\r ", 4);
|
||||
bool delimiters[256];
|
||||
memset(delimiters, 0, sizeof(delimiters));
|
||||
delimiters['\0'] = delimiters['\t'] = delimiters['\n'] = delimiters['\r'] = delimiters[' '] = true;
|
||||
try {
|
||||
while(true) {
|
||||
StringPiece line(from_.ReadLine());
|
||||
writer.StartSentence();
|
||||
for (util::TokenIter<util::AnyCharacter, true> w(line, delimiters); w; ++w) {
|
||||
for (util::TokenIter<util::BoolCharacter, true> w(line, delimiters); w; ++w) {
|
||||
WordIndex word = vocab.Lookup(*w);
|
||||
UTIL_THROW_IF(word <= 2, FormatLoadException, "Special word " << *w << " is not allowed in the corpus. I plan to support models containing <unk> in the future.");
|
||||
writer.Append(word);
|
||||
|
||||
@ -36,6 +36,7 @@ int main(int argc, char *argv[]) {
|
||||
std::string text, arpa;
|
||||
|
||||
options.add_options()
|
||||
("help", po::bool_switch(), "Show this help message")
|
||||
("order,o", po::value<std::size_t>(&pipeline.order)
|
||||
#if BOOST_VERSION >= 104200
|
||||
->required()
|
||||
@ -52,7 +53,10 @@ int main(int argc, char *argv[]) {
|
||||
("verbose_header", po::bool_switch(&pipeline.verbose_header), "Add a verbose header to the ARPA file that includes information such as token count, smoothing type, etc.")
|
||||
("text", po::value<std::string>(&text), "Read text from a file instead of stdin")
|
||||
("arpa", po::value<std::string>(&arpa), "Write ARPA to a file instead of stdout");
|
||||
if (argc == 1) {
|
||||
po::variables_map vm;
|
||||
po::store(po::parse_command_line(argc, argv, options), vm);
|
||||
|
||||
if (argc == 1 || vm["help"].as<bool>()) {
|
||||
std::cerr <<
|
||||
"Builds unpruned language models with modified Kneser-Ney smoothing.\n\n"
|
||||
"Please cite:\n"
|
||||
@ -70,12 +74,17 @@ int main(int argc, char *argv[]) {
|
||||
"setting the temporary file location (-T) and sorting memory (-S) is recommended.\n\n"
|
||||
"Memory sizes are specified like GNU sort: a number followed by a unit character.\n"
|
||||
"Valid units are \% for percentage of memory (supported platforms only) and (in\n"
|
||||
"increasing powers of 1024): b, K, M, G, T, P, E, Z, Y. Default is K (*1024).\n\n";
|
||||
"increasing powers of 1024): b, K, M, G, T, P, E, Z, Y. Default is K (*1024).\n";
|
||||
uint64_t mem = util::GuessPhysicalMemory();
|
||||
if (mem) {
|
||||
std::cerr << "This machine has " << mem << " bytes of memory.\n\n";
|
||||
} else {
|
||||
std::cerr << "Unable to determine the amount of memory on this machine.\n\n";
|
||||
}
|
||||
std::cerr << options << std::endl;
|
||||
return 1;
|
||||
}
|
||||
po::variables_map vm;
|
||||
po::store(po::parse_command_line(argc, argv, options), vm);
|
||||
|
||||
po::notify(vm);
|
||||
|
||||
// required() appeared in Boost 1.42.0.
|
||||
|
||||
@ -226,6 +226,7 @@ void CountText(int text_file /* input */, int vocab_file /* output */, Master &m
|
||||
|
||||
util::stream::Sort<SuffixOrder, AddCombiner> sorter(chain, config.sort, SuffixOrder(config.order), AddCombiner());
|
||||
chain.Wait(true);
|
||||
std::cerr << "Unigram tokens " << token_count << " types " << type_count << std::endl;
|
||||
std::cerr << "=== 2/5 Calculating and sorting adjusted counts ===" << std::endl;
|
||||
master.InitForAdjust(sorter, type_count);
|
||||
}
|
||||
|
||||
19
lm/facade.hh
19
lm/facade.hh
@ -16,11 +16,6 @@ template <class Child, class StateT, class VocabularyT> class ModelFacade : publ
|
||||
typedef StateT State;
|
||||
typedef VocabularyT Vocabulary;
|
||||
|
||||
// Default Score function calls FullScore. Model can override this.
|
||||
float Score(const State &in_state, const WordIndex new_word, State &out_state) const {
|
||||
return static_cast<const Child*>(this)->FullScore(in_state, new_word, out_state).prob;
|
||||
}
|
||||
|
||||
/* Translate from void* to State */
|
||||
FullScoreReturn FullScore(const void *in_state, const WordIndex new_word, void *out_state) const {
|
||||
return static_cast<const Child*>(this)->FullScore(
|
||||
@ -28,6 +23,20 @@ template <class Child, class StateT, class VocabularyT> class ModelFacade : publ
|
||||
new_word,
|
||||
*reinterpret_cast<State*>(out_state));
|
||||
}
|
||||
|
||||
FullScoreReturn FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, void *out_state) const {
|
||||
return static_cast<const Child*>(this)->FullScoreForgotState(
|
||||
context_rbegin,
|
||||
context_rend,
|
||||
new_word,
|
||||
*reinterpret_cast<State*>(out_state));
|
||||
}
|
||||
|
||||
// Default Score function calls FullScore. Model can override this.
|
||||
float Score(const State &in_state, const WordIndex new_word, State &out_state) const {
|
||||
return static_cast<const Child*>(this)->FullScore(in_state, new_word, out_state).prob;
|
||||
}
|
||||
|
||||
float Score(const void *in_state, const WordIndex new_word, void *out_state) const {
|
||||
return static_cast<const Child*>(this)->Score(
|
||||
*reinterpret_cast<const State*>(in_state),
|
||||
|
||||
@ -39,17 +39,15 @@ template <class FilterT> class ContextFilter {
|
||||
explicit ContextFilter(Filter &backend) : backend_(backend) {}
|
||||
|
||||
template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
|
||||
pieces_.clear();
|
||||
// TODO: this copy could be avoided by a lookahead iterator.
|
||||
std::copy(util::TokenIter<util::SingleCharacter, true>(ngram, ' '), util::TokenIter<util::SingleCharacter, true>::end(), std::back_insert_iterator<std::vector<StringPiece> >(pieces_));
|
||||
backend_.AddNGram(pieces_.begin(), pieces_.end() - !pieces_.empty(), line, output);
|
||||
// Find beginning of string or last space.
|
||||
const char *last_space;
|
||||
for (last_space = ngram.data() + ngram.size() - 1; last_space > ngram.data() && *last_space != ' '; --last_space) {}
|
||||
backend_.AddNGram(StringPiece(ngram.data(), last_space - ngram.data()), line, output);
|
||||
}
|
||||
|
||||
void Flush() const {}
|
||||
|
||||
private:
|
||||
std::vector<StringPiece> pieces_;
|
||||
|
||||
Filter backend_;
|
||||
};
|
||||
|
||||
|
||||
@ -11,21 +11,25 @@
|
||||
#include <istream>
|
||||
#include <string>
|
||||
|
||||
#include <math.h>
|
||||
|
||||
namespace lm {
|
||||
namespace ngram {
|
||||
|
||||
template <class Model> void Query(const Model &model, bool sentence_context, std::istream &in_stream, std::ostream &out_stream) {
|
||||
std::cerr << "Loading statistics:\n";
|
||||
util::PrintUsage(std::cerr);
|
||||
typename Model::State state, out;
|
||||
lm::FullScoreReturn ret;
|
||||
std::string word;
|
||||
|
||||
double corpus_total = 0.0;
|
||||
uint64_t corpus_oov = 0;
|
||||
uint64_t corpus_tokens = 0;
|
||||
|
||||
while (in_stream) {
|
||||
state = sentence_context ? model.BeginSentenceState() : model.NullContextState();
|
||||
float total = 0.0;
|
||||
bool got = false;
|
||||
unsigned int oov = 0;
|
||||
uint64_t oov = 0;
|
||||
while (in_stream >> word) {
|
||||
got = true;
|
||||
lm::WordIndex vocab = model.GetVocabulary().Index(word);
|
||||
@ -33,6 +37,7 @@ template <class Model> void Query(const Model &model, bool sentence_context, std
|
||||
ret = model.FullScore(state, vocab, out);
|
||||
total += ret.prob;
|
||||
out_stream << word << '=' << vocab << ' ' << static_cast<unsigned int>(ret.ngram_length) << ' ' << ret.prob << '\t';
|
||||
++corpus_tokens;
|
||||
state = out;
|
||||
char c;
|
||||
while (true) {
|
||||
@ -50,12 +55,14 @@ template <class Model> void Query(const Model &model, bool sentence_context, std
|
||||
if (sentence_context) {
|
||||
ret = model.FullScore(state, model.GetVocabulary().EndSentence(), out);
|
||||
total += ret.prob;
|
||||
++corpus_tokens;
|
||||
out_stream << "</s>=" << model.GetVocabulary().EndSentence() << ' ' << static_cast<unsigned int>(ret.ngram_length) << ' ' << ret.prob << '\t';
|
||||
}
|
||||
out_stream << "Total: " << total << " OOV: " << oov << '\n';
|
||||
corpus_total += total;
|
||||
corpus_oov += oov;
|
||||
}
|
||||
std::cerr << "After queries:\n";
|
||||
util::PrintUsage(std::cerr);
|
||||
out_stream << "Perplexity " << pow(10.0, -(corpus_total / static_cast<double>(corpus_tokens))) << std::endl;
|
||||
}
|
||||
|
||||
template <class M> void Query(const char *file, bool sentence_context, std::istream &in_stream, std::ostream &out_stream) {
|
||||
|
||||
@ -1,42 +1,65 @@
|
||||
#include "lm/ngram_query.hh"
|
||||
|
||||
#ifdef WITH_NPLM
|
||||
#include "lm/wrappers/nplm.hh"
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
void Usage(const char *name) {
|
||||
std::cerr << "KenLM was compiled with maximum order " << KENLM_MAX_ORDER << "." << std::endl;
|
||||
std::cerr << "Usage: " << name << " [-n] lm_file" << std::endl;
|
||||
std::cerr << "Input is wrapped in <s> and </s> unless -n is passed." << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
if (!(argc == 2 || (argc == 3 && !strcmp(argv[2], "null")))) {
|
||||
std::cerr << "KenLM was compiled with maximum order " << KENLM_MAX_ORDER << "." << std::endl;
|
||||
std::cerr << "Usage: " << argv[0] << " lm_file [null]" << std::endl;
|
||||
std::cerr << "Input is wrapped in <s> and </s> unless null is passed." << std::endl;
|
||||
return 1;
|
||||
bool sentence_context = true;
|
||||
const char *file = NULL;
|
||||
for (char **arg = argv + 1; arg != argv + argc; ++arg) {
|
||||
if (!strcmp(*arg, "-n")) {
|
||||
sentence_context = false;
|
||||
} else if (!strcmp(*arg, "-h") || !strcmp(*arg, "--help") || file) {
|
||||
Usage(argv[0]);
|
||||
} else {
|
||||
file = *arg;
|
||||
}
|
||||
}
|
||||
if (!file) Usage(argv[0]);
|
||||
try {
|
||||
bool sentence_context = (argc == 2);
|
||||
using namespace lm::ngram;
|
||||
ModelType model_type;
|
||||
if (RecognizeBinary(argv[1], model_type)) {
|
||||
if (RecognizeBinary(file, model_type)) {
|
||||
switch(model_type) {
|
||||
case PROBING:
|
||||
Query<lm::ngram::ProbingModel>(argv[1], sentence_context, std::cin, std::cout);
|
||||
Query<lm::ngram::ProbingModel>(file, sentence_context, std::cin, std::cout);
|
||||
break;
|
||||
case REST_PROBING:
|
||||
Query<lm::ngram::RestProbingModel>(argv[1], sentence_context, std::cin, std::cout);
|
||||
Query<lm::ngram::RestProbingModel>(file, sentence_context, std::cin, std::cout);
|
||||
break;
|
||||
case TRIE:
|
||||
Query<TrieModel>(argv[1], sentence_context, std::cin, std::cout);
|
||||
Query<TrieModel>(file, sentence_context, std::cin, std::cout);
|
||||
break;
|
||||
case QUANT_TRIE:
|
||||
Query<QuantTrieModel>(argv[1], sentence_context, std::cin, std::cout);
|
||||
Query<QuantTrieModel>(file, sentence_context, std::cin, std::cout);
|
||||
break;
|
||||
case ARRAY_TRIE:
|
||||
Query<ArrayTrieModel>(argv[1], sentence_context, std::cin, std::cout);
|
||||
Query<ArrayTrieModel>(file, sentence_context, std::cin, std::cout);
|
||||
break;
|
||||
case QUANT_ARRAY_TRIE:
|
||||
Query<QuantArrayTrieModel>(argv[1], sentence_context, std::cin, std::cout);
|
||||
Query<QuantArrayTrieModel>(file, sentence_context, std::cin, std::cout);
|
||||
break;
|
||||
default:
|
||||
std::cerr << "Unrecognized kenlm model type " << model_type << std::endl;
|
||||
abort();
|
||||
}
|
||||
#ifdef WITH_NPLM
|
||||
} else if (lm::np::Model::Recognize(file)) {
|
||||
lm::np::Model model(file);
|
||||
Query(model, sentence_context, std::cin, std::cout);
|
||||
#endif
|
||||
} else {
|
||||
Query<ProbingModel>(argv[1], sentence_context, std::cin, std::cout);
|
||||
Query<ProbingModel>(file, sentence_context, std::cin, std::cout);
|
||||
}
|
||||
std::cerr << "Total time including destruction:\n";
|
||||
util::PrintUsage(std::cerr);
|
||||
|
||||
@ -150,7 +150,7 @@ void PositiveProbWarn::Warn(float prob) {
|
||||
case THROW_UP:
|
||||
UTIL_THROW(FormatLoadException, "Positive log probability " << prob << " in the model. This is a bug in IRSTLM; you can set config.positive_log_probability = SILENT or pass -i to build_binary to substitute 0.0 for the log probability. Error");
|
||||
case COMPLAIN:
|
||||
std::cerr << "There's a positive log probability " << prob << " in the APRA file, probably because of a bug in IRSTLM. This and subsequent entires will be mapepd to 0 log probability." << std::endl;
|
||||
std::cerr << "There's a positive log probability " << prob << " in the APRA file, probably because of a bug in IRSTLM. This and subsequent entires will be mapped to 0 log probability." << std::endl;
|
||||
action_ = SILENT;
|
||||
break;
|
||||
case SILENT:
|
||||
|
||||
@ -253,11 +253,6 @@ class FindBlanks {
|
||||
++counts_.back();
|
||||
}
|
||||
|
||||
// Unigrams wrote one past.
|
||||
void Cleanup() {
|
||||
--counts_[0];
|
||||
}
|
||||
|
||||
const std::vector<uint64_t> &Counts() const {
|
||||
return counts_;
|
||||
}
|
||||
@ -310,8 +305,6 @@ template <class Quant, class Bhiksha> class WriteEntries {
|
||||
typename Quant::LongestPointer(quant_, longest_.Insert(words[order_ - 1])).Write(reinterpret_cast<const Prob*>(words + order_)->prob);
|
||||
}
|
||||
|
||||
void Cleanup() {}
|
||||
|
||||
private:
|
||||
RecordReader *contexts_;
|
||||
const Quant &quant_;
|
||||
@ -385,14 +378,14 @@ template <class Doing> void RecursiveInsert(const unsigned char total_order, con
|
||||
util::ErsatzProgress progress(unigram_count + 1, progress_out, message);
|
||||
WordIndex unigram = 0;
|
||||
std::priority_queue<Gram> grams;
|
||||
grams.push(Gram(&unigram, 1));
|
||||
if (unigram_count) grams.push(Gram(&unigram, 1));
|
||||
for (unsigned char i = 2; i <= total_order; ++i) {
|
||||
if (input[i-2]) grams.push(Gram(reinterpret_cast<const WordIndex*>(input[i-2].Data()), i));
|
||||
}
|
||||
|
||||
BlankManager<Doing> blank(total_order, doing);
|
||||
|
||||
while (true) {
|
||||
while (!grams.empty()) {
|
||||
Gram top = grams.top();
|
||||
grams.pop();
|
||||
unsigned char order = top.end - top.begin;
|
||||
@ -400,8 +393,7 @@ template <class Doing> void RecursiveInsert(const unsigned char total_order, con
|
||||
blank.Visit(&unigram, 1, doing.UnigramProb(unigram));
|
||||
doing.Unigram(unigram);
|
||||
progress.Set(unigram);
|
||||
if (++unigram == unigram_count + 1) break;
|
||||
grams.push(top);
|
||||
if (++unigram < unigram_count) grams.push(top);
|
||||
} else {
|
||||
if (order == total_order) {
|
||||
blank.Visit(top.begin, order, reinterpret_cast<const Prob*>(top.end)->prob);
|
||||
@ -414,8 +406,6 @@ template <class Doing> void RecursiveInsert(const unsigned char total_order, con
|
||||
if (++reader) grams.push(top);
|
||||
}
|
||||
}
|
||||
assert(grams.empty());
|
||||
doing.Cleanup();
|
||||
}
|
||||
|
||||
void SanityCheckCounts(const std::vector<uint64_t> &initial, const std::vector<uint64_t> &fixed) {
|
||||
@ -524,6 +514,8 @@ template <class Quant, class Bhiksha> void BuildTrie(SortedFiles &files, std::ve
|
||||
{
|
||||
WriteEntries<Quant, Bhiksha> writer(contexts, quant, unigrams, out.middle_begin_, out.longest_, counts.size(), sri);
|
||||
RecursiveInsert(counts.size(), counts[0], inputs, config.ProgressMessages(), "Writing trie", writer);
|
||||
// Write the last unigram entry, which is the end pointer for the bigrams.
|
||||
writer.Unigram(counts[0]);
|
||||
}
|
||||
|
||||
// Do not disable this error message or else too little state will be returned. Both WriteEntries::Middle and returning state based on found n-grams will need to be fixed to handle this situation.
|
||||
|
||||
@ -91,7 +91,7 @@ inline uint64_t hash_value(const Left &left) {
|
||||
}
|
||||
|
||||
struct ChartState {
|
||||
bool operator==(const ChartState &other) {
|
||||
bool operator==(const ChartState &other) const {
|
||||
return (right == other.right) && (left == other.left);
|
||||
}
|
||||
|
||||
@ -102,7 +102,7 @@ struct ChartState {
|
||||
}
|
||||
|
||||
bool operator<(const ChartState &other) const {
|
||||
return Compare(other) == -1;
|
||||
return Compare(other) < 0;
|
||||
}
|
||||
|
||||
void ZeroRemaining() {
|
||||
|
||||
@ -130,6 +130,9 @@ class Model {
|
||||
// Requires in_state != out_state
|
||||
virtual FullScoreReturn FullScore(const void *in_state, const WordIndex new_word, void *out_state) const = 0;
|
||||
|
||||
// Prefer to use FullScore. The context words should be provided in reverse order.
|
||||
virtual FullScoreReturn FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, void *out_state) const = 0;
|
||||
|
||||
unsigned char Order() const { return order_; }
|
||||
|
||||
const Vocabulary &BaseVocabulary() const { return *base_vocab_; }
|
||||
|
||||
@ -1,35 +1,20 @@
|
||||
#include <ctime>
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include "Util.h"
|
||||
#include "Timer.h"
|
||||
|
||||
#include "util/usage.hh"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
/***
|
||||
* Return the total time that the timer has been in the "running"
|
||||
* state since it was first "started" or last "restarted". For
|
||||
* "short" time periods (less than an hour), the actual cpu time
|
||||
* used is reported instead of the elapsed time.
|
||||
*/
|
||||
double Timer::elapsed_time()
|
||||
{
|
||||
time_t now;
|
||||
time(&now);
|
||||
return difftime(now, start_time);
|
||||
}
|
||||
|
||||
/***
|
||||
* Return the total time that the timer has been in the "running"
|
||||
* state since it was first "started" or last "restarted". For
|
||||
* "short" time periods (less than an hour), the actual cpu time
|
||||
* used is reported instead of the elapsed time.
|
||||
* This function is the public version of elapsed_time()
|
||||
* Return the total wall time that the timer has been in the "running"
|
||||
* state since it was first "started".
|
||||
*/
|
||||
double Timer::get_elapsed_time()
|
||||
{
|
||||
return elapsed_time();
|
||||
return util::WallTime() - start_time;
|
||||
}
|
||||
|
||||
/***
|
||||
@ -47,44 +32,9 @@ void Timer::start(const char* msg)
|
||||
// Change timer status to running
|
||||
running = true;
|
||||
|
||||
// Set the start time;
|
||||
time(&start_time);
|
||||
start_time = util::WallTime();
|
||||
}
|
||||
|
||||
/***
|
||||
* Turn the timer off and start it again from 0. Print an optional message.
|
||||
*/
|
||||
/*
|
||||
inline void Timer::restart(const char* msg)
|
||||
{
|
||||
// Print an optional message, something like "Restarting timer t";
|
||||
if (msg) TRACE_ERR( msg << std::endl;
|
||||
|
||||
// Set the timer status to running
|
||||
running = true;
|
||||
|
||||
// Set the accumulated time to 0 and the start time to now
|
||||
acc_time = 0;
|
||||
start_clock = clock();
|
||||
start_time = time(0);
|
||||
}
|
||||
*/
|
||||
|
||||
/***
|
||||
* Stop the timer and print an optional message.
|
||||
*/
|
||||
/*
|
||||
inline void Timer::stop(const char* msg)
|
||||
{
|
||||
// Print an optional message, something like "Stopping timer t";
|
||||
check(msg);
|
||||
|
||||
// Recalculate and store the total accumulated time up until now
|
||||
if (running) acc_time += elapsed_time();
|
||||
|
||||
running = false;
|
||||
}
|
||||
*/
|
||||
/***
|
||||
* Print out an optional message followed by the current timer timing.
|
||||
*/
|
||||
@ -94,7 +44,7 @@ void Timer::check(const char* msg)
|
||||
if (msg) TRACE_ERR( msg << " : ");
|
||||
|
||||
// TRACE_ERR( "[" << std::setiosflags(std::ios::fixed) << std::setprecision(2) << (running ? elapsed_time() : 0) << "] seconds\n");
|
||||
TRACE_ERR( "[" << (running ? elapsed_time() : 0) << "] seconds\n");
|
||||
TRACE_ERR( "[" << (running ? get_elapsed_time() : 0) << "] seconds\n");
|
||||
}
|
||||
|
||||
/***
|
||||
@ -105,7 +55,7 @@ void Timer::check(const char* msg)
|
||||
std::ostream& operator<<(std::ostream& os, Timer& t)
|
||||
{
|
||||
//os << std::setprecision(2) << std::setiosflags(std::ios::fixed) << (t.running ? t.elapsed_time() : 0);
|
||||
os << (t.running ? t.elapsed_time() : 0);
|
||||
os << (t.running ? t.get_elapsed_time() : 0);
|
||||
return os;
|
||||
}
|
||||
|
||||
|
||||
@ -22,15 +22,12 @@ private:
|
||||
bool running;
|
||||
// note: this only has the resolution of seconds, we'd often like better resolution
|
||||
// we make our best effort to do this on a system-by-system basis
|
||||
time_t start_time;
|
||||
|
||||
// in seconds
|
||||
double elapsed_time();
|
||||
double start_time;
|
||||
|
||||
public:
|
||||
/***
|
||||
* 'running' is initially false. A timer needs to be explicitly started
|
||||
* using 'start' or 'restart'
|
||||
* using 'start'
|
||||
*/
|
||||
Timer() : running(false) {
|
||||
start_time = 0;
|
||||
|
||||
@ -74,7 +74,9 @@ StringPiece FilePiece::ReadLine(char delim) {
|
||||
}
|
||||
}
|
||||
if (at_end_) {
|
||||
if (position_ == position_end_) Shift();
|
||||
if (position_ == position_end_) {
|
||||
Shift();
|
||||
}
|
||||
return Consume(position_end_);
|
||||
}
|
||||
skip = position_end_ - position_;
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <iosfwd>
|
||||
#include <string>
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
|
||||
namespace util {
|
||||
@ -66,8 +67,14 @@ class FilePiece {
|
||||
|
||||
// Skip spaces defined by isspace.
|
||||
void SkipSpaces(const bool *delim = kSpaces) {
|
||||
assert(position_ <= position_end_);
|
||||
for (; ; ++position_) {
|
||||
if (position_ == position_end_) Shift();
|
||||
if (position_ == position_end_) {
|
||||
Shift();
|
||||
// And break out at end of file.
|
||||
if (position_ == position_end_) return;
|
||||
}
|
||||
assert(position_ < position_end_);
|
||||
if (!delim[static_cast<unsigned char>(*position_)]) return;
|
||||
}
|
||||
}
|
||||
@ -86,6 +93,7 @@ class FilePiece {
|
||||
template <class T> T ReadNumber();
|
||||
|
||||
StringPiece Consume(const char *to) {
|
||||
assert(to >= position_);
|
||||
StringPiece ret(position_, to - position_);
|
||||
position_ = to;
|
||||
return ret;
|
||||
|
||||
@ -66,7 +66,7 @@ template <class Iterator, class Output, class Less> void AllIntersection(std::ve
|
||||
|
||||
std::sort(sets.begin(), sets.end(), detail::RangeLessBySize<boost::iterator_range<Iterator> >());
|
||||
boost::optional<Value> ret;
|
||||
for (boost::optional<Value> ret; ret = detail::FirstIntersectionSorted(sets, less); sets.front().advance_begin(1)) {
|
||||
for (boost::optional<Value> ret; (ret = detail::FirstIntersectionSorted(sets, less)); sets.front().advance_begin(1)) {
|
||||
out(*ret);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#define UTIL_PROBING_HASH_TABLE__
|
||||
|
||||
#include "util/exception.hh"
|
||||
#include "util/scoped.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
@ -25,6 +26,8 @@ struct IdentityHash {
|
||||
template <class T> T operator()(T arg) const { return arg; }
|
||||
};
|
||||
|
||||
template <class EntryT, class HashT, class EqualT> class AutoProbing;
|
||||
|
||||
/* Non-standard hash table
|
||||
* Buckets must be set at the beginning and must be greater than maximum number
|
||||
* of elements, else it throws ProbingSizeException.
|
||||
@ -33,7 +36,6 @@ struct IdentityHash {
|
||||
* Uses linear probing to find value.
|
||||
* Only insert and lookup operations.
|
||||
*/
|
||||
|
||||
template <class EntryT, class HashT, class EqualT = std::equal_to<typename EntryT::Key> > class ProbingHashTable {
|
||||
public:
|
||||
typedef EntryT Entry;
|
||||
@ -43,7 +45,6 @@ template <class EntryT, class HashT, class EqualT = std::equal_to<typename Entry
|
||||
typedef HashT Hash;
|
||||
typedef EqualT Equal;
|
||||
|
||||
public:
|
||||
static uint64_t Size(uint64_t entries, float multiplier) {
|
||||
uint64_t buckets = std::max(entries + 1, static_cast<uint64_t>(multiplier * static_cast<float>(entries)));
|
||||
return buckets * sizeof(Entry);
|
||||
@ -82,7 +83,7 @@ template <class EntryT, class HashT, class EqualT = std::equal_to<typename Entry
|
||||
#ifdef DEBUG
|
||||
assert(initialized_);
|
||||
#endif
|
||||
for (MutableIterator i(begin_ + (hash_(t.GetKey()) % buckets_));;) {
|
||||
for (MutableIterator i = Ideal(t);;) {
|
||||
Key got(i->GetKey());
|
||||
if (equal_(got, t.GetKey())) { out = i; return true; }
|
||||
if (equal_(got, invalid_)) {
|
||||
@ -224,6 +225,8 @@ template <class EntryT, class HashT, class EqualT = std::equal_to<typename Entry
|
||||
}
|
||||
|
||||
private:
|
||||
friend class AutoProbing<Entry, Hash, Equal>;
|
||||
|
||||
template <class T> MutableIterator Ideal(const T &t) {
|
||||
return begin_ + (hash_(t.GetKey()) % buckets_);
|
||||
}
|
||||
@ -247,6 +250,74 @@ template <class EntryT, class HashT, class EqualT = std::equal_to<typename Entry
|
||||
#endif
|
||||
};
|
||||
|
||||
// Resizable linear probing hash table. This owns the memory.
|
||||
template <class EntryT, class HashT, class EqualT = std::equal_to<typename EntryT::Key> > class AutoProbing {
|
||||
private:
|
||||
typedef ProbingHashTable<EntryT, HashT, EqualT> Backend;
|
||||
public:
|
||||
typedef EntryT Entry;
|
||||
typedef typename Entry::Key Key;
|
||||
typedef const Entry *ConstIterator;
|
||||
typedef Entry *MutableIterator;
|
||||
typedef HashT Hash;
|
||||
typedef EqualT Equal;
|
||||
|
||||
AutoProbing(std::size_t initial_size = 10, const Key &invalid = Key(), const Hash &hash_func = Hash(), const Equal &equal_func = Equal()) :
|
||||
allocated_(Backend::Size(initial_size, 1.5)), mem_(util::MallocOrThrow(allocated_)), backend_(mem_.get(), allocated_, invalid, hash_func, equal_func) {
|
||||
threshold_ = initial_size * 1.2;
|
||||
}
|
||||
|
||||
// Assumes that the key is unique. Multiple insertions won't cause a failure, just inconsistent lookup.
|
||||
template <class T> MutableIterator Insert(const T &t) {
|
||||
DoubleIfNeeded();
|
||||
return backend_.UncheckedInsert(t);
|
||||
}
|
||||
|
||||
template <class T> bool FindOrInsert(const T &t, MutableIterator &out) {
|
||||
DoubleIfNeeded();
|
||||
return backend_.FindOrInsert(t, out);
|
||||
}
|
||||
|
||||
template <class Key> bool UnsafeMutableFind(const Key key, MutableIterator &out) {
|
||||
return backend_.UnsafeMutableFind(key, out);
|
||||
}
|
||||
|
||||
template <class Key> MutableIterator UnsafeMutableMustFind(const Key key) {
|
||||
return backend_.UnsafeMutableMustFind(key);
|
||||
}
|
||||
|
||||
template <class Key> bool Find(const Key key, ConstIterator &out) const {
|
||||
return backend_.Find(key, out);
|
||||
}
|
||||
|
||||
template <class Key> ConstIterator MustFind(const Key key) const {
|
||||
return backend_.MustFind(key);
|
||||
}
|
||||
|
||||
std::size_t Size() const {
|
||||
return backend_.SizeNoSerialization();
|
||||
}
|
||||
|
||||
void Clear() {
|
||||
backend_.Clear();
|
||||
}
|
||||
|
||||
private:
|
||||
void DoubleIfNeeded() {
|
||||
if (Size() < threshold_)
|
||||
return;
|
||||
mem_.call_realloc(backend_.DoubleTo());
|
||||
allocated_ = backend_.DoubleTo();
|
||||
backend_.Double(mem_.get());
|
||||
threshold_ *= 2;
|
||||
}
|
||||
|
||||
std::size_t allocated_;
|
||||
util::scoped_malloc mem_;
|
||||
Backend backend_;
|
||||
std::size_t threshold_;
|
||||
};
|
||||
|
||||
} // namespace util
|
||||
|
||||
#endif // UTIL_PROBING_HASH_TABLE__
|
||||
|
||||
@ -58,6 +58,23 @@ class AnyCharacter {
|
||||
StringPiece chars_;
|
||||
};
|
||||
|
||||
class BoolCharacter {
|
||||
public:
|
||||
BoolCharacter() {}
|
||||
|
||||
explicit BoolCharacter(const bool *delimiter) { delimiter_ = delimiter; }
|
||||
|
||||
StringPiece Find(const StringPiece &in) const {
|
||||
for (const char *i = in.data(); i != in.data() + in.size(); ++i) {
|
||||
if (delimiter_[static_cast<unsigned char>(*i)]) return StringPiece(i, 1);
|
||||
}
|
||||
return StringPiece(in.data() + in.size(), 0);
|
||||
}
|
||||
|
||||
private:
|
||||
const bool *delimiter_;
|
||||
};
|
||||
|
||||
class AnyCharacterLast {
|
||||
public:
|
||||
AnyCharacterLast() {}
|
||||
|
||||
207
util/usage.cc
207
util/usage.cc
@ -10,61 +10,109 @@
|
||||
|
||||
#include <string.h>
|
||||
#include <ctype.h>
|
||||
#if !defined(_WIN32) && !defined(_WIN64)
|
||||
#include <time.h>
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
// This code lifted from physmem.c in gnulib. See the copyright statement
|
||||
// below.
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# include <windows.h>
|
||||
/* MEMORYSTATUSEX is missing from older windows headers, so define
|
||||
a local replacement. */
|
||||
typedef struct
|
||||
{
|
||||
DWORD dwLength;
|
||||
DWORD dwMemoryLoad;
|
||||
DWORDLONG ullTotalPhys;
|
||||
DWORDLONG ullAvailPhys;
|
||||
DWORDLONG ullTotalPageFile;
|
||||
DWORDLONG ullAvailPageFile;
|
||||
DWORDLONG ullTotalVirtual;
|
||||
DWORDLONG ullAvailVirtual;
|
||||
DWORDLONG ullAvailExtendedVirtual;
|
||||
} lMEMORYSTATUSEX;
|
||||
typedef WINBOOL (WINAPI *PFN_MS_EX) (lMEMORYSTATUSEX*);
|
||||
#else
|
||||
#include <sys/resource.h>
|
||||
#include <sys/time.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace util {
|
||||
#if defined(__MACH__) || defined(__FreeBSD__) || defined(__APPLE__)
|
||||
#include <sys/types.h>
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
#if !defined(_WIN32) && !defined(_WIN64)
|
||||
namespace util {
|
||||
namespace {
|
||||
|
||||
// On Mac OS X, clock_gettime is not implemented.
|
||||
// CLOCK_MONOTONIC is not defined either.
|
||||
#ifdef __MACH__
|
||||
#define CLOCK_MONOTONIC 0
|
||||
|
||||
int clock_gettime(int clk_id, struct timespec *tp) {
|
||||
#if defined(__MACH__)
|
||||
typedef struct timeval Wall;
|
||||
Wall GetWall() {
|
||||
struct timeval tv;
|
||||
gettimeofday(&tv, NULL);
|
||||
tp->tv_sec = tv.tv_sec;
|
||||
tp->tv_nsec = tv.tv_usec * 1000;
|
||||
return 0;
|
||||
return tv;
|
||||
}
|
||||
#endif // __MACH__
|
||||
#elif defined(_WIN32) || defined(_WIN64)
|
||||
typedef time_t Wall;
|
||||
Wall GetWall() {
|
||||
return time(NULL);
|
||||
}
|
||||
#else
|
||||
typedef struct timespec Wall;
|
||||
Wall GetWall() {
|
||||
Wall ret;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ret);
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
float FloatSec(const struct timeval &tv) {
|
||||
return static_cast<float>(tv.tv_sec) + (static_cast<float>(tv.tv_usec) / 1000000.0);
|
||||
// These all assume first > second
|
||||
double Subtract(time_t first, time_t second) {
|
||||
return difftime(first, second);
|
||||
}
|
||||
float FloatSec(const struct timespec &tv) {
|
||||
return static_cast<float>(tv.tv_sec) + (static_cast<float>(tv.tv_nsec) / 1000000000.0);
|
||||
double DoubleSec(time_t tv) {
|
||||
return static_cast<double>(tv);
|
||||
}
|
||||
#if !defined(_WIN32) && !defined(_WIN64)
|
||||
double Subtract(const struct timeval &first, const struct timeval &second) {
|
||||
return static_cast<double>(first.tv_sec - second.tv_sec) + static_cast<double>(first.tv_usec - second.tv_usec) / 1000000.0;
|
||||
}
|
||||
double Subtract(const struct timespec &first, const struct timespec &second) {
|
||||
return static_cast<double>(first.tv_sec - second.tv_sec) + static_cast<double>(first.tv_nsec - second.tv_nsec) / 1000000000.0;
|
||||
}
|
||||
double DoubleSec(const struct timeval &tv) {
|
||||
return static_cast<double>(tv.tv_sec) + (static_cast<double>(tv.tv_usec) / 1000000.0);
|
||||
}
|
||||
double DoubleSec(const struct timespec &tv) {
|
||||
return static_cast<double>(tv.tv_sec) + (static_cast<double>(tv.tv_nsec) / 1000000000.0);
|
||||
}
|
||||
#endif
|
||||
|
||||
class RecordStart {
|
||||
public:
|
||||
RecordStart() {
|
||||
started_ = GetWall();
|
||||
}
|
||||
|
||||
const Wall &Started() const {
|
||||
return started_;
|
||||
}
|
||||
|
||||
private:
|
||||
Wall started_;
|
||||
};
|
||||
|
||||
const RecordStart kRecordStart;
|
||||
|
||||
const char *SkipSpaces(const char *at) {
|
||||
for (; *at == ' ' || *at == '\t'; ++at) {}
|
||||
return at;
|
||||
}
|
||||
|
||||
class RecordStart {
|
||||
public:
|
||||
RecordStart() {
|
||||
clock_gettime(CLOCK_MONOTONIC, &started_);
|
||||
}
|
||||
|
||||
const struct timespec &Started() const {
|
||||
return started_;
|
||||
}
|
||||
|
||||
private:
|
||||
struct timespec started_;
|
||||
};
|
||||
|
||||
const RecordStart kRecordStart;
|
||||
} // namespace
|
||||
#endif
|
||||
|
||||
double WallTime() {
|
||||
return Subtract(GetWall(), kRecordStart.Started());
|
||||
}
|
||||
|
||||
void PrintUsage(std::ostream &out) {
|
||||
#if !defined(_WIN32) && !defined(_WIN64)
|
||||
@ -88,27 +136,84 @@ void PrintUsage(std::ostream &out) {
|
||||
return;
|
||||
}
|
||||
out << "RSSMax:" << usage.ru_maxrss << " kB" << '\t';
|
||||
out << "user:" << FloatSec(usage.ru_utime) << "\tsys:" << FloatSec(usage.ru_stime) << '\t';
|
||||
out << "CPU:" << (FloatSec(usage.ru_utime) + FloatSec(usage.ru_stime));
|
||||
|
||||
struct timespec current;
|
||||
clock_gettime(CLOCK_MONOTONIC, ¤t);
|
||||
out << "\treal:" << (FloatSec(current) - FloatSec(kRecordStart.Started())) << '\n';
|
||||
out << "user:" << DoubleSec(usage.ru_utime) << "\tsys:" << DoubleSec(usage.ru_stime) << '\t';
|
||||
out << "CPU:" << (DoubleSec(usage.ru_utime) + DoubleSec(usage.ru_stime));
|
||||
out << '\t';
|
||||
#endif
|
||||
|
||||
out << "real:" << WallTime() << '\n';
|
||||
}
|
||||
|
||||
/* Adapted from physmem.c in gnulib 831b84c59ef413c57a36b67344467d66a8a2ba70 */
|
||||
/* Calculate the size of physical memory.
|
||||
|
||||
Copyright (C) 2000-2001, 2003, 2005-2006, 2009-2013 Free Software
|
||||
Foundation, Inc.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation; either version 2.1 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Written by Paul Eggert. */
|
||||
uint64_t GuessPhysicalMemory() {
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
return 0;
|
||||
#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
|
||||
long pages = sysconf(_SC_PHYS_PAGES);
|
||||
if (pages == -1) return 0;
|
||||
long page_size = sysconf(_SC_PAGESIZE);
|
||||
if (page_size == -1) return 0;
|
||||
return static_cast<uint64_t>(pages) * static_cast<uint64_t>(page_size);
|
||||
#else
|
||||
return 0;
|
||||
#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
|
||||
{
|
||||
long pages = sysconf(_SC_PHYS_PAGES);
|
||||
long page_size = sysconf(_SC_PAGESIZE);
|
||||
if (pages != -1 && page_size != -1)
|
||||
return static_cast<uint64_t>(pages) * static_cast<uint64_t>(page_size);
|
||||
}
|
||||
#endif
|
||||
#ifdef HW_PHYSMEM
|
||||
{ /* This works on *bsd and darwin. */
|
||||
unsigned int physmem;
|
||||
size_t len = sizeof physmem;
|
||||
static int mib[2] = { CTL_HW, HW_PHYSMEM };
|
||||
|
||||
if (sysctl (mib, sizeof(mib) / sizeof(mib[0]), &physmem, &len, NULL, 0) == 0
|
||||
&& len == sizeof (physmem))
|
||||
return static_cast<uint64_t>(physmem);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
{ /* this works on windows */
|
||||
PFN_MS_EX pfnex;
|
||||
HMODULE h = GetModuleHandle ("kernel32.dll");
|
||||
|
||||
if (!h)
|
||||
return 0;
|
||||
|
||||
/* Use GlobalMemoryStatusEx if available. */
|
||||
if ((pfnex = (PFN_MS_EX) GetProcAddress (h, "GlobalMemoryStatusEx")))
|
||||
{
|
||||
lMEMORYSTATUSEX lms_ex;
|
||||
lms_ex.dwLength = sizeof lms_ex;
|
||||
if (!pfnex (&lms_ex))
|
||||
return 0;
|
||||
return lms_ex.ullTotalPhys;
|
||||
}
|
||||
|
||||
/* Fall back to GlobalMemoryStatus which is always available.
|
||||
but returns wrong results for physical memory > 4GB. */
|
||||
else
|
||||
{
|
||||
MEMORYSTATUS ms;
|
||||
GlobalMemoryStatus (&ms);
|
||||
return ms.dwTotalPhys;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
@ -7,6 +7,9 @@
|
||||
#include <stdint.h>
|
||||
|
||||
namespace util {
|
||||
// Time in seconds since process started. Zero on unsupported platforms.
|
||||
double WallTime();
|
||||
|
||||
void PrintUsage(std::ostream &to);
|
||||
|
||||
// Determine how much physical memory there is. Return 0 on failure.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user