2014-06-02 21:28:02 +04:00
|
|
|
#ifndef UTIL_TOKENIZE_PIECE_H
|
|
|
|
#define UTIL_TOKENIZE_PIECE_H
|
2011-09-22 00:15:03 +04:00
|
|
|
|
2011-11-17 23:12:19 +04:00
|
|
|
#include "util/exception.hh"
|
2011-09-22 00:15:03 +04:00
|
|
|
#include "util/string_piece.hh"
|
|
|
|
|
|
|
|
#include <boost/iterator/iterator_facade.hpp>
|
|
|
|
|
2011-10-14 15:51:15 +04:00
|
|
|
#include <algorithm>
|
2015-03-28 15:37:48 +03:00
|
|
|
#include <cstring>
|
2011-10-14 15:51:15 +04:00
|
|
|
|
2011-09-22 00:15:03 +04:00
|
|
|
namespace util {
|
|
|
|
|
2011-11-17 23:12:19 +04:00
|
|
|
// Thrown on dereference when out of tokens to parse
|
|
|
|
class OutOfTokens : public Exception {
|
2011-09-22 00:15:03 +04:00
|
|
|
public:
|
2011-11-17 23:12:19 +04:00
|
|
|
OutOfTokens() throw() {}
|
|
|
|
~OutOfTokens() throw() {}
|
|
|
|
};
|
2011-09-22 00:15:03 +04:00
|
|
|
|
2011-11-17 23:12:19 +04:00
|
|
|
class SingleCharacter {
|
|
|
|
public:
|
2012-10-30 23:33:22 +04:00
|
|
|
SingleCharacter() {}
|
2011-11-17 23:12:19 +04:00
|
|
|
explicit SingleCharacter(char delim) : delim_(delim) {}
|
2011-09-22 00:15:03 +04:00
|
|
|
|
2011-11-17 23:12:19 +04:00
|
|
|
StringPiece Find(const StringPiece &in) const {
|
|
|
|
return StringPiece(std::find(in.data(), in.data() + in.size(), delim_), 1);
|
2011-09-22 00:15:03 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
2011-11-17 23:12:19 +04:00
|
|
|
char delim_;
|
2011-09-22 00:15:03 +04:00
|
|
|
};
|
|
|
|
|
2011-10-14 20:40:30 +04:00
|
|
|
class MultiCharacter {
|
2011-10-14 15:51:15 +04:00
|
|
|
public:
|
2012-10-30 23:33:22 +04:00
|
|
|
MultiCharacter() {}
|
|
|
|
|
2011-10-14 20:40:30 +04:00
|
|
|
explicit MultiCharacter(const StringPiece &delimiter) : delimiter_(delimiter) {}
|
2011-10-14 15:51:15 +04:00
|
|
|
|
2011-10-14 20:40:30 +04:00
|
|
|
StringPiece Find(const StringPiece &in) const {
|
|
|
|
return StringPiece(std::search(in.data(), in.data() + in.size(), delimiter_.data(), delimiter_.data() + delimiter_.size()), delimiter_.size());
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
StringPiece delimiter_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class AnyCharacter {
|
|
|
|
public:
|
2012-10-30 23:33:22 +04:00
|
|
|
AnyCharacter() {}
|
2011-10-14 20:40:30 +04:00
|
|
|
explicit AnyCharacter(const StringPiece &chars) : chars_(chars) {}
|
|
|
|
|
|
|
|
StringPiece Find(const StringPiece &in) const {
|
|
|
|
return StringPiece(std::find_first_of(in.data(), in.data() + in.size(), chars_.data(), chars_.data() + chars_.size()), 1);
|
|
|
|
}
|
|
|
|
|
2012-03-27 00:17:10 +04:00
|
|
|
private:
|
|
|
|
StringPiece chars_;
|
|
|
|
};
|
|
|
|
|
2014-01-02 01:19:06 +04:00
|
|
|
class BoolCharacter {
|
|
|
|
public:
|
|
|
|
BoolCharacter() {}
|
|
|
|
|
|
|
|
explicit BoolCharacter(const bool *delimiter) { delimiter_ = delimiter; }
|
|
|
|
|
|
|
|
StringPiece Find(const StringPiece &in) const {
|
|
|
|
for (const char *i = in.data(); i != in.data() + in.size(); ++i) {
|
|
|
|
if (delimiter_[static_cast<unsigned char>(*i)]) return StringPiece(i, 1);
|
|
|
|
}
|
|
|
|
return StringPiece(in.data() + in.size(), 0);
|
|
|
|
}
|
|
|
|
|
2014-06-02 21:28:02 +04:00
|
|
|
template <unsigned Length> static void Build(const char (&characters)[Length], bool (&out)[256]) {
|
|
|
|
memset(out, 0, sizeof(out));
|
|
|
|
for (const char *i = characters; i != characters + Length; ++i) {
|
|
|
|
out[static_cast<unsigned char>(*i)] = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-01-02 01:19:06 +04:00
|
|
|
private:
|
|
|
|
const bool *delimiter_;
|
|
|
|
};
|
|
|
|
|
2012-03-27 00:17:10 +04:00
|
|
|
class AnyCharacterLast {
|
|
|
|
public:
|
2012-10-30 23:33:22 +04:00
|
|
|
AnyCharacterLast() {}
|
|
|
|
|
2012-03-27 00:17:10 +04:00
|
|
|
explicit AnyCharacterLast(const StringPiece &chars) : chars_(chars) {}
|
|
|
|
|
|
|
|
StringPiece Find(const StringPiece &in) const {
|
|
|
|
return StringPiece(std::find_end(in.data(), in.data() + in.size(), chars_.data(), chars_.data() + chars_.size()), 1);
|
|
|
|
}
|
|
|
|
|
2011-10-14 20:40:30 +04:00
|
|
|
private:
|
|
|
|
StringPiece chars_;
|
|
|
|
};
|
|
|
|
|
|
|
|
template <class Find, bool SkipEmpty = false> class TokenIter : public boost::iterator_facade<TokenIter<Find, SkipEmpty>, const StringPiece, boost::forward_traversal_tag> {
|
|
|
|
public:
|
|
|
|
TokenIter() {}
|
|
|
|
|
2011-11-17 23:12:19 +04:00
|
|
|
template <class Construct> TokenIter(const StringPiece &str, const Construct &construct) : after_(str), finder_(construct) {
|
2011-10-14 15:51:15 +04:00
|
|
|
increment();
|
|
|
|
}
|
|
|
|
|
|
|
|
bool operator!() const {
|
|
|
|
return current_.data() == 0;
|
|
|
|
}
|
|
|
|
operator bool() const {
|
|
|
|
return current_.data() != 0;
|
|
|
|
}
|
|
|
|
|
2012-10-30 23:33:22 +04:00
|
|
|
static TokenIter<Find, SkipEmpty> end() {
|
|
|
|
return TokenIter<Find, SkipEmpty>();
|
2011-10-14 15:51:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
friend class boost::iterator_core_access;
|
|
|
|
|
|
|
|
void increment() {
|
2011-10-14 20:40:30 +04:00
|
|
|
do {
|
|
|
|
StringPiece found(finder_.Find(after_));
|
|
|
|
current_ = StringPiece(after_.data(), found.data() - after_.data());
|
|
|
|
if (found.data() == after_.data() + after_.size()) {
|
|
|
|
after_ = StringPiece(NULL, 0);
|
|
|
|
} else {
|
|
|
|
after_ = StringPiece(found.data() + found.size(), after_.data() - found.data() + after_.size() - found.size());
|
|
|
|
}
|
2015-04-30 08:05:11 +03:00
|
|
|
} while (SkipEmpty && current_.data() && current_.empty()); // Compiler should optimize this away if SkipEmpty is false.
|
2011-10-14 15:51:15 +04:00
|
|
|
}
|
|
|
|
|
2012-10-30 23:33:22 +04:00
|
|
|
bool equal(const TokenIter<Find, SkipEmpty> &other) const {
|
|
|
|
return current_.data() == other.current_.data();
|
2011-10-14 15:51:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
const StringPiece &dereference() const {
|
2011-11-17 23:12:19 +04:00
|
|
|
UTIL_THROW_IF(!current_.data(), OutOfTokens, "Ran out of tokens");
|
2011-10-14 15:51:15 +04:00
|
|
|
return current_;
|
|
|
|
}
|
|
|
|
|
|
|
|
StringPiece current_;
|
|
|
|
StringPiece after_;
|
|
|
|
|
2011-10-14 20:40:30 +04:00
|
|
|
Find finder_;
|
2011-10-14 15:51:15 +04:00
|
|
|
};
|
|
|
|
|
2011-09-22 00:15:03 +04:00
|
|
|
} // namespace util
|
|
|
|
|
2014-06-02 21:28:02 +04:00
|
|
|
#endif // UTIL_TOKENIZE_PIECE_H
|