mirror of
https://github.com/tstack/lnav.git
synced 2024-09-11 13:05:51 +03:00
[log format] some more log format work
This commit is contained in:
parent
2cd356b4c5
commit
2d33cb7f7e
@ -1,6 +1,31 @@
|
||||
{
|
||||
"syslog_log" : {
|
||||
"regex" : [
|
||||
"^(?P<timestamp>\\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2})(?: (?P<log_hostname>[a-zA-Z0-9][^ ]+))?(?:(?: (?P<log_procname>[^ \\[:]+)(?:\\[(?P<log_pid>\\d+)])?:(?P<body>.*))|(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
|
||||
],
|
||||
"level-field" : "body",
|
||||
"level" : {
|
||||
"error" : "(?:failed|failure|error)",
|
||||
"warning" : "(?:warn|not responding|init: cannot execute)"
|
||||
},
|
||||
"value" : {
|
||||
"log_hostname" : {
|
||||
"kind" : "string",
|
||||
"collate" : "ipaddress",
|
||||
"identifier" : true
|
||||
},
|
||||
"log_procname" : {
|
||||
"kind" : "string",
|
||||
"identifier" : true
|
||||
},
|
||||
"log_pid" : {
|
||||
"kind" : "string",
|
||||
"identifier" : true
|
||||
}
|
||||
}
|
||||
},
|
||||
"access_log" : {
|
||||
"regex" : "^(?<c_ip>[\\w\\.\\-]+) [\\w\\.\\-]+ (?<cs_username>[\\w\\.\\-]+) \\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?.*",
|
||||
"regex" : ["^(?<c_ip>[\\w\\.\\-]+) [\\w\\.\\-]+ (?<cs_username>[\\w\\.\\-]+) \\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?.*"],
|
||||
"level-field": "sc_status",
|
||||
"level" : {
|
||||
"error" : "^[^123]"
|
||||
@ -47,7 +72,7 @@
|
||||
}
|
||||
},
|
||||
"vmw_log" : {
|
||||
"regex" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z) \\[(?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[\\w\\-]+))?\\](?<body>.*)$",
|
||||
"regex" : ["^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z) \\[(?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[\\w\\-]+))?\\](?<body>.*)$"],
|
||||
"level-field": "level",
|
||||
"level" : {
|
||||
"error" : "error",
|
||||
@ -74,7 +99,7 @@
|
||||
}
|
||||
},
|
||||
"snaplogic_log" : {
|
||||
"regex" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?<level>\\w+) (?<logger>[^ ]+) (?<facility>[^ ]+) (?<msgid>[^ ]+) (?<pipe_rid>[^ \\.]+)(?:\\.(?<comp_rid>[^ ]+))? (?<resource_name>[^ ]+) (?<invoker>[^ ]+)(?<body>.*)",
|
||||
"regex" : ["^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?<level>\\w+) (?<logger>[^ ]+) (?<facility>[^ ]+) (?<msgid>[^ ]+) (?<pipe_rid>[^ \\.]+)(?:\\.(?<comp_rid>[^ ]+))? (?<resource_name>[^ ]+) (?<invoker>[^ ]+)(?<body>.*)"],
|
||||
"level-field" : "level",
|
||||
"level" : {
|
||||
"error" : "ERROR",
|
||||
|
@ -115,6 +115,7 @@ line_buffer::line_buffer()
|
||||
lb_bz_file(false),
|
||||
lb_file_size((size_t)-1),
|
||||
lb_file_offset(0),
|
||||
lb_file_time(0),
|
||||
lb_buffer_size(0),
|
||||
lb_buffer_max(DEFAULT_LINE_BUFFER_SIZE),
|
||||
lb_seekable(false),
|
||||
@ -161,7 +162,7 @@ throw (error)
|
||||
this->lb_seekable = false;
|
||||
}
|
||||
else {
|
||||
char gz_id[2];
|
||||
char gz_id[2 + 1 + 1 + 4];
|
||||
|
||||
if (pread(fd, gz_id, sizeof(gz_id), 0) == sizeof(gz_id)) {
|
||||
if (gz_id[0] == '\037' && gz_id[1] == '\213') {
|
||||
@ -174,6 +175,9 @@ throw (error)
|
||||
throw error(errno);
|
||||
}
|
||||
}
|
||||
this->lb_file_time = *((time_t *)&gz_id[4]);
|
||||
if (this->lb_file_time < 0)
|
||||
this->lb_file_time = 0;
|
||||
this->lb_gz_offset = lseek(this->lb_fd, 0, SEEK_CUR);
|
||||
}
|
||||
#ifdef HAVE_BZLIB_H
|
||||
|
@ -74,6 +74,8 @@ public:
|
||||
/** @return The file descriptor that data should be pulled from. */
|
||||
int get_fd() const { return this->lb_fd; };
|
||||
|
||||
time_t get_file_time() const { return this->lb_file_time; };
|
||||
|
||||
/**
|
||||
* @return The size of the file or the amount of data pulled from a pipe.
|
||||
*/
|
||||
@ -217,6 +219,7 @@ private:
|
||||
* Data cached in the buffer comes from this
|
||||
* offset in the file.
|
||||
*/
|
||||
time_t lb_file_time;
|
||||
size_t lb_buffer_size; /*< The amount of cached data in the buffer. */
|
||||
size_t lb_buffer_max; /*< The size of the buffer memory. */
|
||||
bool lb_seekable; /*< Flag set for seekable file descriptors. */
|
||||
|
11
src/lnav.cc
11
src/lnav.cc
@ -3189,6 +3189,7 @@ int sql_progress(const struct log_cursor &lc)
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
std::vector<std::string> loader_errors;
|
||||
int lpc, c, retval = EXIT_SUCCESS;
|
||||
|
||||
auto_ptr<piper_proc> stdin_reader;
|
||||
@ -3196,7 +3197,15 @@ int main(int argc, char *argv[])
|
||||
|
||||
setlocale(LC_NUMERIC, "");
|
||||
|
||||
load_formats();
|
||||
load_formats(loader_errors);
|
||||
if (!loader_errors.empty()) {
|
||||
for (std::vector<std::string>::iterator iter = loader_errors.begin();
|
||||
iter != loader_errors.end();
|
||||
++iter) {
|
||||
fprintf(stderr, "%s\n", iter->c_str());
|
||||
}
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
/* If we statically linked against an ncurses library that had a non-
|
||||
* standard path to the terminfo database, we need to set this variable
|
||||
|
@ -192,6 +192,28 @@ static bool next_format(const char *fmt[], int &index, int &locked_index)
|
||||
return retval;
|
||||
}
|
||||
|
||||
static bool next_format(const std::vector<external_log_format::pattern> &patterns,
|
||||
int &index,
|
||||
int &locked_index)
|
||||
{
|
||||
bool retval = true;
|
||||
|
||||
if (locked_index == -1) {
|
||||
index += 1;
|
||||
if (index >= (int)patterns.size()) {
|
||||
retval = false;
|
||||
}
|
||||
}
|
||||
else if (index == locked_index) {
|
||||
retval = false;
|
||||
}
|
||||
else {
|
||||
index = locked_index;
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
static const char *std_time_fmt[] = {
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
@ -226,17 +248,15 @@ const char *log_format::time_scanf(const char *time_dest,
|
||||
while (next_format(time_fmt,
|
||||
curr_time_fmt,
|
||||
this->lf_time_fmt_lock)) {
|
||||
memset(tm_out, 0, sizeof(struct tm));
|
||||
localtime_r(&this->lf_base_time, tm_out);
|
||||
if ((retval = strptime(time_dest,
|
||||
time_fmt[curr_time_fmt],
|
||||
tm_out)) != NULL) {
|
||||
time_fmt[curr_time_fmt],
|
||||
tm_out)) != NULL) {
|
||||
if (tm_out->tm_year < 70) {
|
||||
/* XXX We should pull the time from the file mtime (?) */
|
||||
tm_out->tm_year = 80;
|
||||
}
|
||||
time_out = tm2sec(tm_out);
|
||||
|
||||
// this->lf_fmt_lock = curr_fmt;
|
||||
this->lf_time_fmt_lock = curr_time_fmt;
|
||||
this->lf_time_fmt_len = retval - time_dest;
|
||||
|
||||
@ -303,22 +323,27 @@ bool external_log_format::scan(std::vector<logline> &dst,
|
||||
pcre_input pi(prefix, 0, len);
|
||||
pcre_context_static<30> pc;
|
||||
bool retval = false;
|
||||
int curr_fmt = -1;
|
||||
|
||||
while (next_format(this->elf_patterns, curr_fmt, this->lf_fmt_lock)) {
|
||||
if (!this->elf_patterns[curr_fmt].p_pcre->match(pc, pi)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (this->elf_pcre->match(pc, pi)) {
|
||||
pcre_context::capture_t *ts = pc["timestamp"];
|
||||
pcre_context::capture_t *level_cap = pc[this->elf_level_field];
|
||||
const char *ts_str = pi.get_substr_start(ts);
|
||||
const char *last;
|
||||
time_t line_time;
|
||||
struct tm log_time;
|
||||
time_t log_time;
|
||||
struct tm log_time_tm;
|
||||
uint16_t millis = 0;
|
||||
logline::level_t level = logline::LEVEL_INFO;
|
||||
|
||||
if ((last = this->time_scanf(ts_str,
|
||||
NULL,
|
||||
&log_time,
|
||||
line_time)) == NULL) {
|
||||
return false;
|
||||
&log_time_tm,
|
||||
log_time)) == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Try to pull out the milliseconds value. */
|
||||
@ -347,12 +372,28 @@ bool external_log_format::scan(std::vector<logline> &dst,
|
||||
}
|
||||
}
|
||||
|
||||
if (!dst.empty() &&
|
||||
((dst.back().get_time() - log_time) > (24 * 60 * 60))) {
|
||||
vector<logline>::iterator iter;
|
||||
|
||||
for (iter = dst.begin(); iter != dst.end(); iter++) {
|
||||
time_t ot = iter->get_time();
|
||||
struct tm *otm;
|
||||
|
||||
otm = gmtime(&ot);
|
||||
otm->tm_year -= 1;
|
||||
iter->set_time(tm2sec(otm));
|
||||
}
|
||||
}
|
||||
|
||||
dst.push_back(logline(offset,
|
||||
line_time,
|
||||
log_time,
|
||||
millis,
|
||||
level));
|
||||
|
||||
this->lf_fmt_lock = curr_fmt;
|
||||
retval = true;
|
||||
break;
|
||||
}
|
||||
|
||||
return retval;
|
||||
@ -367,7 +408,7 @@ void external_log_format::annotate(const std::string &line,
|
||||
struct line_range lr;
|
||||
pcre_context::capture_t *cap;
|
||||
|
||||
if (!this->elf_pcre->match(pc, pi)) {
|
||||
if (!this->elf_patterns[this->lf_fmt_lock].p_pcre->match(pc, pi)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -389,8 +430,8 @@ void external_log_format::annotate(const std::string &line,
|
||||
|
||||
view_colors &vc = view_colors::singleton();
|
||||
|
||||
for (size_t lpc = 0; lpc < this->elf_value_by_index.size(); lpc++) {
|
||||
const value_def &vd = this->elf_value_by_index[lpc];
|
||||
for (size_t lpc = 0; lpc < this->elf_patterns[this->lf_fmt_lock].p_value_by_index.size(); lpc++) {
|
||||
const value_def &vd = this->elf_patterns[this->lf_fmt_lock].p_value_by_index[lpc];
|
||||
|
||||
values.push_back(logline_value(vd.vd_name,
|
||||
vd.vd_kind,
|
||||
@ -398,7 +439,6 @@ void external_log_format::annotate(const std::string &line,
|
||||
vd.vd_identifier));
|
||||
|
||||
if (pc[vd.vd_index]->c_begin != -1 && vd.vd_identifier) {
|
||||
fprintf(stderr, "ident %s \n", vd.vd_name.c_str());
|
||||
lr.lr_start = pc[vd.vd_index]->c_begin;
|
||||
lr.lr_end = pc[vd.vd_index]->c_end;
|
||||
sa[lr].insert(make_string_attr("style", vc.attrs_for_ident(pi.get_substr_start(pc[vd.vd_index]), lr.length())));
|
||||
@ -406,29 +446,39 @@ void external_log_format::annotate(const std::string &line,
|
||||
}
|
||||
}
|
||||
|
||||
void external_log_format::build(void)
|
||||
void external_log_format::build(std::vector<std::string> &errors)
|
||||
{
|
||||
this->elf_pcre = new pcrepp(this->elf_regex.c_str());
|
||||
for (std::vector<pattern>::iterator iter = this->elf_patterns.begin();
|
||||
iter != this->elf_patterns.end();
|
||||
++iter) {
|
||||
iter->p_pcre = new pcrepp(iter->p_string.c_str());
|
||||
for (pcre_named_capture::iterator name_iter = iter->p_pcre->named_begin();
|
||||
name_iter != iter->p_pcre->named_end();
|
||||
++name_iter) {
|
||||
std::map<std::string, value_def>::iterator value_iter;
|
||||
|
||||
value_iter = this->elf_value_defs.find(std::string(name_iter->pnc_name));
|
||||
if (value_iter != this->elf_value_defs.end()) {
|
||||
value_iter->second.vd_index = name_iter->index();
|
||||
iter->p_value_by_index.push_back(value_iter->second);
|
||||
}
|
||||
}
|
||||
|
||||
stable_sort(iter->p_value_by_index.begin(),
|
||||
iter->p_value_by_index.end());
|
||||
}
|
||||
|
||||
if (this->elf_patterns.empty()) {
|
||||
errors.push_back("error:" +
|
||||
this->elf_name +
|
||||
": 'regex' field is empty");
|
||||
}
|
||||
|
||||
for (std::map<logline::level_t, level_pattern>::iterator iter = this->elf_level_patterns.begin();
|
||||
iter != this->elf_level_patterns.end();
|
||||
++iter) {
|
||||
iter->second.lp_pcre = new pcrepp(iter->second.lp_regex.c_str());
|
||||
}
|
||||
|
||||
for (pcre_named_capture::iterator iter = this->elf_pcre->named_begin();
|
||||
iter != this->elf_pcre->named_end();
|
||||
++iter) {
|
||||
std::map<std::string, value_def>::iterator value_iter;
|
||||
|
||||
value_iter = this->elf_value_defs.find(std::string(iter->pnc_name));
|
||||
if (value_iter != this->elf_value_defs.end()) {
|
||||
value_iter->second.vd_index = iter->index();
|
||||
this->elf_value_by_index.push_back(value_iter->second);
|
||||
}
|
||||
}
|
||||
|
||||
stable_sort(this->elf_value_by_index.begin(),
|
||||
this->elf_value_by_index.end());
|
||||
}
|
||||
|
||||
class external_log_table : public log_vtab_impl {
|
||||
@ -440,8 +490,8 @@ public:
|
||||
void get_columns(vector<vtab_column> &cols) {
|
||||
std::vector<external_log_format::value_def>::const_iterator iter;
|
||||
|
||||
for (iter = this->elt_format.elf_value_by_index.begin();
|
||||
iter != this->elt_format.elf_value_by_index.end();
|
||||
for (iter = this->elt_format.elf_patterns[0].p_value_by_index.begin();
|
||||
iter != this->elt_format.elf_patterns[0].p_value_by_index.end();
|
||||
++iter) {
|
||||
int type;
|
||||
|
||||
|
@ -352,6 +352,7 @@ public:
|
||||
|
||||
virtual void clear(void)
|
||||
{
|
||||
this->lf_base_time = 0;
|
||||
this->lf_fmt_lock = -1;
|
||||
this->lf_time_fmt_lock = -1;
|
||||
this->lf_time_fmt_len = -1;
|
||||
@ -399,6 +400,7 @@ public:
|
||||
return NULL;
|
||||
};
|
||||
|
||||
time_t lf_base_time;
|
||||
protected:
|
||||
static std::vector<log_format *> lf_root_formats;
|
||||
|
||||
@ -448,6 +450,14 @@ public:
|
||||
};
|
||||
};
|
||||
|
||||
struct pattern {
|
||||
pattern(const std::string &str) : p_string(str) { };
|
||||
|
||||
std::string p_string;
|
||||
pcrepp *p_pcre;
|
||||
std::vector<value_def> p_value_by_index;
|
||||
};
|
||||
|
||||
struct level_pattern {
|
||||
std::string lp_regex;
|
||||
pcrepp *lp_pcre;
|
||||
@ -468,7 +478,7 @@ public:
|
||||
string_attrs_t &sa,
|
||||
std::vector<logline_value> &values) const;
|
||||
|
||||
void build(void);
|
||||
void build(std::vector<std::string> &errors);
|
||||
|
||||
std::auto_ptr<log_format> specialized() {
|
||||
std::auto_ptr<log_format> retval((log_format *)
|
||||
@ -479,10 +489,8 @@ public:
|
||||
|
||||
log_vtab_impl *get_vtab_impl(void) const;
|
||||
|
||||
std::string elf_regex;
|
||||
pcrepp *elf_pcre;
|
||||
std::vector<pattern> elf_patterns;
|
||||
std::vector<sample> elf_samples;
|
||||
std::vector<value_def> elf_value_by_index;
|
||||
std::map<std::string, value_def> elf_value_defs;
|
||||
std::string elf_level_field;
|
||||
std::map<logline::level_t, level_pattern> elf_level_patterns;
|
||||
|
@ -75,215 +75,6 @@ static string scrub_rdns(const string &str)
|
||||
return retval;
|
||||
}
|
||||
|
||||
class syslog_log_format : public log_format {
|
||||
static const int TIMESTAMP_LENGTH = 16;
|
||||
|
||||
static pcrepp &repeated_pattern(void)
|
||||
{
|
||||
static pcrepp REPEATED_PATTERN("last message repeated \\d+ times");
|
||||
|
||||
return REPEATED_PATTERN;
|
||||
}
|
||||
|
||||
static pcrepp &scrub_pattern(void)
|
||||
{
|
||||
static pcrepp SCRUB_PATTERN(
|
||||
"(\\w+\\s[\\s\\d]\\d \\d+:\\d+:\\d+) [\\.\\-\\w]+( .*)");
|
||||
|
||||
return SCRUB_PATTERN;
|
||||
}
|
||||
|
||||
static pcrepp &error_pattern(void)
|
||||
{
|
||||
static pcrepp ERROR_PATTERN("(?:failed|failure|error)", PCRE_CASELESS);
|
||||
|
||||
return ERROR_PATTERN;
|
||||
}
|
||||
|
||||
static pcrepp &warning_pattern(void)
|
||||
{
|
||||
static pcrepp WARNING_PATTERN(
|
||||
"(?:warn|not responding|init: cannot execute)", PCRE_CASELESS);
|
||||
|
||||
return WARNING_PATTERN;
|
||||
}
|
||||
|
||||
string get_name() const { return "syslog_log"; };
|
||||
|
||||
void scrub(string &line)
|
||||
{
|
||||
pcre_context_static<30> context;
|
||||
pcre_input pi(line);
|
||||
string new_line = "";
|
||||
|
||||
if (scrub_pattern().match(context, pi)) {
|
||||
pcre_context::capture_t *cap;
|
||||
|
||||
for (cap = context.begin(); cap != context.end(); cap++) {
|
||||
new_line += pi.get_substr(cap);
|
||||
}
|
||||
|
||||
line = new_line;
|
||||
}
|
||||
};
|
||||
|
||||
void annotate(const string &line,
|
||||
string_attrs_t &sa,
|
||||
std::vector<logline_value> &values) const
|
||||
{
|
||||
bool found_hostname = false, found_procname = false, found_prefix =
|
||||
false;
|
||||
struct line_range lr, hostname_range = { 0, 0 };
|
||||
struct line_range procname_range = { 0, 0 };
|
||||
int log_pid = -1;
|
||||
|
||||
lr.lr_start = 0;
|
||||
lr.lr_end = TIMESTAMP_LENGTH;
|
||||
sa[lr].insert(make_string_attr("timestamp", 0));
|
||||
|
||||
for (size_t lpc = TIMESTAMP_LENGTH; lpc < line.size(); lpc++) {
|
||||
if (line[lpc] == ' ' && !found_hostname) {
|
||||
hostname_range.lr_start = TIMESTAMP_LENGTH;
|
||||
hostname_range.lr_end = lpc;
|
||||
sa[hostname_range].insert(make_string_attr("log_hostname", 0));
|
||||
|
||||
found_hostname = true;
|
||||
}
|
||||
|
||||
if (line[lpc] == ' ' && found_hostname && !found_procname) {
|
||||
bool done = false;
|
||||
|
||||
procname_range.lr_start = procname_range.lr_end = lpc + 1;
|
||||
while (!done) {
|
||||
switch (line[lpc]) {
|
||||
case '\0':
|
||||
done = true;
|
||||
break;
|
||||
|
||||
case ':':
|
||||
case '[':
|
||||
procname_range.lr_end = lpc;
|
||||
done = true;
|
||||
break;
|
||||
|
||||
default:
|
||||
lpc += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sa[procname_range].insert(make_string_attr("log_procname", 0));
|
||||
|
||||
found_procname = true;
|
||||
}
|
||||
|
||||
if (line[lpc] == '[' && log_pid == -1) {
|
||||
const char *line_c_str = line.c_str();
|
||||
|
||||
sscanf(&line_c_str[lpc + 1], "%d", &log_pid);
|
||||
}
|
||||
|
||||
if (line[lpc] == ':') {
|
||||
lr.lr_start = 0;
|
||||
lr.lr_end = lpc + 1;
|
||||
sa[lr].insert(make_string_attr("prefix", 0));
|
||||
|
||||
lr.lr_start = lpc + 1;
|
||||
lr.lr_end = line.length();
|
||||
sa[lr].insert(make_string_attr("body", 0));
|
||||
|
||||
found_prefix = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found_prefix) {
|
||||
lr.lr_start = 0;
|
||||
lr.lr_end = line.length();
|
||||
sa[lr].insert(make_string_attr("prefix", 0));
|
||||
|
||||
lr.lr_start = line.length();
|
||||
lr.lr_end = line.length();
|
||||
sa[lr].insert(make_string_attr("body", 0));
|
||||
|
||||
hostname_range.lr_start = 0;
|
||||
hostname_range.lr_end = 0;
|
||||
}
|
||||
|
||||
values.push_back(logline_value("log_hostname",
|
||||
line.substr(hostname_range.lr_start,
|
||||
hostname_range.length())));
|
||||
values.push_back(logline_value("log_procname",
|
||||
line.substr(procname_range.lr_start,
|
||||
procname_range.length())));
|
||||
values.push_back(logline_value("log_pid", (int64_t)log_pid));
|
||||
};
|
||||
|
||||
bool scan(vector<logline> &dst,
|
||||
off_t offset,
|
||||
char *prefix,
|
||||
int len)
|
||||
{
|
||||
bool retval = false;
|
||||
struct tm log_time;
|
||||
short millis = 0;
|
||||
time_t now;
|
||||
char * rest;
|
||||
|
||||
now = time(NULL);
|
||||
localtime_r(&now, &log_time);
|
||||
|
||||
log_time.tm_isdst = 0;
|
||||
|
||||
if ((rest = strptime(prefix,
|
||||
"%b %d %H:%M:%S",
|
||||
&log_time)) != NULL) {
|
||||
pcre_context_static<20> context;
|
||||
pcre_input pi(prefix, 0, len);
|
||||
logline::level_t ll = logline::LEVEL_UNKNOWN;
|
||||
time_t log_gmt;
|
||||
|
||||
if (repeated_pattern().match(context, pi)) {
|
||||
ll = logline::LEVEL_CONTINUED;
|
||||
}
|
||||
else if (error_pattern().match(context, pi)) {
|
||||
ll = logline::LEVEL_ERROR;
|
||||
}
|
||||
else if (warning_pattern().match(context, pi)) {
|
||||
ll = logline::LEVEL_WARNING;
|
||||
}
|
||||
log_gmt = tm2sec(&log_time);
|
||||
if (!dst.empty() &&
|
||||
((dst.back().get_time() - log_gmt) > (24 * 60 * 60))) {
|
||||
vector<logline>::iterator iter;
|
||||
|
||||
for (iter = dst.begin(); iter != dst.end(); iter++) {
|
||||
time_t ot = iter->get_time();
|
||||
struct tm *otm;
|
||||
|
||||
otm = gmtime(&ot);
|
||||
otm->tm_year -= 1;
|
||||
iter->set_time(tm2sec(otm));
|
||||
}
|
||||
}
|
||||
dst.push_back(logline(offset, log_gmt, millis, ll));
|
||||
|
||||
retval = true;
|
||||
}
|
||||
|
||||
return retval;
|
||||
};
|
||||
|
||||
auto_ptr<log_format> specialized()
|
||||
{
|
||||
auto_ptr<log_format> retval((log_format *)new syslog_log_format(*this));
|
||||
|
||||
return retval;
|
||||
};
|
||||
};
|
||||
|
||||
log_format::register_root_format<syslog_log_format> syslog_instance;
|
||||
|
||||
class tcsh_history_format : public log_format {
|
||||
string get_name() const { return "tcsh_history"; };
|
||||
|
||||
|
@ -54,6 +54,17 @@ static external_log_format *ensure_format(const std::string &name)
|
||||
return retval;
|
||||
}
|
||||
|
||||
static int read_format_regex(void *ctx, const unsigned char *str, size_t len)
|
||||
{
|
||||
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
|
||||
external_log_format *elf = ensure_format(ypc->get_path_fragment(0));
|
||||
string value = string((const char *)str, len);
|
||||
|
||||
elf->elf_patterns.insert(elf->elf_patterns.begin(), value);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int read_format_field(void *ctx, const unsigned char *str, size_t len)
|
||||
{
|
||||
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
|
||||
@ -61,9 +72,7 @@ static int read_format_field(void *ctx, const unsigned char *str, size_t len)
|
||||
string value = string((const char *)str, len);
|
||||
string field_name = ypc->get_path_fragment(1);
|
||||
|
||||
if (field_name == "regex")
|
||||
elf->elf_regex = value;
|
||||
else if (field_name == "level-field")
|
||||
if (field_name == "level-field")
|
||||
elf->elf_level_field = value;
|
||||
|
||||
return 1;
|
||||
@ -116,7 +125,8 @@ static int read_value_ident(void *ctx, int val)
|
||||
}
|
||||
|
||||
static struct json_path_handler format_handlers[] = {
|
||||
json_path_handler("/\\w+/(regex|level-field)", read_format_field),
|
||||
json_path_handler("/\\w+/regex#", read_format_regex),
|
||||
json_path_handler("/\\w+/(level-field)", read_format_field),
|
||||
json_path_handler("/\\w+/level/"
|
||||
"(trace|debug|info|warning|error|critical|fatal)",
|
||||
read_levels),
|
||||
@ -126,9 +136,10 @@ static struct json_path_handler format_handlers[] = {
|
||||
json_path_handler()
|
||||
};
|
||||
|
||||
void load_formats(void)
|
||||
void load_formats(std::vector<std::string> &errors)
|
||||
{
|
||||
yajlpp_parse_context ypc(format_handlers);
|
||||
std::vector<std::string> retval;
|
||||
yajl_handle handle;
|
||||
|
||||
handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc);
|
||||
@ -141,8 +152,10 @@ void load_formats(void)
|
||||
for (map<string, external_log_format *>::iterator iter = LOG_FORMATS.begin();
|
||||
iter != LOG_FORMATS.end();
|
||||
++iter) {
|
||||
iter->second->build();
|
||||
iter->second->build(errors);
|
||||
|
||||
log_format::get_root_formats().insert(log_format::get_root_formats().begin(), iter->second);
|
||||
if (errors.empty()) {
|
||||
log_format::get_root_formats().insert(log_format::get_root_formats().begin(), iter->second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -32,6 +32,9 @@
|
||||
#ifndef __log_format_loader_hh
|
||||
#define __log_format_loader_hh
|
||||
|
||||
void load_formats(void);
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
void load_formats(std::vector<std::string> &errors);
|
||||
|
||||
#endif
|
||||
|
@ -134,6 +134,10 @@ void logfile::process_prefix(off_t offset, char *prefix, int len)
|
||||
for (iter = root_formats.begin();
|
||||
iter != root_formats.end() && !found;
|
||||
++iter) {
|
||||
(*iter)->lf_base_time = this->lf_line_buffer.get_file_time();
|
||||
if ((*iter)->lf_base_time == 0) {
|
||||
(*iter)->lf_base_time = this->lf_stat.st_mtimespec.tv_sec;
|
||||
}
|
||||
(*iter)->clear();
|
||||
if ((*iter)->scan(this->lf_index, offset, prefix, len)) {
|
||||
#if 0
|
||||
@ -254,7 +258,9 @@ throw (line_buffer::error)
|
||||
retval = true;
|
||||
}
|
||||
|
||||
this->lf_index_time = st.st_mtime;
|
||||
this->lf_index_time = this->lf_line_buffer.get_file_time();
|
||||
if (!this->lf_index_time)
|
||||
this->lf_index_time = st.st_mtime;
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
@ -100,8 +100,7 @@ public:
|
||||
/** @param filename The new filename for this log file. */
|
||||
void set_filename(const std::string &filename)
|
||||
{
|
||||
this->lf_filename =
|
||||
filename;
|
||||
this->lf_filename = filename;
|
||||
};
|
||||
|
||||
/** @return The inode for this log file. */
|
||||
|
@ -213,7 +213,7 @@ void view_colors::init(void)
|
||||
|
||||
for (int z = 0; z < 6; z++) {
|
||||
for (int x = 1; x < 6; x += 2) {
|
||||
for (int y = 0; y < 6; y += 2) {
|
||||
for (int y = 1; y < 6; y += 2) {
|
||||
int fg = 16 + x + (y * 6) + (z * 6 * 6);
|
||||
|
||||
init_pair(color_pair_base,
|
||||
|
@ -397,9 +397,9 @@ public:
|
||||
role_t next_plain_highlight();
|
||||
|
||||
int attrs_for_ident(const char *str, size_t len) {
|
||||
int index = crc32(0, (const Bytef*)str, len);
|
||||
int index = crc32(1, (const Bytef*)str, len);
|
||||
|
||||
return this->vc_role_colors[VCR_HIGHLIGHT_START + (index % HL_COLOR_COUNT)];
|
||||
return this->vc_role_colors[VCR_HIGHLIGHT_START + (abs(index) % HL_COLOR_COUNT)];
|
||||
};
|
||||
|
||||
static inline int ansi_color_pair_index(int fg, int bg)
|
||||
|
@ -116,7 +116,11 @@ drive_sequencer_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB)
|
||||
|
||||
drive_data_scanner_SOURCES = \
|
||||
drive_data_scanner.cc
|
||||
drive_data_scanner_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB)
|
||||
drive_data_scanner_LDADD = \
|
||||
../src/libdiag.a \
|
||||
../src/default-log-formats-json.o \
|
||||
-lcrypto \
|
||||
$(CURSES_LIB)
|
||||
|
||||
drive_view_colors_SOURCES = \
|
||||
../src/view_curses.cc \
|
||||
|
@ -119,7 +119,7 @@ am_drive_data_scanner_OBJECTS = drive_data_scanner.$(OBJEXT)
|
||||
drive_data_scanner_OBJECTS = $(am_drive_data_scanner_OBJECTS)
|
||||
am__DEPENDENCIES_1 =
|
||||
drive_data_scanner_DEPENDENCIES = ../src/libdiag.a \
|
||||
$(am__DEPENDENCIES_1)
|
||||
../src/default-log-formats-json.o $(am__DEPENDENCIES_1)
|
||||
am_drive_grep_proc_OBJECTS = pcrepp.$(OBJEXT) line_buffer.$(OBJEXT) \
|
||||
grep_proc.$(OBJEXT) drive_grep_proc.$(OBJEXT)
|
||||
drive_grep_proc_OBJECTS = $(am_drive_grep_proc_OBJECTS)
|
||||
@ -698,7 +698,12 @@ drive_sequencer_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB)
|
||||
drive_data_scanner_SOURCES = \
|
||||
drive_data_scanner.cc
|
||||
|
||||
drive_data_scanner_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB)
|
||||
drive_data_scanner_LDADD = \
|
||||
../src/libdiag.a \
|
||||
../src/default-log-formats-json.o \
|
||||
-lcrypto \
|
||||
$(CURSES_LIB)
|
||||
|
||||
drive_view_colors_SOURCES = \
|
||||
../src/view_curses.cc \
|
||||
drive_view_colors.cc
|
||||
|
@ -39,6 +39,7 @@
|
||||
#include "data_scanner.hh"
|
||||
#include "data_parser.hh"
|
||||
#include "log_format.hh"
|
||||
#include "log_format_loader.hh"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -49,6 +50,12 @@ int main(int argc, char *argv[])
|
||||
int c, retval = EXIT_SUCCESS;
|
||||
bool prompt = false;
|
||||
|
||||
{
|
||||
std::vector<std::string> errors;
|
||||
|
||||
load_formats(errors);
|
||||
}
|
||||
|
||||
while ((c = getopt(argc, argv, "p")) != -1) {
|
||||
switch (c) {
|
||||
case 'p':
|
||||
|
@ -1,22 +1,4 @@
|
||||
2013-06-05T14:20:24 DEBUG cc2.main CC - 4672610200547811617359537811896212984085567168.114723023 Json_Reader - Doing prepare for resource name "Json_Reader", component "com.json.components.JSONReader"
|
||||
key 26:26 ^
|
||||
sym 26:34 ^------^ cc2.main
|
||||
pair 26:34 ^------^ cc2.main
|
||||
key 35:35 ^
|
||||
sym 35:37 ^^ CC
|
||||
pair 35:37 ^^ CC
|
||||
key 38:38 ^
|
||||
sym 38:39 ^ -
|
||||
pair 38:39 ^ -
|
||||
key 40:40 ^
|
||||
num 40:96 ^------------------------------------------------------^ 4672610200547811617359537811896212984085567168.114723023
|
||||
pair 40:96 ^------------------------------------------------------^ 4672610200547811617359537811896212984085567168.114723023
|
||||
key 97:97 ^
|
||||
sym 97:108 ^---------^ Json_Reader
|
||||
pair 97:108 ^---------^ Json_Reader
|
||||
key 109:109 ^
|
||||
sym 109:110 ^ -
|
||||
pair 109:110 ^ -
|
||||
key 144:144 ^
|
||||
quot 144:155 ^---------^ Json_Reader
|
||||
pair 144:155 ^---------^ Json_Reader
|
||||
|
Loading…
Reference in New Issue
Block a user