From e1dbc7262eb3b95e45e00488b62f9871ff5f2deb Mon Sep 17 00:00:00 2001 From: Matt Dordal Date: Wed, 15 May 2013 01:04:58 -0700 Subject: [PATCH 1/3] Add support for glog logs Glog is an open-source C++ logging framework from Google. Add support for the log format. Testing: run on an example log --- src/lnav.cc | 78 +++++++++++++++++++++++++++++++++++++++++ src/log_format_impls.cc | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/src/lnav.cc b/src/lnav.cc index fe4cdc4c..0555631b 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -2430,6 +2430,7 @@ static void looper(void) "access_log", "syslog_log", "generic_log", + "glog_log", "strace_log", "line_number", @@ -2822,6 +2823,82 @@ private: pcrecpp::RE alt_regex; }; +class glog_log_table : public log_vtab_impl { +public: + + glog_log_table() + : log_vtab_impl("glog_log"), + slt_regex( + "\\s*([IWECF])([0-9]* [0-9:.]*)" // level, date + "\\s*([0-9]*)" // thread + "\\s*(.*:[0-9]*)\\]" // filename:number + "\\s*(.*)" + ) { + }; + + void get_columns(vector &cols) { + cols.push_back(vtab_column("glog_level", "text")); + cols.push_back(vtab_column("glog_thread", "text")); + cols.push_back(vtab_column("glog_date", "text")); + cols.push_back(vtab_column("glog_file", "text")); + cols.push_back(vtab_column("glog_message", "text")); + }; + + void extract(const std::string &line, + int column, + sqlite3_context *ctx) { + string level, thread, date, file, message = "0"; + + if (!this->slt_regex.FullMatch(line, + &level, + &thread, + &date, + &file, + &message + )) { + fprintf(stderr, "bad match! %s\n", line.c_str()); + } + switch (column) { + case 0: + sqlite3_result_text(ctx, + level.c_str(), + level.length(), + SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(ctx, + thread.c_str(), + thread.length(), + SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_text(ctx, + date.c_str(), + date.length(), + SQLITE_TRANSIENT); + break; + case 3: + sqlite3_result_text(ctx, + file.c_str(), + file.length(), + SQLITE_TRANSIENT); + break; + case 4: + sqlite3_result_text(ctx, + message.c_str(), + message.length(), + SQLITE_TRANSIENT); + break; + default: + fprintf(stderr, "bad match! %s\n", line.c_str()); + break; + } + }; + +private: + pcrecpp::RE slt_regex; +}; + class strace_log_table : public log_vtab_impl { public: @@ -3135,6 +3212,7 @@ int main(int argc, char *argv[]) lnav_data.ld_vtab_manager->register_vtab(new log_vtab_impl("syslog_log")); lnav_data.ld_vtab_manager->register_vtab(new log_vtab_impl("generic_log")); lnav_data.ld_vtab_manager->register_vtab(new access_log_table()); + lnav_data.ld_vtab_manager->register_vtab(new glog_log_table()); lnav_data.ld_vtab_manager->register_vtab(new strace_log_table()); lnav_data.ld_vtab_manager->register_vtab(new log_data_table()); diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 832c9655..e3f15ec0 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -235,6 +235,67 @@ class generic_log_format : public log_format { log_format::register_root_format generic_log_instance; +class glog_log_format : public log_format { + string get_name() { return "glog_log"; }; + + bool scan(vector < logline > &dst, + off_t offset, + char *prefix, + int len) { + bool retval = false; + struct tm log_time; + short millis = 0; + time_t now; + char *rest; + + now = time(NULL); + localtime_r(&now, &log_time); + + log_time.tm_isdst = 0; + + if ((rest = strptime(prefix + 1, + "%m%d %H:%M:%S.", + &log_time)) != NULL) { + logline::level_t ll = logline::LEVEL_UNKNOWN; + time_t log_gmt; + + millis = atoi(rest) / 1000; + + switch (*prefix) { + case 'I': // info + ll = logline::LEVEL_INFO; + break; + case 'W': // warning + ll = logline::LEVEL_WARNING; + break; + case 'E': // error + ll = logline::LEVEL_ERROR; + break; + case 'C': // critical + ll = logline::LEVEL_CRITICAL; + break; + case 'F': // fatal + ll = logline::LEVEL_CRITICAL; + break; + } + log_gmt = tm2sec(&log_time); + dst.push_back(logline(offset, log_gmt, millis, ll)); + + retval = true; + } + + return retval; + }; + + auto_ptr specialized() { + auto_ptr retval((log_format *)new glog_log_format(*this)); + + return retval; + }; +}; + +log_format::register_root_format glog_instance; + class strace_log_format : public log_format { string get_name() { return "strace_log"; }; From e35c5006cd2018f2c96bd4e5252cc0bff759c398 Mon Sep 17 00:00:00 2001 From: Matt Dordal Date: Wed, 15 May 2013 01:06:56 -0700 Subject: [PATCH 2/3] update .gitignore with automake stuff --- .gitignore | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 69dca4f6..87443d31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,15 @@ autom4te.cache - +Makefile +TESTS_ENVIRONMENT +config.status +src/.deps/ +src/Makefile +src/bin2c +src/config.h +src/help.c +src/libdiag.a +src/lnav +src/stamp-h1 +src/static-libs/ +test/.deps/ +test/Makefile From fa18aa4b60d711b9254b182e2685eeb22b9b44c8 Mon Sep 17 00:00:00 2001 From: Matt Dordal Date: Fri, 17 May 2013 14:53:01 -0700 Subject: [PATCH 3/3] Fix up some clowniness with glog db --- src/lnav.cc | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/lnav.cc b/src/lnav.cc index 0555631b..bde140dc 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -13,10 +13,10 @@ #include #include +#include #include #include #include -#include #include #include @@ -2829,7 +2829,7 @@ public: glog_log_table() : log_vtab_impl("glog_log"), slt_regex( - "\\s*([IWECF])([0-9]* [0-9:.]*)" // level, date + "\\s*([IWECF])([0-9]*) ([0-9:.]*)" // level, date "\\s*([0-9]*)" // thread "\\s*(.*:[0-9]*)\\]" // filename:number "\\s*(.*)" @@ -2838,26 +2838,31 @@ public: void get_columns(vector &cols) { cols.push_back(vtab_column("glog_level", "text")); - cols.push_back(vtab_column("glog_thread", "text")); - cols.push_back(vtab_column("glog_date", "text")); - cols.push_back(vtab_column("glog_file", "text")); - cols.push_back(vtab_column("glog_message", "text")); + cols.push_back(vtab_column("timestamp", "text")); + cols.push_back(vtab_column("thread", "text")); + cols.push_back(vtab_column("file", "text")); + cols.push_back(vtab_column("message", "text")); }; void extract(const std::string &line, int column, sqlite3_context *ctx) { - string level, thread, date, file, message = "0"; + string level, date, time, thread, file, message = "0"; if (!this->slt_regex.FullMatch(line, &level, - &thread, &date, + &time, + &thread, &file, &message )) { fprintf(stderr, "bad match! %s\n", line.c_str()); } + struct tm log_time; + time_t now = ::time(NULL); + stringstream timestamp; + char buf[128]; switch (column) { case 0: sqlite3_result_text(ctx, @@ -2866,15 +2871,24 @@ public: SQLITE_TRANSIENT); break; case 1: + localtime_r(&now, &log_time); // need year data + strptime(date.data(), "%m%d", &log_time); + strftime(buf, sizeof(buf), "%Y-%m-%d", &log_time); + // C++11 can do this much more nicely: + //timestamp << std::put_time(&log_time, "%Y-%m-%d "); + timestamp + << buf + << " " + << time; sqlite3_result_text(ctx, - thread.c_str(), - thread.length(), + timestamp.str().c_str(), + timestamp.str().length(), SQLITE_TRANSIENT); break; case 2: sqlite3_result_text(ctx, - date.c_str(), - date.length(), + thread.c_str(), + thread.length(), SQLITE_TRANSIENT); break; case 3: