tstack/lnav
1
1
Fork 0
mirror of https://github.com/tstack/lnav.git synced 2024-09-11 13:05:51 +03:00
Code Issues Projects Releases Wiki Activity

[line_buffer] split long lines

Browse Source 
Fixes #81
This commit is contained in:
Timothy Stack 2014-03-15 04:40:58 -07:00
parent eba916749b
commit 9ff2d3774d
23 changed files with 294 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.in
View File

@ -201,6 +201,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@ AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@ AUTOMAKE = @AUTOMAKE@
AWK = @AWK@ AWK = @AWK@
BZIP2_CMD = @BZIP2_CMD@
CC = @CC@ CC = @CC@
CCDEPMODE = @CCDEPMODE@ CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@ CFLAGS = @CFLAGS@

6
TESTS_ENVIRONMENT.in
View File

@ -3,6 +3,9 @@
top_srcdir="@abssrcdir@" top_srcdir="@abssrcdir@"
export top_srcdir export top_srcdir
srcdir="@abssrcdir@/test"
export srcdir
# The top build directory, derived from the path to this script. # The top build directory, derived from the path to this script.
top_builddir=`dirname $0` top_builddir=`dirname $0`
export top_builddir export top_builddir
@ -10,6 +13,9 @@ export top_builddir
test_dir="@abssrcdir@/test" test_dir="@abssrcdir@/test"
export test_dir export test_dir
BZIP2_CMD="@BZIP2_CMD@"
export BZIP2_CMD
# The full path of the test case # The full path of the test case
test_file=$1 test_file=$1
# The base name of the test case # The base name of the test case

42
configure vendored
View File

@ -646,6 +646,7 @@ CCDEPMODE
ac_ct_CC ac_ct_CC
CFLAGS CFLAGS
CC CC
BZIP2_CMD
SQLITE3_CMD SQLITE3_CMD
pkgpyexecdir pkgpyexecdir
pyexecdir pyexecdir
@ -4451,6 +4452,47 @@ if test x"$SQLITE3_CMD" = x""; then
as_fn_error $? "The sqlite3 command is required" "$LINENO" 5 as_fn_error $? "The sqlite3 command is required" "$LINENO" 5
fi fi
# Extract the first word of "bzip2", so it can be a program name with args.
set dummy bzip2; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
if ${ac_cv_path_BZIP2_CMD+:} false; then :
$as_echo_n "(cached) " >&6
else
case $BZIP2_CMD in
[\\/]* | ?:[\\/]*)
ac_cv_path_BZIP2_CMD="$BZIP2_CMD" # Let the user override the test with a path.
;;
*)
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_path_BZIP2_CMD="$as_dir/$ac_word$ac_exec_ext"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
done
IFS=$as_save_IFS
;;
esac
fi
BZIP2_CMD=$ac_cv_path_BZIP2_CMD
if test -n "$BZIP2_CMD"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BZIP2_CMD" >&5
$as_echo "$BZIP2_CMD" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
ac_ext=c ac_ext=c
ac_cpp='$CPP $CPPFLAGS' ac_cpp='$CPP $CPPFLAGS'
ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'

2
configure.ac
View File

@ -73,6 +73,8 @@ if test x"$SQLITE3_CMD" = x""; then
AC_MSG_ERROR([The sqlite3 command is required]) AC_MSG_ERROR([The sqlite3 command is required])
fi fi
AC_PATH_PROG(BZIP2_CMD, [bzip2])
AC_CHECK_SIZEOF(off_t) AC_CHECK_SIZEOF(off_t)
AC_CHECK_SIZEOF(size_t) AC_CHECK_SIZEOF(size_t)

1
src/Makefile.in
View File

@ -230,6 +230,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@ AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@ AUTOMAKE = @AUTOMAKE@
AWK = @AWK@ AWK = @AWK@
BZIP2_CMD = @BZIP2_CMD@
CC = @CC@ CC = @CC@
CCDEPMODE = @CCDEPMODE@ CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@ CFLAGS = @CFLAGS@

2
src/default-log-formats.json
View File

@ -286,7 +286,7 @@
"url" : "http://www.snaplogic.com/docs/user-guide/user-guide.htm", "url" : "http://www.snaplogic.com/docs/user-guide/user-guide.htm",
"regex" : { "regex" : {
"std" : { "std" : {
"pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?:(?:(?<level>\\w+) (?<logger>[^ ]+) (?<facility>[^ ]+) (?<msgid>[^ ]+) (?<pipe_rid>[^ \\.]+)(?:\\.(?<comp_rid>[^ ]+))? (?<resource_name>[^ ]+) (?<invoker>[^ ]+))|(?:(?:stdout|stderr): ))(?<body>.*)" "pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?:(?:(?<level>\\w{4,}) (?<logger>[^ ]+) (?<facility>[^ ]+) (?<msgid>[^ ]+) (?<pipe_rid>-|\\d+)(?:\\.(?<comp_rid>[^ ]+))? (?<resource_name>[^ ]+) (?<invoker>[^ ]+))|(?:(?:stdout|stderr): ))(?<body>.*)"
} }
}, },
"level-field" : "level", "level-field" : "level",

10
src/grep_proc.cc
View File

@ -355,14 +355,12 @@ void grep_proc::check_fd_set(fd_set &ready_fds)
static const int MAX_LOOPS = 100; static const int MAX_LOOPS = 100;
int loop_count = 0; int loop_count = 0;
size_t len; line_value lv;
char * line;
while ((loop_count < MAX_LOOPS) && while ((loop_count < MAX_LOOPS) &&
(line = this->gp_line_buffer.read_line(this->gp_pipe_offset, (this->gp_line_buffer.read_line(this->gp_pipe_offset, lv))) {
len)) != NULL) { lv.lv_start[lv.lv_len] = '\0';
line[len] = '\0'; this->dispatch_line(lv.lv_start);
this->dispatch_line(line);
loop_count += 1; loop_count += 1;
} }

124
src/line_buffer.cc
View File

@ -114,6 +114,7 @@ private:
line_buffer::line_buffer() line_buffer::line_buffer()
: lb_gz_file(NULL), : lb_gz_file(NULL),
lb_bz_file(false), lb_bz_file(false),
lb_gz_offset(0),
lb_file_size((size_t)-1), lb_file_size((size_t)-1),
lb_file_offset(0), lb_file_offset(0),
lb_file_time(0), lb_file_time(0),
@ -181,7 +182,7 @@ throw (error)
throw error(errno); throw error(errno);
} }
} }
this->lb_file_time = *((time_t *)&gz_id[4]); this->lb_file_time = *((int32_t *)&gz_id[4]);
if (this->lb_file_time < 0) if (this->lb_file_time < 0)
this->lb_file_time = 0; this->lb_file_time = 0;
this->lb_gz_offset = lseek(this->lb_fd, 0, SEEK_CUR); this->lb_gz_offset = lseek(this->lb_fd, 0, SEEK_CUR);
@ -214,6 +215,9 @@ throw (error)
{ {
char *tmp, *old; char *tmp, *old;
require(this->lb_bz_file || this->lb_gz_file ||
new_max <= MAX_LINE_BUFFER_SIZE);
/* Still need more space, try a realloc. */ /* Still need more space, try a realloc. */
old = this->lb_buffer.release(); old = this->lb_buffer.release();
tmp = (char *)realloc(old, new_max); tmp = (char *)realloc(old, new_max);
@ -233,10 +237,7 @@ throw (error)
{ {
size_t prefill, available; size_t prefill, available;
/* The file is probably bogus if a line has gotten this big. */ require(max_length <= MAX_LINE_BUFFER_SIZE);
if (max_length > MAX_LINE_BUFFER_SIZE) {
throw error(EFBIG);
}
/* /*
* Check to see if the start is inside the cached range or immediately * Check to see if the start is inside the cached range or immediately
@ -273,7 +274,7 @@ throw (error)
require(this->lb_file_offset <= start); require(this->lb_file_offset <= start);
require(prefill <= this->lb_buffer_size); require(prefill <= this->lb_buffer_size);
available = this->lb_buffer_max - this->lb_buffer_size; available = this->lb_buffer_max - (start - this->lb_file_offset);
require(available <= this->lb_buffer_max); require(available <= this->lb_buffer_max);
if (max_length > available) { if (max_length > available) {
@ -287,7 +288,7 @@ throw (error)
&this->lb_buffer[prefill], &this->lb_buffer[prefill],
this->lb_buffer_size); this->lb_buffer_size);
available = this->lb_buffer_max - this->lb_buffer_size; available = this->lb_buffer_max - (start - this->lb_file_offset);
if (max_length > available) { if (max_length > available) {
this->resize_buffer(this->lb_buffer_max + this->resize_buffer(this->lb_buffer_max +
DEFAULT_LINE_BUFFER_SIZE); DEFAULT_LINE_BUFFER_SIZE);
@ -376,6 +377,12 @@ throw (error)
&this->lb_buffer[this->lb_buffer_size], &this->lb_buffer[this->lb_buffer_size],
this->lb_buffer_max - this->lb_buffer_size); this->lb_buffer_max - this->lb_buffer_size);
BZ2_bzclose(bz_file); BZ2_bzclose(bz_file);
if (rc != -1 && (
rc < (this->lb_buffer_max - this->lb_buffer_size))) {
this->lb_file_size = (
this->lb_file_offset + this->lb_buffer_size + rc);
}
} }
} }
#endif #endif
@ -398,7 +405,9 @@ throw (error)
} }
switch (rc) { switch (rc) {
case 0: case 0:
this->lb_file_size = this->lb_file_offset + this->lb_buffer_size; if (!this->lb_seekable) {
this->lb_file_size = this->lb_file_offset + this->lb_buffer_size;
}
if (start < (off_t) this->lb_file_size) { if (start < (off_t) this->lb_file_size) {
retval = true; retval = true;
} }
@ -408,9 +417,7 @@ throw (error)
* For compressed files, increase the buffer size so we don't * For compressed files, increase the buffer size so we don't
* have to spend as much time uncompressing the data. * have to spend as much time uncompressing the data.
*/ */
this->resize_buffer(std::min(this->lb_file_size + this->resize_buffer(MAX_COMPRESSED_BUFFER_SIZE);
DEFAULT_INCREMENT,
MAX_COMPRESSED_BUFFER_SIZE));
} }
break; break;
@ -442,11 +449,11 @@ throw (error)
return retval; return retval;
} }
char *line_buffer::read_line(off_t &offset, size_t &len_out, bool include_delim) bool line_buffer::read_line(off_t &offset, line_value &lv, bool include_delim)
throw (error) throw (error)
{ {
size_t request_size = DEFAULT_INCREMENT; size_t request_size = DEFAULT_INCREMENT;
char * retval = NULL; bool retval = false;
require(this->lb_fd != -1); require(this->lb_fd != -1);
@ -456,70 +463,92 @@ throw (error)
* Don't return anything past the last known line. The caller needs * Don't return anything past the last known line. The caller needs
* to try reading at the offset of the last line again. * to try reading at the offset of the last line again.
*/ */
return NULL; return false;
} }
len_out = 0; lv.lv_len = 0;
while (retval == NULL) { lv.lv_partial = false;
while (!retval) {
char *line_start, *lf; char *line_start, *lf;
this->fill_range(offset, request_size); this->fill_range(offset, request_size);
/* Find the data in the cache and */ /* Find the data in the cache and */
line_start = this->get_range(offset, len_out); line_start = this->get_range(offset, lv.lv_len);
/* ... look for the end-of-line or end-of-file. */ /* ... look for the end-of-line or end-of-file. */
if (((lf = (char *)memchr(line_start, '\n', len_out)) != NULL) || if (((lf = (char *)memchr(line_start, '\n', lv.lv_len)) != NULL) ||
(request_size >= (MAX_LINE_BUFFER_SIZE - DEFAULT_INCREMENT)) || (lv.lv_len >= MAX_LINE_BUFFER_SIZE) ||
(((offset + len_out) == this->lb_file_size) && len_out > 0)) { (request_size == MAX_LINE_BUFFER_SIZE) ||
((request_size > lv.lv_len) && lv.lv_len > 0)) {
if ((lf != NULL) && ((lf - line_start) >= MAX_LINE_BUFFER_SIZE - 1)) {
lf = NULL;
}
if (lf != NULL) { if (lf != NULL) {
len_out = lf - line_start; lv.lv_partial = false;
lv.lv_len = lf - line_start;
if (include_delim) { if (include_delim) {
len_out += 1; lv.lv_len += 1;
} }
else { else {
offset += 1; /* Skip the delimiter. */ offset += 1; /* Skip the delimiter. */
} }
if (offset >= this->lb_last_line_offset) { if (offset >= this->lb_last_line_offset) {
this->lb_last_line_offset = offset + len_out; this->lb_last_line_offset = offset + lv.lv_len;
} }
} }
else { else {
if (lv.lv_len >= MAX_LINE_BUFFER_SIZE) {
lv.lv_len = MAX_LINE_BUFFER_SIZE - 1;
lv.lv_partial = false;
}
else {
lv.lv_partial = true;
}
/* /*
* Be nice and make sure there is room for the caller to * Be nice and make sure there is room for the caller to
* add a NULL-terminator. * add a NULL-terminator.
*/ */
this->ensure_available(offset, len_out + 1); this->ensure_available(offset, lv.lv_len + 1);
line_start = this->get_range(offset, len_out); line_start = this->get_range(offset, lv.lv_len);
/* if (lv.lv_len >= MAX_LINE_BUFFER_SIZE) {
* Since no delimiter was seen, we need to remember the offset lv.lv_len = MAX_LINE_BUFFER_SIZE - 1;
* of the last line in the file so we don't mistakenly return }
* two partial lines to the caller. if (lv.lv_partial) {
* /*
* 1. read_line() - returns partial line * Since no delimiter was seen, we need to remember the offset
* 2. file is written * of the last line in the file so we don't mistakenly return
* 3. read_line() - returns the middle of partial line. * two partial lines to the caller.
*/ *
this->lb_last_line_offset = offset; * 1. read_line() - returns partial line
* 2. file is written
* 3. read_line() - returns the middle of partial line.
*/
this->lb_last_line_offset = offset;
}
else if (offset >= this->lb_last_line_offset) {
this->lb_last_line_offset = offset + lv.lv_len;
}
} }
offset += len_out; lv.lv_start = line_start;
offset += lv.lv_len;
retval = line_start; retval = true;
} }
else { else {
request_size += DEFAULT_INCREMENT; request_size += DEFAULT_INCREMENT;
} }
if ((retval == NULL) && !this->fill_range(offset, request_size)) { if (!retval && !this->fill_range(offset, request_size)) {
break; break;
} }
} }
ensure((retval == NULL) || ensure(lv.lv_len <= this->lb_buffer_size);
(retval >= this->lb_buffer && ensure(!retval ||
retval < (this->lb_buffer + this->lb_buffer_size))); (lv.lv_start >= this->lb_buffer &&
ensure(len_out <= this->lb_buffer_size); (lv.lv_start + lv.lv_len) <= (this->lb_buffer + this->lb_buffer_size)));
ensure(this->invariant()); ensure(this->invariant());
return retval; return retval;
@ -528,18 +557,17 @@ throw (error)
bool line_buffer::read_line(off_t &offset_inout, shared_buffer_ref &sbr) bool line_buffer::read_line(off_t &offset_inout, shared_buffer_ref &sbr)
throw (error) throw (error)
{ {
char *line; line_value lv;
size_t len; bool retval;
// Clear the incoming ref right away so that an invalidate // Clear the incoming ref right away so that an invalidate
// does not cause a wasted malloc/copy. // does not cause a wasted malloc/copy.
sbr.disown(); sbr.disown();
line = this->read_line(offset_inout, len); if ((retval = this->read_line(offset_inout, lv))) {
if (line != NULL) { sbr.share(this->lb_share_manager, lv.lv_start, lv.lv_len);
sbr.share(this->lb_share_manager, line, len);
} }
return line != NULL; return retval;
} }
bool line_buffer::read_range(off_t offset, size_t len, shared_buffer_ref &sbr) bool line_buffer::read_range(off_t offset, size_t len, shared_buffer_ref &sbr)

17
src/line_buffer.hh
View File

@ -44,6 +44,16 @@
#include "auto_mem.hh" #include "auto_mem.hh"
#include "shared_buffer.hh" #include "shared_buffer.hh"
struct line_value {
char *lv_start;
size_t lv_len;
bool lv_partial;
void terminate() {
this->lv_start[this->lv_len] = '\0';
};
};
/** /**
* Buffer for reading whole lines out of file descriptors. The class presents * Buffer for reading whole lines out of file descriptors. The class presents
* a stateless interface, callers specify the offset where a line starts and * a stateless interface, callers specify the offset where a line starts and
@ -82,6 +92,10 @@ public:
*/ */
ssize_t get_file_size() const { return this->lb_file_size; }; ssize_t get_file_size() const { return this->lb_file_size; };
bool is_compressed() const {
return this->lb_gz_file != NULL || this->lb_bz_file;
};
off_t get_read_offset(off_t off) const off_t get_read_offset(off_t off) const
{ {
if (this->lb_gz_file) { if (this->lb_gz_file) {
@ -108,7 +122,7 @@ public:
* NULL termination, the invalidate() must be called before re-reading the * NULL termination, the invalidate() must be called before re-reading the
* line to refresh the buffer. * line to refresh the buffer.
*/ */
char *read_line(off_t &offset_inout, size_t &len_out, bool read_line(off_t &offset_inout, line_value &lv,
bool include_delim = false) bool include_delim = false)
throw (error); throw (error);
@ -128,6 +142,7 @@ public:
this->lb_file_offset += this->lb_buffer_size; this->lb_file_offset += this->lb_buffer_size;
this->lb_buffer_size = 0; this->lb_buffer_size = 0;
this->lb_file_size = -1; this->lb_file_size = -1;
log_info("invalidate %p", this);
}; };
/** Release any resources held by this object. */ /** Release any resources held by this object. */

9
src/lnav.cc
View File

@ -623,7 +623,7 @@ public:
off = total; off = total;
} }
if ((size_t)off == total || if ((((size_t)off == total) && (this->lo_last_offset != off)) ||
ui_periodic_timer::singleton().time_to_update(index_counter)) { ui_periodic_timer::singleton().time_to_update(index_counter)) {
lnav_data.ld_bottom_source.update_loading(off, total); lnav_data.ld_bottom_source.update_loading(off, total);
this->do_update(); this->do_update();
@ -3085,10 +3085,9 @@ static string execute_action(log_data_helper &ldh,
static int exec_count = 0; static int exec_count = 0;
string value = lv.to_string(); string value = lv.to_string();
const char *line;
line_buffer lb; line_buffer lb;
size_t len;
off_t off = 0; off_t off = 0;
line_value lv;
lnav_data.ld_children.push_back(child_pid); lnav_data.ld_children.push_back(child_pid);
@ -3099,7 +3098,7 @@ static string execute_action(log_data_helper &ldh,
lb.set_fd(err_pipe.read_end()); lb.set_fd(err_pipe.read_end());
line = lb.read_line(off, len); lb.read_line(off, lv);
if (out_pipe.read_end() != -1) { if (out_pipe.read_end() != -1) {
piper_proc *pp = new piper_proc(out_pipe.read_end(), false); piper_proc *pp = new piper_proc(out_pipe.read_end(), false);
@ -3116,7 +3115,7 @@ static string execute_action(log_data_helper &ldh,
lnav_data.ld_files_to_front.push_back(make_pair(desc, 0)); lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
} }
retval = string(line, len); retval = string(lv.lv_start, lv.lv_len);
} }
break; break;
} }

1
src/lnav_util.cc
View File

@ -286,6 +286,7 @@ const char *date_time_scanner::scan(const char *time_dest,
gmt = tm2sec(tm_out); gmt = tm2sec(tm_out);
} }
tv_out.tv_sec = gmt; tv_out.tv_sec = gmt;
tv_out.tv_usec = 0;
this->dts_fmt_lock = curr_time_fmt; this->dts_fmt_lock = curr_time_fmt;
this->dts_fmt_len = off; this->dts_fmt_len = off;

35
src/logfile.cc
View File

@ -232,8 +232,7 @@ throw (line_buffer::error, logfile::error)
/* Check for new data based on the file size. */ /* Check for new data based on the file size. */
if (this->lf_index_size < st.st_size) { if (this->lf_index_size < st.st_size) {
off_t last_off, off; off_t last_off, off;
char * line; line_value lv;
size_t len;
if (!this->lf_index.empty()) { if (!this->lf_index.empty()) {
off = this->lf_index.back().get_offset(); off = this->lf_index.back().get_offset();
@ -251,26 +250,20 @@ throw (line_buffer::error, logfile::error)
off = 0; off = 0;
} }
last_off = off; last_off = off;
while ((line = this->lf_line_buffer.read_line(off, len, true)) != NULL) { while (this->lf_line_buffer.read_line(off, lv)) {
if (len > 0 && line[len - 1] == '\n') { char tmp = lv.lv_start[lv.lv_len];
this->lf_partial_line = false;
line[len - 1] = '\0'; this->lf_partial_line = lv.lv_partial;
len -= 1; lv.lv_start[lv.lv_len] = '\0';
} this->process_prefix(last_off, lv.lv_start, lv.lv_len);
else { lv.lv_start[lv.lv_len] = tmp;
this->lf_partial_line = true;
line[len] = '\0';
}
this->process_prefix(last_off, line, len);
if (!this->lf_partial_line) {
line[len - 1] = '\n';
}
last_off = off; last_off = off;
if (lo != NULL) { if (lo != NULL) {
lo->logfile_indexing(*this, lo->logfile_indexing(
this->lf_line_buffer.get_read_offset(off), *this,
st.st_size); this->lf_line_buffer.get_read_offset(off),
st.st_size);
} }
} }
@ -279,9 +272,9 @@ throw (line_buffer::error, logfile::error)
* doing the scanning, so use the line buffer's notion of the file * doing the scanning, so use the line buffer's notion of the file
* size. * size.
*/ */
this->lf_index_size = this->lf_line_buffer.get_file_size(); this->lf_index_size = off;
this->lf_line_buffer.invalidate(); // this->lf_line_buffer.invalidate();
retval = true; retval = true;
} }

4
src/logfile.hh
View File

@ -113,6 +113,10 @@ public:
/** @return The inode for this log file. */ /** @return The inode for this log file. */
const struct stat &get_stat() const { return this->lf_stat; }; const struct stat &get_stat() const { return this->lf_stat; };
bool is_compressed() const {
return this->lf_line_buffer.is_compressed();
};
/** /**
* @return The detected format, rebuild_index() must be called before this * @return The detected format, rebuild_index() must be called before this
* will return a value other than NULL. * will return a value other than NULL.

9
src/piper_proc.cc
View File

@ -108,7 +108,6 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
line_buffer lb; line_buffer lb;
off_t woff = 0, last_woff = 0; off_t woff = 0, last_woff = 0;
off_t off = 0, last_off = 0; off_t off = 0, last_off = 0;
size_t len;
int nullfd; int nullfd;
nullfd = open("/dev/null", O_RDWR); nullfd = open("/dev/null", O_RDWR);
@ -116,14 +115,14 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
fcntl(infd.get(), F_SETFL, O_NONBLOCK); fcntl(infd.get(), F_SETFL, O_NONBLOCK);
lb.set_fd(infd); lb.set_fd(infd);
do { do {
line_value lv;
fd_set rready; fd_set rready;
char * line;
FD_ZERO(&rready); FD_ZERO(&rready);
FD_SET(lb.get_fd(), &rready); FD_SET(lb.get_fd(), &rready);
select(lb.get_fd() + 1, &rready, NULL, NULL, NULL); select(lb.get_fd() + 1, &rready, NULL, NULL, NULL);
last_off = off; last_off = off;
while ((line = lb.read_line(off, len, true)) != NULL) { while (lb.read_line(off, lv, true)) {
int wrc; int wrc;
last_woff = woff; last_woff = woff;
@ -139,14 +138,14 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
/* Need to do pwrite here since the fd is used by the main /* Need to do pwrite here since the fd is used by the main
* lnav process as well. * lnav process as well.
*/ */
wrc = pwrite(this->pp_fd, line, len, woff); wrc = pwrite(this->pp_fd, lv.lv_start, lv.lv_len, woff);
if (wrc == -1) { if (wrc == -1) {
perror("Unable to write to output file for stdin"); perror("Unable to write to output file for stdin");
break; break;
} }
woff += wrc; woff += wrc;
if (line[len - 1] != '\n') { if (lv.lv_start[lv.lv_len - 1] != '\n') {
off = last_off; off = last_off;
woff = last_woff; woff = last_woff;
} }

7
test/Makefile.am
View File

@ -86,6 +86,8 @@ drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz
drive_logfile_SOURCES = drive_logfile.cc drive_logfile_SOURCES = drive_logfile.cc
drive_logfile_LDADD = \ drive_logfile_LDADD = \
../src/libdiag.a \ ../src/libdiag.a \
../src/default-log-formats-json.o \
../src/dump-pid-sh.o \
$(CURSES_LIB) \ $(CURSES_LIB) \
$(SQLITE3_LIBS) \ $(SQLITE3_LIBS) \
$(PCRE_LIBS) \ $(PCRE_LIBS) \
@ -213,6 +215,7 @@ TESTS = \
test_grep_proc2 \ test_grep_proc2 \
test_hist_source \ test_hist_source \
test_json_format.sh \ test_json_format.sh \
test_logfile.sh \
test_pcrepp \ test_pcrepp \
test_sessions.sh \ test_sessions.sh \
test_sql.sh \ test_sql.sh \
@ -235,7 +238,9 @@ DISTCLEANFILES = \
*.dpt \ *.dpt \
*.diff \ *.diff \
*.index \ *.index \
*.tmp *.tmp \
*.gz \
*.bz2
distclean-local: distclean-local:
rm -rf sessions rm -rf sessions

30
test/Makefile.in
View File

@ -96,11 +96,11 @@ TESTS = test_ansi_scrubber$(EXEEXT) test_auto_fd$(EXEEXT) \
test_line_buffer2$(EXEEXT) test_line_buffer.sh \ test_line_buffer2$(EXEEXT) test_line_buffer.sh \
test_listview.sh test_mvwattrline.sh test_grep_proc.sh \ test_listview.sh test_mvwattrline.sh test_grep_proc.sh \
test_grep_proc2$(EXEEXT) test_hist_source$(EXEEXT) \ test_grep_proc2$(EXEEXT) test_hist_source$(EXEEXT) \
test_json_format.sh test_pcrepp$(EXEEXT) test_sessions.sh \ test_json_format.sh test_logfile.sh test_pcrepp$(EXEEXT) \
test_sql.sh test_sql_coll_func.sh test_sql_fs_func.sh \ test_sessions.sh test_sql.sh test_sql_coll_func.sh \
test_sql_str_func.sh test_view_colors.sh test_vt52_curses.sh \ test_sql_fs_func.sh test_sql_str_func.sh test_view_colors.sh \
test_top_status$(EXEEXT) test_data_parser.sh \ test_vt52_curses.sh test_top_status$(EXEEXT) \
test_yajlpp$(EXEEXT) test_data_parser.sh test_yajlpp$(EXEEXT)
subdir = test subdir = test
DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
$(top_srcdir)/mkinstalldirs $(dist_noinst_SCRIPTS) \ $(top_srcdir)/mkinstalldirs $(dist_noinst_SCRIPTS) \
@ -135,8 +135,10 @@ drive_listview_OBJECTS = $(am_drive_listview_OBJECTS)
drive_listview_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1) drive_listview_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1)
am_drive_logfile_OBJECTS = drive_logfile.$(OBJEXT) am_drive_logfile_OBJECTS = drive_logfile.$(OBJEXT)
drive_logfile_OBJECTS = $(am_drive_logfile_OBJECTS) drive_logfile_OBJECTS = $(am_drive_logfile_OBJECTS)
drive_logfile_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1) \ drive_logfile_DEPENDENCIES = ../src/libdiag.a \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) ../src/default-log-formats-json.o ../src/dump-pid-sh.o \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1)
am_drive_mvwattrline_OBJECTS = drive_mvwattrline.$(OBJEXT) am_drive_mvwattrline_OBJECTS = drive_mvwattrline.$(OBJEXT)
drive_mvwattrline_OBJECTS = $(am_drive_mvwattrline_OBJECTS) drive_mvwattrline_OBJECTS = $(am_drive_mvwattrline_OBJECTS)
drive_mvwattrline_DEPENDENCIES = ../src/libdiag.a \ drive_mvwattrline_DEPENDENCIES = ../src/libdiag.a \
@ -492,6 +494,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@ AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@ AUTOMAKE = @AUTOMAKE@
AWK = @AWK@ AWK = @AWK@
BZIP2_CMD = @BZIP2_CMD@
CC = @CC@ CC = @CC@
CCDEPMODE = @CCDEPMODE@ CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@ CFLAGS = @CFLAGS@
@ -648,6 +651,8 @@ drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz
drive_logfile_SOURCES = drive_logfile.cc drive_logfile_SOURCES = drive_logfile.cc
drive_logfile_LDADD = \ drive_logfile_LDADD = \
../src/libdiag.a \ ../src/libdiag.a \
../src/default-log-formats-json.o \
../src/dump-pid-sh.o \
$(CURSES_LIB) \ $(CURSES_LIB) \
$(SQLITE3_LIBS) \ $(SQLITE3_LIBS) \
$(PCRE_LIBS) \ $(PCRE_LIBS) \
@ -766,7 +771,9 @@ DISTCLEANFILES = \
*.dpt \ *.dpt \
*.diff \ *.diff \
*.index \ *.index \
*.tmp *.tmp \
*.gz \
*.bz2
all: all-am all: all-am
@ -1228,6 +1235,13 @@ test_json_format.sh.log: test_json_format.sh
--log-file $$b.log --trs-file $$b.trs \ --log-file $$b.log --trs-file $$b.trs \
$(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \ $(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \
"$$tst" $(AM_TESTS_FD_REDIRECT) "$$tst" $(AM_TESTS_FD_REDIRECT)
test_logfile.sh.log: test_logfile.sh
@p='test_logfile.sh'; \
b='test_logfile.sh'; \
$(am__check_pre) $(LOG_DRIVER) --test-name "$$f" \
--log-file $$b.log --trs-file $$b.trs \
$(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \
"$$tst" $(AM_TESTS_FD_REDIRECT)
test_pcrepp.log: test_pcrepp$(EXEEXT) test_pcrepp.log: test_pcrepp$(EXEEXT)
@p='test_pcrepp$(EXEEXT)'; \ @p='test_pcrepp$(EXEEXT)'; \
b='test_pcrepp'; \ b='test_pcrepp'; \

8
test/drive_grep_proc.cc
View File

@ -54,12 +54,10 @@ public:
bool retval = false; bool retval = false;
try { try {
size_t len; line_value lv;
char *line;
if ((line = this->ms_buffer.read_line(this->ms_offset, if (this->ms_buffer.read_line(this->ms_offset, lv)) {
len)) != NULL) { value_out = string(lv.lv_start, lv.lv_len);
value_out = string(line, len);
retval = true; retval = true;
} }
} }

22
test/drive_line_buffer.cc
View File

@ -124,16 +124,15 @@ int main(int argc, char *argv[])
try { try {
off_t last_offset = offset; off_t last_offset = offset;
line_buffer lb; line_buffer lb;
line_value lv;
char *maddr; char *maddr;
char *line;
size_t len;
lb.set_fd(fd); lb.set_fd(fd);
if (index.size() == 0) { if (index.size() == 0) {
while ((line = lb.read_line(offset, len)) != NULL) { while (lb.read_line(offset, lv)) {
line[len] = '\0'; lv.terminate();
printf("%s", line); printf("%s", lv.lv_start);
if ((last_offset + len) < offset) if ((last_offset + lv.lv_len) < offset)
printf("\n"); printf("\n");
last_offset = offset; last_offset = offset;
} }
@ -150,22 +149,23 @@ int main(int argc, char *argv[])
else { else {
off_t seq_offset = 0; off_t seq_offset = 0;
while (lb.read_line(seq_offset, len) != NULL) { } while (lb.read_line(seq_offset, lv)) { }
do { do {
bool ret;
int lpc; int lpc;
random_shuffle(index.begin(), index.end()); random_shuffle(index.begin(), index.end());
for (lpc = 0; lpc < index.size(); lpc++) { for (lpc = 0; lpc < index.size(); lpc++) {
offset = index[lpc].second; offset = index[lpc].second;
line = lb.read_line(offset, len); ret = lb.read_line(offset, lv);
assert(line != NULL); assert(ret);
assert(offset >= 0); assert(offset >= 0);
assert(offset <= st.st_size); assert(offset <= st.st_size);
assert(memcmp(line, assert(memcmp(lv.lv_start,
&maddr[index[lpc].second], &maddr[index[lpc].second],
len) == 0); lv.lv_len) == 0);
} }
rnd_iters -= 1; rnd_iters -= 1;

26
test/drive_logfile.cc
View File

@ -40,15 +40,17 @@
#include <algorithm> #include <algorithm>
#include "logfile.hh" #include "logfile.hh"
#include "log_format.hh"
#include "log_format_loader.hh"
using namespace std; using namespace std;
typedef enum { typedef enum {
MODE_NONE, MODE_NONE,
MODE_ECHO, MODE_ECHO,
MODE_LINE_COUNT, MODE_LINE_COUNT,
MODE_TIMES, MODE_TIMES,
MODE_LEVELS, MODE_LEVELS,
} dl_mode_t; } dl_mode_t;
time_t time(time_t *_unused) time_t time(time_t *_unused)
@ -62,10 +64,16 @@ int main(int argc, char *argv[])
dl_mode_t mode = MODE_NONE; dl_mode_t mode = MODE_NONE;
string expected_format; string expected_format;
{
std::vector<std::string> paths, errors;
load_formats(paths, errors);
}
while ((c = getopt(argc, argv, "ef:ltv")) != -1) { while ((c = getopt(argc, argv, "ef:ltv")) != -1) {
switch (c) { switch (c) {
case 'f': case 'f':
expected_format = optarg; expected_format = optarg;
break; break;
case 'e': case 'e':
mode = MODE_ECHO; mode = MODE_ECHO;
@ -102,10 +110,12 @@ int main(int argc, char *argv[])
assert(lf.get_format() == NULL); assert(lf.get_format() == NULL);
} }
else { else {
// printf("%s %s\n", lf.get_format()->get_name().c_str(), expected_format.c_str()); //printf("%s %s\n", lf.get_format()->get_name().c_str(), expected_format.c_str());
assert(lf.get_format()->get_name() == expected_format); assert(lf.get_format()->get_name() == expected_format);
} }
assert(lf.get_modified_time() == st.st_mtime); if (!lf.is_compressed()) {
assert(lf.get_modified_time() == st.st_mtime);
}
switch (mode) { switch (mode) {
case MODE_NONE: case MODE_NONE:

8
test/drive_sequencer.cc
View File

@ -61,12 +61,10 @@ public:
bool retval = false; bool retval = false;
try { try {
size_t len; line_value lv;
char *line;
if ((line = this->ms_buffer.read_line(this->ms_offset, if (this->ms_buffer.read_line(this->ms_offset, lv)) {
len)) != NULL) { value_out = string(lv.lv_start, lv.lv_len);
value_out = string(line, len);
retval = true; retval = true;
} }
} }

9
test/slicer.cc
View File

@ -58,12 +58,11 @@ int main(int argc, char *argv[])
int line_number, start, end; int line_number, start, end;
off_t offset = 0; off_t offset = 0;
line_buffer lb; line_buffer lb;
char *line; line_value lv;
size_t len;
lb.set_fd(fd); lb.set_fd(fd);
index.push_back(offset); index.push_back(offset);
while ((line = lb.read_line(offset, len)) != NULL) { while (lb.read_line(offset, lv)) {
index.push_back(offset); index.push_back(offset);
} }
index.pop_back(); index.pop_back();
@ -73,8 +72,8 @@ int main(int argc, char *argv[])
string str; string str;
offset = index[line_number]; offset = index[line_number];
line = lb.read_line(offset, len); lb.read_line(offset, lv);
str = string(line, len); str = string(lv.lv_start, lv.lv_len);
str = str.substr(start, end); str = str.substr(start, end);
printf("%s\n", str.c_str()); printf("%s\n", str.c_str());
} }

15
test/test_line_buffer2.cc
View File

@ -40,24 +40,23 @@
static void single_line(const char *data, int lendiff) static void single_line(const char *data, int lendiff)
{ {
line_buffer lb; line_buffer lb;
line_value lv;
auto_fd pi[2]; auto_fd pi[2];
off_t off = 0; off_t off = 0;
size_t len; bool ret;
char *line;
assert(auto_fd::pipe(pi) == 0); assert(auto_fd::pipe(pi) == 0);
write(pi[1], data, strlen(data)); write(pi[1], data, strlen(data));
pi[1].reset(); pi[1].reset();
lb.set_fd(pi[0]); lb.set_fd(pi[0]);
line = lb.read_line(off, len); ret = lb.read_line(off, lv);
assert(line != NULL); assert(ret);
assert(off == strlen(data)); assert(off == strlen(data));
assert(len == strlen(data) - lendiff); assert(lv.lv_len == strlen(data) - lendiff);
line = lb.read_line(off, len); ret = lb.read_line(off, lv);
assert(line == NULL); assert(!ret);
assert(lb.get_file_size() == strlen(data));
} }
int main(int argc, char *argv[]) int main(int argc, char *argv[])

60
test/test_logfile.sh
View File

@ -21,7 +21,7 @@ run_test ./drive_logfile ${srcdir}/logfile_empty.0
on_error_fail_with "Didn't handle empty log?" on_error_fail_with "Didn't handle empty log?"
touch -t 200711030923 ${srcdir}/logfile_syslog.0
run_test ./drive_logfile -t -f syslog_log ${srcdir}/logfile_syslog.0 run_test ./drive_logfile -t -f syslog_log ${srcdir}/logfile_syslog.0
check_output "Syslog timestamp interpreted incorrectly?" <<EOF check_output "Syslog timestamp interpreted incorrectly?" <<EOF
@ -31,6 +31,7 @@ Nov 03 09:23:38 2007 -- 000
Nov 03 09:47:02 2007 -- 000 Nov 03 09:47:02 2007 -- 000
EOF EOF
touch -t 200711030923 ${srcdir}/logfile_syslog.1
run_test ./drive_logfile -t -f syslog_log ${srcdir}/logfile_syslog.1 run_test ./drive_logfile -t -f syslog_log ${srcdir}/logfile_syslog.1
check_output "Syslog timestamp interpreted incorrectly for year end?" <<EOF check_output "Syslog timestamp interpreted incorrectly for year end?" <<EOF
@ -40,6 +41,31 @@ Dec 03 09:23:38 2006 -- 000
Jan 03 09:47:02 2007 -- 000 Jan 03 09:47:02 2007 -- 000
EOF EOF
gzip -c ${srcdir}/logfile_syslog.1 > logfile_syslog.1.gz
run_test ./drive_logfile -t -f syslog_log logfile_syslog.1.gz
check_output "Syslog timestamp incorrect for gzipped file?" <<EOF
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Jan 03 09:47:02 2007 -- 000
EOF
if test x"$BZIP2_CMD" != x""; then
$BZIP2_CMD -z -c "${srcdir}/logfile_syslog.1" > logfile_syslog.1.bz2
touch -t 200711030923 logfile_syslog.1.bz2
run_test ./drive_logfile -t -f syslog_log logfile_syslog.1.bz2
check_output "bzip2 file not loaded?" <<EOF
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Jan 03 09:47:02 2007 -- 000
EOF
fi
run_test ./drive_logfile -t -f tcsh_history ${srcdir}/logfile_tcsh_history.0 run_test ./drive_logfile -t -f tcsh_history ${srcdir}/logfile_tcsh_history.0
check_output "TCSH timestamp interpreted incorrectly?" <<EOF check_output "TCSH timestamp interpreted incorrectly?" <<EOF
@ -57,18 +83,19 @@ Jul 20 22:59:29 2009 -- 000
Jul 20 22:59:29 2009 -- 000 Jul 20 22:59:29 2009 -- 000
EOF EOF
touch -t 200711030923 ${srcdir}/logfile_strace_log.0
run_test ./drive_logfile -t -f strace_log ${srcdir}/logfile_strace_log.0 run_test ./drive_logfile -t -f strace_log ${srcdir}/logfile_strace_log.0
check_output "strace_log timestamp interpreted incorrectly?" <<EOF check_output "strace_log timestamp interpreted incorrectly?" <<EOF
Dec 31 08:09:33 1979 -- 814 Nov 03 08:09:33 2007 -- 814
Dec 31 08:09:33 1979 -- 815 Nov 03 08:09:33 2007 -- 815
Dec 31 08:09:33 1979 -- 815 Nov 03 08:09:33 2007 -- 815
Dec 31 08:09:33 1979 -- 815 Nov 03 08:09:33 2007 -- 815
Dec 31 08:09:33 1979 -- 816 Nov 03 08:09:33 2007 -- 816
Dec 31 08:09:33 1979 -- 816 Nov 03 08:09:33 2007 -- 816
Dec 31 08:09:33 1979 -- 816 Nov 03 08:09:33 2007 -- 816
Dec 31 08:09:33 1979 -- 816 Nov 03 08:09:33 2007 -- 816
Dec 31 08:09:33 1979 -- 816 Nov 03 08:09:33 2007 -- 816
EOF EOF
## ##
@ -77,18 +104,18 @@ run_test ./drive_logfile -v -f syslog_log ${srcdir}/logfile_syslog.0
check_output "Syslog level interpreted incorrectly?" <<EOF check_output "Syslog level interpreted incorrectly?" <<EOF
0x05 0x05
0x00 0x03
0x05 0x05
0x00 0x03
EOF EOF
run_test ./drive_logfile -v -f tcsh_history ${srcdir}/logfile_tcsh_history.0 run_test ./drive_logfile -v -f tcsh_history ${srcdir}/logfile_tcsh_history.0
check_output "TCSH level interpreted incorrectly?" <<EOF check_output "TCSH level interpreted incorrectly?" <<EOF
0x40 0x03
0xc0 0x83
0x40 0x03
0xc0 0x83
EOF EOF
run_test ./drive_logfile -v -f access_log ${srcdir}/logfile_access_log.0 run_test ./drive_logfile -v -f access_log ${srcdir}/logfile_access_log.0
@ -125,6 +152,7 @@ check_output "generic_log level interpreted incorrectly?" <<EOF
0x02 0x02
EOF EOF
touch -t 200711030923 ${srcdir}/logfile_glog.0
run_test ./drive_logfile -t -f glog_log ${srcdir}/logfile_glog.0 run_test ./drive_logfile -t -f glog_log ${srcdir}/logfile_glog.0
check_output "glog_log timestamp interpreted incorrectly?" <<EOF check_output "glog_log timestamp interpreted incorrectly?" <<EOF