mirror of
https://github.com/tstack/lnav.git
synced 2024-08-16 08:20:29 +03:00
[demux] fixing various bugs
This commit is contained in:
parent
8c7d7c3e82
commit
964e84707b
@ -37,9 +37,6 @@ export TSHARK_CMD
|
||||
LIBARCHIVE_LIBS="@LIBARCHIVE_LIBS@"
|
||||
export LIBARCHIVE_LIBS
|
||||
|
||||
HOME="${top_builddir}/test"
|
||||
export HOME
|
||||
|
||||
# The full path of the test case
|
||||
test_file=$1
|
||||
# The base name of the test case
|
||||
@ -47,6 +44,12 @@ test_file_base=`basename $1`
|
||||
# The current test number for shell based tests.
|
||||
test_num=0
|
||||
|
||||
HOME="${top_builddir}/test/cfg/${test_file_base}"
|
||||
export HOME
|
||||
|
||||
rm -rf "${top_builddir}/test/cfg/${test_file_base}"
|
||||
mkdir -p ${HOME}
|
||||
|
||||
test_hash=""
|
||||
|
||||
lnav_test="${top_builddir}/src/lnav-test"
|
||||
|
@ -40,6 +40,9 @@
|
||||
},
|
||||
"level-thread-class": {
|
||||
"pattern": "(?<timestamp>\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{3}) (?<level>\\w+)\\s+\\[(?<thread>[^\\]]+)\\] (?<class>[^:]+): (?<body>.*)"
|
||||
},
|
||||
"level-class-src": {
|
||||
"pattern": "(?<timestamp>\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{3}) (?<level>\\w+)\\s+(?<class>[\\.\\w]+) \\((?<srcfile>[^:]+):(?<function>[^(]+)\\((?<srcline>\\d+)\\)\\) - (?<body>.*)"
|
||||
}
|
||||
},
|
||||
"level-field": "level",
|
||||
@ -166,6 +169,9 @@
|
||||
},
|
||||
{
|
||||
"line": "2024-04-23 09:17:27,871 INFO [main] server.Server: jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 11.0.21+9-LTS"
|
||||
},
|
||||
{
|
||||
"line": "2024-04-23 09:17:33,457 INFO namenode.NameNode (LogAdapter.java:info(51)) - registered UNIX signal handlers for [TERM, HUP, INT]"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
@ -135,8 +135,7 @@ struct logline_value_meta {
|
||||
logline_value_meta(intern_string_t name,
|
||||
value_kind_t kind,
|
||||
column_t col = external_column{},
|
||||
const std::optional<log_format*>& format
|
||||
= std::nullopt)
|
||||
const std::optional<log_format*>& format = std::nullopt)
|
||||
: lvm_name(name), lvm_kind(kind), lvm_column(col), lvm_format(format)
|
||||
{
|
||||
}
|
||||
@ -329,9 +328,9 @@ struct logline_value_stats {
|
||||
};
|
||||
|
||||
struct logline_value_cmp {
|
||||
explicit logline_value_cmp(
|
||||
const intern_string_t* name = nullptr,
|
||||
std::optional<logline_value_meta::column_t> col = std::nullopt)
|
||||
explicit logline_value_cmp(const intern_string_t* name = nullptr,
|
||||
std::optional<logline_value_meta::column_t> col
|
||||
= std::nullopt)
|
||||
: lvc_name(name), lvc_column(col)
|
||||
{
|
||||
}
|
||||
@ -567,6 +566,7 @@ public:
|
||||
};
|
||||
|
||||
std::string lf_description;
|
||||
log_format* lf_root_format{this};
|
||||
uint8_t lf_mod_index{0};
|
||||
bool lf_multiline{true};
|
||||
bool lf_structured{false};
|
||||
|
@ -134,7 +134,6 @@ logfile::open(ghc::filesystem::path filename,
|
||||
lf->lf_valid_filename = false;
|
||||
}
|
||||
|
||||
lf->lf_content_id = hasher().update(lf->lf_filename).to_string();
|
||||
lf->lf_line_buffer.set_fd(lf_fd);
|
||||
lf->lf_index.reserve(INDEX_RESERVE_INCREMENT);
|
||||
|
||||
@ -186,6 +185,7 @@ logfile::open(ghc::filesystem::path filename,
|
||||
}
|
||||
|
||||
lf->file_options_have_changed();
|
||||
lf->lf_content_id = hasher().update(lf->lf_filename).to_string();
|
||||
|
||||
ensure(lf->invariant());
|
||||
|
||||
@ -319,18 +319,20 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
time_t prescan_time = 0;
|
||||
bool retval = false;
|
||||
|
||||
if (this->lf_format.get() != nullptr) {
|
||||
if (!this->lf_index.empty()) {
|
||||
prescan_time = this->lf_index[prescan_size - 1].get_time();
|
||||
}
|
||||
/* We've locked onto a format, just use that scanner. */
|
||||
found = this->lf_format->scan(*this, this->lf_index, li, sbr, sbc);
|
||||
} else if (this->lf_options.loo_detect_format) {
|
||||
if (this->lf_options.loo_detect_format
|
||||
&& (this->lf_format == nullptr || this->lf_index.size() < 250))
|
||||
{
|
||||
const auto& root_formats = log_format::get_root_formats();
|
||||
std::optional<std::pair<log_format*, log_format::scan_match>>
|
||||
best_match;
|
||||
size_t scan_count = 0;
|
||||
|
||||
if (this->lf_format != nullptr) {
|
||||
best_match = std::make_pair(
|
||||
this->lf_format.get(),
|
||||
log_format::scan_match{this->lf_format_quality});
|
||||
}
|
||||
|
||||
/*
|
||||
* Try each scanner until we get a match. Fortunately, the formats
|
||||
* tend to be sufficiently different that there are few ambiguities...
|
||||
@ -379,22 +381,41 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
scan_count += 1;
|
||||
curr->clear();
|
||||
this->set_format_base_time(curr.get());
|
||||
auto scan_res = curr->scan(*this, this->lf_index, li, sbr, sbc);
|
||||
log_format::scan_result_t scan_res{mapbox::util::no_init{}};
|
||||
if (this->lf_format != nullptr
|
||||
&& this->lf_format->lf_root_format == curr.get())
|
||||
{
|
||||
scan_res = this->lf_format->scan(
|
||||
*this, this->lf_index, li, sbr, sbc);
|
||||
} else {
|
||||
scan_res = curr->scan(*this, this->lf_index, li, sbr, sbc);
|
||||
}
|
||||
|
||||
scan_res.match(
|
||||
[this, &curr, &best_match, &prev_index_size](
|
||||
const log_format::scan_match& sm) {
|
||||
if (!best_match
|
||||
|| sm.sm_quality > best_match->second.sm_quality)
|
||||
[this,
|
||||
&found,
|
||||
&curr,
|
||||
&best_match,
|
||||
&prev_index_size,
|
||||
starting_index_size](const log_format::scan_match& sm) {
|
||||
if (best_match && this->lf_format != nullptr
|
||||
&& this->lf_format->lf_root_format == curr.get())
|
||||
{
|
||||
prev_index_size = this->lf_index.size();
|
||||
found = best_match->second;
|
||||
} else if (!best_match
|
||||
|| sm.sm_quality > best_match->second.sm_quality)
|
||||
{
|
||||
log_info(
|
||||
" scan with format (%s) matched with quality (%d)",
|
||||
curr->get_name().c_str(),
|
||||
sm.sm_quality);
|
||||
if (best_match) {
|
||||
auto last = this->lf_index.begin();
|
||||
std::advance(last, prev_index_size);
|
||||
this->lf_index.erase(this->lf_index.begin(), last);
|
||||
auto starting_iter = std::next(
|
||||
this->lf_index.begin(), starting_index_size);
|
||||
auto last_iter = std::next(this->lf_index.begin(),
|
||||
prev_index_size);
|
||||
this->lf_index.erase(starting_iter, last_iter);
|
||||
}
|
||||
best_match = std::make_pair(curr.get(), sm);
|
||||
prev_index_size = this->lf_index.size();
|
||||
@ -415,10 +436,13 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
"more data required",
|
||||
curr->get_name().c_str());
|
||||
},
|
||||
[curr](const log_format::scan_no_match& snm) {
|
||||
log_trace(" scan with format (%s) does not match -- %s",
|
||||
curr->get_name().c_str(),
|
||||
snm.snm_reason);
|
||||
[this, curr](const log_format::scan_no_match& snm) {
|
||||
if (this->lf_format == nullptr) {
|
||||
log_trace(
|
||||
" scan with format (%s) does not match -- %s",
|
||||
curr->get_name().c_str(),
|
||||
snm.snm_reason);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@ -428,7 +452,12 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
this->lf_options.loo_detect_format = false;
|
||||
}
|
||||
|
||||
if (best_match) {
|
||||
if (best_match
|
||||
&& (this->lf_format == nullptr
|
||||
|| ((this->lf_format->lf_root_format != best_match->first)
|
||||
&& best_match->second.sm_quality
|
||||
> this->lf_format_quality)))
|
||||
{
|
||||
auto winner = best_match.value();
|
||||
auto* curr = winner.first;
|
||||
log_info("%s:%d:log format found -- %s",
|
||||
@ -438,10 +467,14 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
|
||||
this->lf_text_format = text_format_t::TF_LOG;
|
||||
this->lf_format = curr->specialized();
|
||||
this->lf_format_quality = winner.second.sm_quality;
|
||||
this->set_format_base_time(this->lf_format.get());
|
||||
this->lf_content_id
|
||||
= hasher().update(sbr.get_data(), sbr.length()).to_string();
|
||||
if (this->lf_format->lf_date_time.dts_fmt_lock != -1) {
|
||||
this->lf_content_id
|
||||
= hasher().update(sbr.get_data(), sbr.length()).to_string();
|
||||
}
|
||||
|
||||
this->lf_applicable_taggers.clear();
|
||||
for (auto& td_pair : this->lf_format->lf_tag_defs) {
|
||||
bool matches = td_pair.second->ftd_paths.empty();
|
||||
for (const auto& pr : td_pair.second->ftd_paths) {
|
||||
@ -461,6 +494,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
this->lf_applicable_taggers.emplace_back(td_pair.second);
|
||||
}
|
||||
|
||||
this->lf_applicable_partitioners.clear();
|
||||
for (auto& pd_pair : this->lf_format->lf_partition_defs) {
|
||||
bool matches = pd_pair.second->fpd_paths.empty();
|
||||
for (const auto& pr : pd_pair.second->fpd_paths) {
|
||||
@ -502,10 +536,17 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
this->lf_index[lpc].set_millis(last_line.get_millis());
|
||||
this->lf_index[lpc].set_level(LEVEL_INVALID);
|
||||
}
|
||||
retval = true;
|
||||
}
|
||||
|
||||
found = best_match->second;
|
||||
}
|
||||
} else if (this->lf_format.get() != nullptr) {
|
||||
if (!this->lf_index.empty()) {
|
||||
prescan_time = this->lf_index[prescan_size - 1].get_time();
|
||||
}
|
||||
/* We've locked onto a format, just use that scanner. */
|
||||
found = this->lf_format->scan(*this, this->lf_index, li, sbr, sbc);
|
||||
}
|
||||
|
||||
if (found.is<log_format::scan_match>()) {
|
||||
|
@ -327,8 +327,8 @@ public:
|
||||
* indexing.
|
||||
* @return True if any new lines were indexed.
|
||||
*/
|
||||
rebuild_result_t rebuild_index(
|
||||
std::optional<ui_clock::time_point> deadline = std::nullopt);
|
||||
rebuild_result_t rebuild_index(std::optional<ui_clock::time_point> deadline
|
||||
= std::nullopt);
|
||||
|
||||
void reobserve_from(iterator iter);
|
||||
|
||||
@ -410,8 +410,8 @@ public:
|
||||
return this->lf_embedded_metadata;
|
||||
}
|
||||
|
||||
std::optional<std::pair<std::string, lnav::file_options>>
|
||||
get_file_options() const
|
||||
std::optional<std::pair<std::string, lnav::file_options>> get_file_options()
|
||||
const
|
||||
{
|
||||
return this->lf_file_options;
|
||||
}
|
||||
@ -445,6 +445,7 @@ private:
|
||||
std::string lf_content_id;
|
||||
struct stat lf_stat {};
|
||||
std::shared_ptr<log_format> lf_format;
|
||||
uint32_t lf_format_quality{0};
|
||||
std::vector<logline> lf_index;
|
||||
time_t lf_index_time{0};
|
||||
file_off_t lf_index_size{0};
|
||||
@ -479,8 +480,7 @@ private:
|
||||
lf_applicable_partitioners;
|
||||
std::map<std::string, metadata> lf_embedded_metadata;
|
||||
size_t lf_file_options_generation{0};
|
||||
std::optional<std::pair<std::string, lnav::file_options>>
|
||||
lf_file_options;
|
||||
std::optional<std::pair<std::string, lnav::file_options>> lf_file_options;
|
||||
};
|
||||
|
||||
class logline_observer {
|
||||
|
@ -583,6 +583,7 @@ test_remote.sh.log: remote/ssh_host_dsa_key remote/ssh_host_rsa_key remote/id_rs
|
||||
distclean-local:
|
||||
$(RM_V)rm -rf remote remote-tmp not:a:remote:dir
|
||||
$(RM_V)rm -rf sessions
|
||||
$(RM_V)rm -rf cfg
|
||||
$(RM_V)rm -rf tmp
|
||||
$(RM_V)rm -rf piper-tmp
|
||||
$(RM_V)rm -rf rotmp
|
||||
|
@ -1,22 +1,22 @@
|
||||
2013-06-06T19:13:20.123+0000 * Serving Flask app 'app.py'
|
||||
2013-06-06T19:13:20.123+0000 web-1 | [1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
|
||||
2013-06-06T19:13:20.123+0000 * Debug mode: on
|
||||
2013-06-06T19:13:20.123+0000 * Running on all addresses (0.0.0.0)
|
||||
2013-06-06T19:13:20.123+0000 * Running on http://127.0.0.1:5000
|
||||
2013-06-06T19:13:20.123+0000 web-1 | [33mPress CTRL+C to quit[0m
|
||||
2013-06-06T19:13:20.123+0000 * Running on http://172.18.0.3:5000
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:C 25 Apr 2024 05:32:35.018 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
|
||||
2013-06-06T19:13:20.123+0000 * Restarting with stat
|
||||
2013-06-06T19:13:20.123+0000 * Debugger is active!
|
||||
2013-06-06T19:13:20.123+0000 * Debugger PIN: 593-762-075
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:C 25 Apr 2024 05:32:35.018 * Redis version=7.2.0, bits=64, commit=00000000, modified=0, pid=1, just started
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:C 25 Apr 2024 05:32:35.018 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:32:35.019 * monotonic clock: POSIX clock_gettime
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:32:35.019 * Running mode=standalone, port=6379.
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:32:35.020 * Server initialized
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:32:35.022 * Ready to accept connections tcp
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:signal-handler (1714024483) Received SIGTERM scheduling shutdown...
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:54:43.946 * User requested shutdown...
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:54:43.946 * Saving the final RDB snapshot before exiting.
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:54:43.948 * DB saved on disk
|
||||
2013-06-06T19:13:20.123+0000 redis-1 | 1:M 25 Apr 2024 05:54:43.948 # Redis is now ready to exit, bye bye...
|
||||
2013-06-06T19:13:20.123+0000 * Serving Flask app 'app.py'
|
||||
2013-06-06T19:13:20.123+0000 * Debug mode: on
|
||||
2013-06-06T19:13:20.123+0000 [1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
|
||||
2013-06-06T19:13:20.123+0000 * Running on all addresses (0.0.0.0)
|
||||
2013-06-06T19:13:20.123+0000 * Running on http://127.0.0.1:5000
|
||||
2013-06-06T19:13:20.123+0000 * Running on http://172.18.0.3:5000
|
||||
2013-06-06T19:13:20.123+0000 [33mPress CTRL+C to quit[0m
|
||||
2013-06-06T19:13:20.123+0000 * Restarting with stat
|
||||
2013-06-06T19:13:20.123+0000 * Debugger is active!
|
||||
[4m2013-06-06T19:13:20.123+0000 * Debugger PIN: 593-762-075[0m
|
||||
1:C 25 Apr 2024 05:32:35.018 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
|
||||
1:C 25 Apr 2024 05:32:35.018 * Redis version=7.2.0, bits=64, commit=00000000, modified=0, pid=1, just started
|
||||
[33m1[0m[33m:C [0m[33m25 Apr 2024 05:32:35.018[0m[33m # [0m[33mWarning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf[0m
|
||||
1:M 25 Apr 2024 05:32:35.019 * monotonic clock: POSIX clock_gettime
|
||||
1:M 25 Apr 2024 05:32:35.019 * Running mode=standalone, port=6379.
|
||||
1:M 25 Apr 2024 05:32:35.020 * Server initialized
|
||||
1:M 25 Apr 2024 05:32:35.022 * Ready to accept connections tcp
|
||||
1:signal-handler (25 Apr 2024 05:54:43.000) Received SIGTERM scheduling shutdown...
|
||||
1:M 25 Apr 2024 05:54:43.946 * User requested shutdown...
|
||||
1:M 25 Apr 2024 05:54:43.946 * Saving the final RDB snapshot before exiting.
|
||||
1:M 25 Apr 2024 05:54:43.948 * DB saved on disk
|
||||
[33m1[0m[33m:M [0m[33m25 Apr 2024 05:54:43.948[0m[33m # [0m[33mRedis is now ready to exit, bye bye...[0m
|
||||
|
@ -1,3 +1,3 @@
|
||||
2600-01-03 09:23:00 0:
|
||||
2600-12-03 09:23:00 0:
|
||||
00:2 0 00:00:00 0:
|
||||
2600-01-03 09:23:00 0:
|
||||
2600-12-03 09:23:00 0:
|
||||
|
@ -2,7 +2,7 @@
|
||||
til ^
|
||||
wsp ^
|
||||
eof ^
|
||||
eval -- ../test foo
|
||||
eval -- ../test/cfg/test_shlexer.sh foo
|
||||
split:
|
||||
0 ^ -- ../test
|
||||
0 ^ -- ../test/cfg/test_shlexer.sh
|
||||
1 ^-^ -- foo
|
||||
|
@ -1,6 +1,6 @@
|
||||
[1m[31m✘ error[0m: no data was redirected to lnav's standard-input
|
||||
[36m --> [0m[1mcommand-option[0m:1
|
||||
[36m | [0m[37m[40m|rename-stdin foo [0m
|
||||
[36m --> [0m[1m../test/.lnav/formats/default/rename-stdin.lnav[0m:7
|
||||
[36m --> [0m[1m../test/cfg/test_sql.sh/.lnav/formats/default/rename-stdin.lnav[0m:7
|
||||
[36m | [0m[37m[40m;[0m[1m[36m[40mSELECT[0m[37m[40m [0m[1m[37m[40mraise_error[0m[37m[40m([0m[35m[40m'no data was redirected to lnav''s standard-input'[0m[37m[40m) [0m
|
||||
[36m | [0m[37m[40m [0m[1m[36m[40mWHERE[0m[37m[40m ([0m[1m[36m[40mSELECT[0m[37m[40m [0m[1m[37m[40mcount[0m[37m[40m([0m[1m[37m[40m1[0m[37m[40m) [0m[1m[36m[40mFROM[0m[37m[40m [0m[37m[40mlnav_file[0m[37m[40m [0m[1m[36m[40mWHERE[0m[37m[40m [0m[37m[40mfilepath[0m[1m[37m[40m=[0m[35m[40m'stdin'[0m[37m[40m) [0m[1m[37m[40m=[0m[37m[40m [0m[1m[37m[40m0[0m
|
||||
|
@ -1,5 +1,5 @@
|
||||
[1m[31m✘ error[0m: expecting the new name for stdin as the first argument
|
||||
[36m --> [0m[1mcommand-option[0m:1
|
||||
[36m | [0m[37m[40m|rename-stdin [0m
|
||||
[36m --> [0m[1m../test/.lnav/formats/default/rename-stdin.lnav[0m:6
|
||||
[36m --> [0m[1m../test/cfg/test_sql.sh/.lnav/formats/default/rename-stdin.lnav[0m:6
|
||||
[36m | [0m[37m[40m;[0m[1m[36m[40mSELECT[0m[37m[40m [0m[1m[37m[40mraise_error[0m[37m[40m([0m[35m[40m'expecting the new name for stdin as the first argument'[0m[37m[40m) [0m[1m[36m[40mWHERE[0m[37m[40m $[0m[1m[37m[40m1[0m[37m[40m [0m[1m[36m[40mIS[0m[37m[40m [0m[1m[36m[40mNULL[0m
|
||||
|
Loading…
Reference in New Issue
Block a user