mirror of
https://github.com/tstack/lnav.git
synced 2024-09-11 13:05:51 +03:00
[perf] fix some text file loading issues
This commit is contained in:
parent
01031a9fa3
commit
8b3b8a6e3c
@ -503,6 +503,12 @@
|
||||
"json",
|
||||
"csv"
|
||||
]
|
||||
},
|
||||
"max-unrecognized-lines": {
|
||||
"title": "/<format_name>/max-unrecognized-lines",
|
||||
"description": "The maximum number of lines in a file to use when detecting the format",
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
|
@ -269,6 +269,15 @@ date_time_scanner::scan(const char* time_dest,
|
||||
return retval;
|
||||
}
|
||||
|
||||
void
|
||||
date_time_scanner::set_base_time(time_t base_time, const tm& local_tm)
|
||||
{
|
||||
this->dts_base_time = base_time;
|
||||
this->dts_base_tm.et_tm = local_tm;
|
||||
this->dts_last_tm = exttm{};
|
||||
this->dts_last_tv = timeval{};
|
||||
}
|
||||
|
||||
void
|
||||
date_time_scanner::to_localtime(time_t t, exttm& tm_out)
|
||||
{
|
||||
|
@ -68,13 +68,7 @@ struct date_time_scanner {
|
||||
this->dts_fmt_len = -1;
|
||||
}
|
||||
|
||||
void set_base_time(time_t base_time)
|
||||
{
|
||||
this->dts_base_time = base_time;
|
||||
localtime_r(&base_time, &this->dts_base_tm.et_tm);
|
||||
this->dts_last_tm = exttm{};
|
||||
this->dts_last_tv = timeval{};
|
||||
}
|
||||
void set_base_time(time_t base_time, const tm& local_tm);
|
||||
|
||||
/**
|
||||
* Convert a timestamp to local time.
|
||||
|
@ -152,7 +152,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
|
||||
date_time_scanner dts;
|
||||
struct exttm tm;
|
||||
|
||||
dts.set_base_time(format->lf_date_time.dts_base_time);
|
||||
dts.set_base_time(format->lf_date_time.dts_base_time,
|
||||
format->lf_date_time.dts_base_tm.et_tm);
|
||||
if (format->lf_date_time.scan(time_src,
|
||||
time_range.length(),
|
||||
format->get_timestamp_formats(),
|
||||
|
@ -14,6 +14,7 @@
|
||||
"error": "ALERT",
|
||||
"warning": "WARNING"
|
||||
},
|
||||
"max-unrecognized-lines": 15000,
|
||||
"value": {
|
||||
"cpu": {
|
||||
"kind": "integer",
|
||||
|
@ -398,7 +398,6 @@ line_buffer::set_fd(auto_fd& fd)
|
||||
this->lb_seekable = true;
|
||||
}
|
||||
}
|
||||
log_debug("newoff %d", newoff);
|
||||
this->lb_file_offset = newoff;
|
||||
this->lb_buffer.clear();
|
||||
this->lb_fd = std::move(fd);
|
||||
@ -839,12 +838,6 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
|
||||
#endif
|
||||
else if (this->lb_seekable)
|
||||
{
|
||||
#if 1
|
||||
log_debug("doing pread file_off=%d read_off=%d count=%d",
|
||||
this->lb_file_offset,
|
||||
this->lb_file_offset + this->lb_buffer.size(),
|
||||
this->lb_buffer.available());
|
||||
#endif
|
||||
rc = pread(this->lb_fd,
|
||||
this->lb_buffer.end(),
|
||||
this->lb_buffer.available(),
|
||||
|
@ -196,7 +196,6 @@ listview_curses::do_update()
|
||||
attr_line_t overlay_line;
|
||||
struct line_range lr;
|
||||
unsigned long width, wrap_width;
|
||||
size_t row_count;
|
||||
int y = this->lv_y, bottom;
|
||||
attr_t role_attrs = vc.attrs_for_role(this->vc_default_role);
|
||||
|
||||
@ -211,7 +210,7 @@ listview_curses::do_update()
|
||||
: this->lv_show_scrollbar ? 1
|
||||
: 0);
|
||||
|
||||
row_count = this->get_inner_height();
|
||||
size_t row_count = this->get_inner_height();
|
||||
row = this->lv_top;
|
||||
bottom = y + height;
|
||||
std::vector<attr_line_t> rows(
|
||||
@ -484,13 +483,17 @@ listview_curses::set_top(vis_line_t top, bool suppress_flash)
|
||||
if (this->lv_selectable) {
|
||||
if (this->lv_selection < 0_vl) {
|
||||
} else if (this->lv_selection < top) {
|
||||
this->lv_selection = top;
|
||||
this->set_selection(top);
|
||||
} else {
|
||||
auto bot = this->get_bottom();
|
||||
unsigned long width;
|
||||
vis_line_t height;
|
||||
|
||||
if (bot != -1_vl) {
|
||||
this->get_dimensions(height, width);
|
||||
|
||||
if (bot != -1_vl && (bot - top) >= (height - 1)) {
|
||||
if (this->lv_selection > (bot - this->lv_tail_space)) {
|
||||
this->lv_selection = bot - this->lv_tail_space;
|
||||
this->set_selection(bot - this->lv_tail_space);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -567,7 +570,8 @@ listview_curses::scroll_selection_into_view()
|
||||
if (this->lv_selection < 0_vl) {
|
||||
this->set_top(0_vl);
|
||||
} else if (this->lv_selection
|
||||
>= (this->lv_top + height - this->lv_tail_space)) {
|
||||
>= (this->lv_top + height - this->lv_tail_space - 1_vl))
|
||||
{
|
||||
this->set_top(this->lv_selection - height + 1_vl + this->lv_tail_space,
|
||||
true);
|
||||
} else if (this->lv_selection < this->lv_top) {
|
||||
|
@ -458,6 +458,9 @@ public:
|
||||
if (this->lv_height < 0) {
|
||||
height_out = vis_line_t(height) + this->lv_height
|
||||
- vis_line_t(this->lv_y);
|
||||
if (height_out < 0_vl) {
|
||||
height_out = 0_vl;
|
||||
}
|
||||
} else {
|
||||
height_out = this->lv_height;
|
||||
}
|
||||
|
16
src/lnav.cc
16
src/lnav.cc
@ -2186,7 +2186,6 @@ main(int argc, char* argv[])
|
||||
"missing format files to install")
|
||||
.with_reason(install_reason)
|
||||
.with_help(install_help));
|
||||
usage();
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
@ -2771,19 +2770,24 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
|
||||
}
|
||||
}
|
||||
|
||||
if (lnav_data.ld_active_files.fc_file_names.empty()
|
||||
if (retval == EXIT_SUCCESS
|
||||
&& lnav_data.ld_active_files.fc_file_names.empty()
|
||||
&& lnav_data.ld_commands.empty() && lnav_data.ld_pt_search.empty()
|
||||
&& !(lnav_data.ld_show_help_view || mode_flags.mf_no_default))
|
||||
{
|
||||
lnav::console::print(
|
||||
stderr,
|
||||
lnav::console::user_message::error("no log files given/found"));
|
||||
lnav::console::user_message::error("nothing to do")
|
||||
.with_reason("no files given or default files found")
|
||||
.with_help(
|
||||
attr_line_t("use the ")
|
||||
.append_quoted(lnav::roles::keyword("-N"))
|
||||
.append(
|
||||
" option to open lnav without loading any files")));
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
|
||||
if (retval != EXIT_SUCCESS) {
|
||||
usage();
|
||||
} else {
|
||||
if (retval == EXIT_SUCCESS) {
|
||||
isc::supervisor root_superv(injector::get<isc::service_list>());
|
||||
|
||||
try {
|
||||
|
@ -229,17 +229,19 @@ com_adjust_log_time(exec_context& ec,
|
||||
date_time_scanner dts;
|
||||
vis_line_t top_line;
|
||||
struct exttm tm;
|
||||
struct tm base_tm;
|
||||
std::shared_ptr<logfile> lf;
|
||||
|
||||
top_line = lnav_data.ld_views[LNV_LOG].get_selection();
|
||||
top_content = lss.at(top_line);
|
||||
lf = lss.find(top_content);
|
||||
|
||||
logline& ll = (*lf)[top_content];
|
||||
auto& ll = (*lf)[top_content];
|
||||
|
||||
top_time = ll.get_timeval();
|
||||
localtime_r(&top_time.tv_sec, &base_tm);
|
||||
|
||||
dts.set_base_time(top_time.tv_sec);
|
||||
dts.set_base_time(top_time.tv_sec, base_tm);
|
||||
args[1] = remaining_args(cmdline, args);
|
||||
|
||||
auto parse_res = relative_time::from_str(args[1]);
|
||||
@ -3279,11 +3281,13 @@ com_pt_time(exec_context& ec,
|
||||
date_time_scanner dts;
|
||||
struct exttm tm;
|
||||
time_t now;
|
||||
struct tm base_tm;
|
||||
auto parse_res = relative_time::from_str(all_args);
|
||||
|
||||
time(&now);
|
||||
localtime_r(&now, &base_tm);
|
||||
dts.dts_keep_base_tz = true;
|
||||
dts.set_base_time(now);
|
||||
dts.set_base_time(now, base_tm);
|
||||
if (parse_res.isOk()) {
|
||||
tm.et_tm = *gmtime(&now);
|
||||
tm = parse_res.unwrap().adjust(tm);
|
||||
|
@ -489,14 +489,16 @@ public:
|
||||
bool lf_is_self_describing{false};
|
||||
bool lf_time_ordered{true};
|
||||
bool lf_specialized{false};
|
||||
nonstd::optional<int64_t> lf_max_unrecognized_lines;
|
||||
|
||||
protected:
|
||||
static std::vector<std::shared_ptr<log_format>> lf_root_formats;
|
||||
|
||||
struct pcre_format {
|
||||
pcre_format(const char* regex) : name(regex), pcre(regex)
|
||||
explicit pcre_format(const char* regex)
|
||||
: name(regex), pcre(regex),
|
||||
pf_timestamp_index(this->pcre.name_index("timestamp"))
|
||||
{
|
||||
this->pf_timestamp_index = this->pcre.name_index("timestamp");
|
||||
}
|
||||
|
||||
pcre_format() : name(nullptr), pcre("") {}
|
||||
|
@ -1070,8 +1070,10 @@ public:
|
||||
nullptr,
|
||||
&tm,
|
||||
tv)) {
|
||||
this->lf_date_time.set_base_time(tv.tv_sec);
|
||||
this->wlf_time_scanner.set_base_time(tv.tv_sec);
|
||||
this->lf_date_time.set_base_time(tv.tv_sec,
|
||||
tm.et_tm);
|
||||
this->wlf_time_scanner.set_base_time(tv.tv_sec,
|
||||
tm.et_tm);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1210,8 +1212,8 @@ public:
|
||||
&tm,
|
||||
tv))
|
||||
{
|
||||
this->lf_date_time.set_base_time(tv.tv_sec);
|
||||
this->wlf_time_scanner.set_base_time(tv.tv_sec);
|
||||
this->lf_date_time.set_base_time(tv.tv_sec, tm.et_tm);
|
||||
this->wlf_time_scanner.set_base_time(tv.tv_sec, tm.et_tm);
|
||||
}
|
||||
} else if (directive == "#Fields:" && this->wlf_field_defs.empty())
|
||||
{
|
||||
|
@ -869,6 +869,13 @@ struct json_path_container format_handlers = {
|
||||
.with_description("The type of file that contains the log messages")
|
||||
.with_enum_values(TYPE_ENUM)
|
||||
.for_field(&external_log_format::elf_type),
|
||||
|
||||
yajlpp::property_handler("max-unrecognized-lines")
|
||||
.with_synopsis("<lines>")
|
||||
.with_description("The maximum number of lines in a file to use when "
|
||||
"detecting the format")
|
||||
.with_min_value(1)
|
||||
.for_field(&log_format::lf_max_unrecognized_lines),
|
||||
};
|
||||
|
||||
static int
|
||||
|
@ -172,7 +172,17 @@ logfile::set_format_base_time(log_format* lf)
|
||||
if (file_time == 0) {
|
||||
file_time = this->lf_stat.st_mtime;
|
||||
}
|
||||
lf->lf_date_time.set_base_time(file_time);
|
||||
|
||||
if (!this->lf_cached_base_time
|
||||
|| this->lf_cached_base_time.value() != file_time)
|
||||
{
|
||||
struct tm new_base_tm;
|
||||
this->lf_cached_base_time = file_time;
|
||||
localtime_r(&file_time, &new_base_tm);
|
||||
this->lf_cached_base_tm = new_base_tm;
|
||||
}
|
||||
lf->lf_date_time.set_base_time(this->lf_cached_base_time.value(),
|
||||
this->lf_cached_base_tm.value());
|
||||
}
|
||||
|
||||
bool
|
||||
@ -180,6 +190,10 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
const line_info& li,
|
||||
scan_batch_context& sbc)
|
||||
{
|
||||
static auto max_unrecognized_lines
|
||||
= injector::get<const lnav::logfile::config&>()
|
||||
.lc_max_unrecognized_lines;
|
||||
|
||||
log_format::scan_result_t found = log_format::SCAN_NO_MATCH;
|
||||
size_t prescan_size = this->lf_index.size();
|
||||
time_t prescan_time = 0;
|
||||
@ -191,11 +205,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
}
|
||||
/* We've locked onto a format, just use that scanner. */
|
||||
found = this->lf_format->scan(*this, this->lf_index, li, sbr, sbc);
|
||||
} else if (this->lf_options.loo_detect_format
|
||||
&& this->lf_index.size()
|
||||
< injector::get<const lnav::logfile::config&>()
|
||||
.lc_max_unrecognized_lines)
|
||||
{
|
||||
} else if (this->lf_options.loo_detect_format) {
|
||||
const auto& root_formats = log_format::get_root_formats();
|
||||
|
||||
/*
|
||||
@ -206,6 +216,13 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
iter != root_formats.end() && (found != log_format::SCAN_MATCH);
|
||||
++iter)
|
||||
{
|
||||
if (this->lf_index.size()
|
||||
>= (*iter)->lf_max_unrecognized_lines.value_or(
|
||||
max_unrecognized_lines))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(*iter)->match_name(this->lf_filename)) {
|
||||
if (li.li_file_range.fr_offset == 0) {
|
||||
log_debug("(%s) does not match file name: %s",
|
||||
|
@ -36,7 +36,7 @@ namespace lnav {
|
||||
namespace logfile {
|
||||
|
||||
struct config {
|
||||
uint64_t lc_max_unrecognized_lines{15000};
|
||||
uint64_t lc_max_unrecognized_lines{1000};
|
||||
};
|
||||
|
||||
} // namespace logfile
|
||||
|
@ -420,6 +420,8 @@ private:
|
||||
safe_opid_map lf_opids;
|
||||
size_t lf_watch_count{0};
|
||||
ArenaAlloc::Alloc<char> lf_allocator{64 * 1024};
|
||||
nonstd::optional<time_t> lf_cached_base_time;
|
||||
nonstd::optional<tm> lf_cached_base_tm;
|
||||
|
||||
nonstd::optional<std::pair<file_off_t, size_t>> lf_next_line_cache;
|
||||
};
|
||||
|
@ -225,6 +225,10 @@ spectrogram_source::list_value_for_overlay(const listview_curses& lv,
|
||||
auto mark_offset = this->ss_cursor_column.value();
|
||||
auto mark_is_before = true;
|
||||
|
||||
if (desc.length() + 8 > width) {
|
||||
desc.clear();
|
||||
}
|
||||
|
||||
if (this->ss_cursor_column.value() + desc.length() + 1 > width) {
|
||||
mark_offset -= desc.length();
|
||||
mark_is_before = false;
|
||||
@ -287,16 +291,26 @@ spectrogram_source::list_value_for_overlay(const listview_curses& lv,
|
||||
st.st_yellow_threshold - 1,
|
||||
role_t::VCR_HIGH_THRESHOLD,
|
||||
st.st_yellow_threshold);
|
||||
line.append(width / 2 - strlen(buf) / 3 - line.length(), ' ');
|
||||
auto buflen = strlen(buf);
|
||||
if (line.length() + buflen + 20 < width) {
|
||||
line.append(width / 2 - buflen / 3 - line.length(), ' ');
|
||||
} else {
|
||||
line.append(" ");
|
||||
}
|
||||
line.append(buf);
|
||||
scrub_ansi_string(line, value_out.get_attrs());
|
||||
|
||||
snprintf(buf, sizeof(buf), "Max: %'.10lg", sb.sb_max_value_out);
|
||||
line.append(width - strlen(buf) - line.length() - 2, ' ');
|
||||
buflen = strlen(buf);
|
||||
if (line.length() + buflen + 4 < width) {
|
||||
line.append(width - buflen - line.length() - 2, ' ');
|
||||
} else {
|
||||
line.append(" ");
|
||||
}
|
||||
line.append(buf);
|
||||
|
||||
value_out.with_attr(string_attr(line_range(0, -1),
|
||||
VC_STYLE.value(A_UNDERLINE)));
|
||||
value_out.with_attr(
|
||||
string_attr(line_range(0, -1), VC_STYLE.value(A_UNDERLINE)));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
@ -279,9 +279,6 @@ textfile_sub_source::text_crumbs_for_line(
|
||||
}
|
||||
|
||||
auto lf = this->current_file();
|
||||
if (lf->size() == 0) {
|
||||
return;
|
||||
}
|
||||
crumbs.emplace_back(
|
||||
lf->get_unique_path(),
|
||||
attr_line_t().append(lnav::roles::identifier(lf->get_unique_path())),
|
||||
@ -308,6 +305,9 @@ textfile_sub_source::text_crumbs_for_line(
|
||||
this->to_front(lf_opt.value());
|
||||
this->tss_view->reload_data();
|
||||
});
|
||||
if (lf->size() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto rend_iter = this->tss_rendered_files.find(lf->get_filename());
|
||||
if (rend_iter != this->tss_rendered_files.end()) {
|
||||
|
@ -488,6 +488,16 @@ public:
|
||||
return yajl_gen_integer(this->yg_handle, value);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
yajl_gen_status operator()(nonstd::optional<T> value)
|
||||
{
|
||||
if (!value.has_value()) {
|
||||
return yajl_gen_status_ok;
|
||||
}
|
||||
|
||||
return (*this)(value.value());
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
yajl_gen_status operator()(
|
||||
const T& container,
|
||||
|
@ -570,6 +570,12 @@ struct json_path_handler : public json_path_handler_base {
|
||||
= std::is_integral<U>::value && !std::is_same<U, bool>::value;
|
||||
};
|
||||
|
||||
template<typename T, typename U>
|
||||
struct LastIsNumber<nonstd::optional<U> T::*> {
|
||||
static constexpr bool value
|
||||
= std::is_integral<U>::value && !std::is_same<U, bool>::value;
|
||||
};
|
||||
|
||||
template<typename T, typename... Args>
|
||||
struct LastIsVector {
|
||||
static constexpr bool value = LastIsVector<Args...>::value;
|
||||
@ -585,6 +591,18 @@ struct json_path_handler : public json_path_handler_base {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
static bool is_field_set(const nonstd::optional<T>& field)
|
||||
{
|
||||
return field.has_value();
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static bool is_field_set(const T&)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
template<typename... Args,
|
||||
std::enable_if_t<LastIs<bool, Args...>::value, bool> = true>
|
||||
json_path_handler& for_field(Args... args)
|
||||
@ -1153,6 +1171,10 @@ struct json_path_handler : public json_path_handler_base {
|
||||
}
|
||||
}
|
||||
|
||||
if (!is_field_set(field)) {
|
||||
return yajl_gen_status_ok;
|
||||
}
|
||||
|
||||
if (ygc.ygc_depth) {
|
||||
yajl_gen_string(handle, jph.jph_property);
|
||||
}
|
||||
|
@ -230,7 +230,6 @@ mv test_logfile.unreadable.out `test_err_filename`
|
||||
check_error_output "able to read an unreadable log file?" <<EOF
|
||||
✘ error: cannot read file: unreadable.log
|
||||
reason: Permission denied
|
||||
✘ error: no log files given/found
|
||||
EOF
|
||||
|
||||
run_test ${lnav_test} -n 'unreadable.*'
|
||||
|
Loading…
Reference in New Issue
Block a user