mirror of
https://github.com/tstack/lnav.git
synced 2024-08-16 16:30:25 +03:00
[format] move foreign_key flag to value meta
Fix some chart issues as well.
This commit is contained in:
parent
e805ca8a8a
commit
f69878eaf5
1
NEWS.md
1
NEWS.md
@ -101,6 +101,7 @@ Bug Fixes:
|
||||
* A crash during initialization on Apple Silicon and MacOS 12
|
||||
has been fixed.
|
||||
* A crash when previewing non-text files.
|
||||
* Optimized ANSI-escape processing.
|
||||
* Various fixes to make lnav usable as a `PAGER`.
|
||||
|
||||
## lnav v0.12.1
|
||||
|
@ -286,6 +286,7 @@ db_label_source::push_column(const scoped_value_t& sv)
|
||||
}
|
||||
}
|
||||
}
|
||||
hm.hm_chart.next_row();
|
||||
}
|
||||
|
||||
void
|
||||
|
@ -359,6 +359,7 @@ files_sub_source::text_value_for_line(textview_curses& tc,
|
||||
.append(start_time)
|
||||
.append(" \u2014 ")
|
||||
.append(end_time)
|
||||
.append(" ")
|
||||
.appendf(FMT_STRING("{}"), fmt::join(file_notes, "; "));
|
||||
if (selected) {
|
||||
al.with_attr_for_all(VC_ROLE.value(role_t::VCR_FOCUSED));
|
||||
|
@ -67,9 +67,13 @@ hist_source2::text_value_for_line(textview_curses& tc,
|
||||
std::string& value_out,
|
||||
text_sub_source::line_flags_t flags)
|
||||
{
|
||||
bucket_t& bucket = this->find_bucket(row);
|
||||
auto& bucket = this->find_bucket(row);
|
||||
struct tm bucket_tm;
|
||||
|
||||
if (this->hs_needs_flush) {
|
||||
this->end_of_row();
|
||||
}
|
||||
|
||||
value_out.clear();
|
||||
if (gmtime_r(&bucket.b_time, &bucket_tm) != nullptr) {
|
||||
fmt::format_to(std::back_inserter(value_out),
|
||||
@ -130,12 +134,14 @@ hist_source2::add_value(time_t row,
|
||||
auto& bucket = this->find_bucket(this->hs_last_bucket);
|
||||
bucket.b_time = row;
|
||||
bucket.b_values[htype].hv_value += value;
|
||||
|
||||
this->hs_needs_flush = true;
|
||||
}
|
||||
|
||||
void
|
||||
hist_source2::init()
|
||||
{
|
||||
view_colors& vc = view_colors::singleton();
|
||||
auto& vc = view_colors::singleton();
|
||||
|
||||
this->hs_chart.with_show_state(stacked_bar_chart_base::show_all{})
|
||||
.with_attrs_for_ident(HT_NORMAL, vc.attrs_for_role(role_t::VCR_TEXT))
|
||||
@ -160,12 +166,13 @@ void
|
||||
hist_source2::end_of_row()
|
||||
{
|
||||
if (this->hs_last_bucket >= 0) {
|
||||
bucket_t& last_bucket = this->find_bucket(this->hs_last_bucket);
|
||||
auto& last_bucket = this->find_bucket(this->hs_last_bucket);
|
||||
|
||||
for (int lpc = 0; lpc < HT__MAX; lpc++) {
|
||||
this->hs_chart.add_value((const hist_type_t) lpc,
|
||||
last_bucket.b_values[lpc].hv_value);
|
||||
}
|
||||
this->hs_chart.next_row();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -204,18 +204,23 @@ public:
|
||||
if (this->sbc_show_state.template is<show_all>()
|
||||
|| lpc == (size_t) ident_to_show)
|
||||
{
|
||||
overall_stats.merge(this->sbc_idents[lpc].ci_stats,
|
||||
this->sbc_do_stacking);
|
||||
overall_stats.merge(this->sbc_idents[lpc].ci_stats);
|
||||
}
|
||||
}
|
||||
if (this->sbc_max_row_value > overall_stats.bs_max_value) {
|
||||
overall_stats.bs_max_value = this->sbc_max_row_value;
|
||||
}
|
||||
if (this->sbc_row_sum > overall_stats.bs_max_value) {
|
||||
overall_stats.bs_max_value = this->sbc_row_sum;
|
||||
}
|
||||
|
||||
if (this->sbc_show_state.template is<show_all>()) {
|
||||
if (this->sbc_idents.size() == 1) {
|
||||
avail_width = width;
|
||||
} else if (width < this->sbc_idents.size()) {
|
||||
} else if (width < this->sbc_max_row_items) {
|
||||
avail_width = 0;
|
||||
} else {
|
||||
avail_width = width - this->sbc_idents.size();
|
||||
avail_width = width;
|
||||
}
|
||||
} else {
|
||||
avail_width = width - 1;
|
||||
@ -256,12 +261,34 @@ public:
|
||||
this->sbc_idents.clear();
|
||||
this->sbc_ident_lookup.clear();
|
||||
this->sbc_show_state = show_none();
|
||||
this->sbc_row_sum = 0;
|
||||
this->sbc_row_items = 0;
|
||||
this->sbc_max_row_value = 0;
|
||||
this->sbc_max_row_items = 0;
|
||||
}
|
||||
|
||||
void add_value(const T& ident, double amount = 1.0)
|
||||
{
|
||||
struct chart_ident& ci = this->find_ident(ident);
|
||||
ci.ci_stats.update(amount);
|
||||
this->sbc_row_sum += amount;
|
||||
if (ci.ci_last_seen_row != this->sbc_row_counter) {
|
||||
ci.ci_last_seen_row = this->sbc_row_counter;
|
||||
this->sbc_row_items += 1;
|
||||
}
|
||||
}
|
||||
|
||||
void next_row()
|
||||
{
|
||||
if (this->sbc_row_sum > this->sbc_max_row_value) {
|
||||
this->sbc_max_row_value = this->sbc_row_sum;
|
||||
}
|
||||
if (this->sbc_row_items > this->sbc_max_row_items) {
|
||||
this->sbc_max_row_items = this->sbc_row_items;
|
||||
}
|
||||
this->sbc_row_sum = 0;
|
||||
this->sbc_row_items = 0;
|
||||
this->sbc_row_counter += 1;
|
||||
}
|
||||
|
||||
struct bucket_stats_t {
|
||||
@ -270,15 +297,10 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
void merge(const bucket_stats_t& rhs, bool do_stacking)
|
||||
void merge(const bucket_stats_t& rhs)
|
||||
{
|
||||
this->bs_min_value = std::min(this->bs_min_value, rhs.bs_min_value);
|
||||
if (do_stacking) {
|
||||
this->bs_max_value += rhs.bs_max_value;
|
||||
} else {
|
||||
this->bs_max_value
|
||||
= std::max(this->bs_max_value, rhs.bs_max_value);
|
||||
}
|
||||
this->bs_max_value = std::max(this->bs_max_value, rhs.bs_max_value);
|
||||
}
|
||||
|
||||
double width() const
|
||||
@ -310,6 +332,7 @@ protected:
|
||||
T ci_ident;
|
||||
text_attrs ci_attrs;
|
||||
bucket_stats_t ci_stats;
|
||||
ssize_t ci_last_seen_row{-1};
|
||||
};
|
||||
|
||||
struct chart_ident& find_ident(const T& ident)
|
||||
@ -328,6 +351,12 @@ protected:
|
||||
std::vector<struct chart_ident> sbc_idents;
|
||||
std::unordered_map<T, unsigned int> sbc_ident_lookup;
|
||||
show_state sbc_show_state{show_none()};
|
||||
|
||||
ssize_t sbc_row_counter{0};
|
||||
double sbc_row_sum{0};
|
||||
size_t sbc_row_items{0};
|
||||
double sbc_max_row_value{0};
|
||||
size_t sbc_max_row_items{0};
|
||||
};
|
||||
|
||||
class hist_source2
|
||||
@ -417,6 +446,7 @@ private:
|
||||
time_t hs_last_row;
|
||||
std::map<int64_t, struct bucket_block> hs_blocks;
|
||||
stacked_bar_chart<hist_type_t> hs_chart;
|
||||
bool hs_needs_flush{false};
|
||||
};
|
||||
|
||||
#endif
|
||||
|
@ -267,6 +267,8 @@ listview_curses::handle_key(int ch)
|
||||
if (this->lv_selection <= top_for_last) {
|
||||
this->set_selection(top_for_last + 1_vl);
|
||||
}
|
||||
} else if (this->lv_top > top_for_last) {
|
||||
this->set_top(top_for_last);
|
||||
} else {
|
||||
this->shift_top(rows_avail);
|
||||
|
||||
|
@ -245,7 +245,7 @@ logline_value_meta::to_chart_type() const
|
||||
retval = chart_type_t::none;
|
||||
break;
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
if (!this->lvm_identifier) {
|
||||
if (!this->lvm_identifier && !this->lvm_foreign_key) {
|
||||
retval = chart_type_t::spectro;
|
||||
}
|
||||
break;
|
||||
@ -2843,7 +2843,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
|
||||
auto& ivd = pat.p_value_by_index[lpc];
|
||||
auto vd = ivd.ivd_value_def;
|
||||
|
||||
if (!vd->vd_foreign_key && !vd->vd_meta.lvm_identifier) {
|
||||
if (!vd->vd_meta.lvm_foreign_key && !vd->vd_meta.lvm_identifier) {
|
||||
switch (vd->vd_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
@ -3606,7 +3606,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
|
||||
elf_value_def->vd_meta.lvm_values_index
|
||||
= nonstd::make_optional(value_def_index++);
|
||||
|
||||
if (elf_value_def->vd_foreign_key
|
||||
if (elf_value_def->vd_meta.lvm_foreign_key
|
||||
|| elf_value_def->vd_meta.lvm_identifier)
|
||||
{
|
||||
continue;
|
||||
@ -3856,7 +3856,7 @@ public:
|
||||
log_vtab_impl::get_foreign_keys(keys_inout);
|
||||
|
||||
for (const auto& elf_value_def : this->elt_format.elf_value_defs) {
|
||||
if (elf_value_def.second->vd_foreign_key) {
|
||||
if (elf_value_def.second->vd_meta.lvm_foreign_key) {
|
||||
keys_inout.emplace_back(elf_value_def.first.to_string());
|
||||
}
|
||||
}
|
||||
|
@ -163,6 +163,7 @@ struct logline_value_meta {
|
||||
column_t lvm_column{external_column{}};
|
||||
nonstd::optional<size_t> lvm_values_index;
|
||||
bool lvm_identifier{false};
|
||||
bool lvm_foreign_key{false};
|
||||
bool lvm_hidden{false};
|
||||
nonstd::optional<bool> lvm_user_hidden;
|
||||
bool lvm_from_module{false};
|
||||
|
@ -68,7 +68,6 @@ public:
|
||||
|
||||
logline_value_meta vd_meta;
|
||||
std::string vd_collate;
|
||||
bool vd_foreign_key{false};
|
||||
intern_string_t vd_unit_field;
|
||||
std::map<const intern_string_t, scaling_factor> vd_unit_scaling;
|
||||
bool vd_internal{false};
|
||||
|
@ -489,10 +489,12 @@ public:
|
||||
|
||||
field_def& with_kind(value_kind_t kind,
|
||||
bool identifier = false,
|
||||
bool foreign_key = false,
|
||||
const std::string& collator = "")
|
||||
{
|
||||
this->fd_meta.lvm_kind = kind;
|
||||
this->fd_meta.lvm_identifier = identifier;
|
||||
this->fd_meta.lvm_foreign_key = foreign_key;
|
||||
this->fd_collator = collator;
|
||||
return *this;
|
||||
}
|
||||
@ -768,12 +770,14 @@ public:
|
||||
"bro_referrer",
|
||||
"bro_resp_fuids",
|
||||
"bro_service",
|
||||
"bro_status_code",
|
||||
"bro_uid",
|
||||
"bro_uri",
|
||||
"bro_user_agent",
|
||||
"bro_username",
|
||||
};
|
||||
static const char* KNOWN_FOREIGN[] = {
|
||||
"bro_status_code",
|
||||
};
|
||||
|
||||
int numeric_count = 0;
|
||||
|
||||
@ -792,7 +796,12 @@ public:
|
||||
bool ident = std::binary_search(std::begin(KNOWN_IDS),
|
||||
std::end(KNOWN_IDS),
|
||||
fd.fd_meta.lvm_name);
|
||||
fd.with_kind(value_kind_t::VALUE_INTEGER, ident)
|
||||
bool foreign
|
||||
= std::binary_search(std::begin(KNOWN_FOREIGN),
|
||||
std::end(KNOWN_FOREIGN),
|
||||
fd.fd_meta.lvm_name);
|
||||
fd.with_kind(
|
||||
value_kind_t::VALUE_INTEGER, ident, foreign)
|
||||
.with_numeric_index(numeric_count);
|
||||
numeric_count += 1;
|
||||
} else if (field_type == "bool") {
|
||||
@ -950,7 +959,7 @@ public:
|
||||
this->log_vtab_impl::get_foreign_keys(keys_inout);
|
||||
|
||||
for (const auto& fd : this->blt_format.blf_field_defs) {
|
||||
if (fd.fd_meta.lvm_identifier) {
|
||||
if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
|
||||
keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
|
||||
}
|
||||
}
|
||||
@ -1124,6 +1133,7 @@ public:
|
||||
const char* name,
|
||||
value_kind_t kind,
|
||||
bool ident = false,
|
||||
bool foreign_key = false,
|
||||
std::string coll = "")
|
||||
: fd_name(intern_string::lookup(name)),
|
||||
fd_meta(
|
||||
@ -1133,6 +1143,7 @@ public:
|
||||
fd_collator(std::move(coll))
|
||||
{
|
||||
this->fd_meta.lvm_identifier = ident;
|
||||
this->fd_meta.lvm_foreign_key = foreign_key;
|
||||
}
|
||||
|
||||
field_def& with_kind(value_kind_t kind,
|
||||
@ -1625,7 +1636,7 @@ public:
|
||||
this->log_vtab_impl::get_foreign_keys(keys_inout);
|
||||
|
||||
for (const auto& fd : KNOWN_FIELDS) {
|
||||
if (fd.fd_meta.lvm_identifier) {
|
||||
if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
|
||||
keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
|
||||
}
|
||||
}
|
||||
@ -1687,6 +1698,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
|
||||
"c-ip",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
false,
|
||||
"ipaddress",
|
||||
},
|
||||
{
|
||||
@ -1706,6 +1718,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
|
||||
"cs-uri-stem",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
false,
|
||||
"naturalnocase",
|
||||
},
|
||||
{
|
||||
@ -1731,6 +1744,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
|
||||
"s-ip",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
false,
|
||||
"ipaddress",
|
||||
},
|
||||
{
|
||||
@ -1762,6 +1776,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
|
||||
"sc-status",
|
||||
value_kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
true,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
|
@ -456,7 +456,8 @@ static const json_path_handler_base::enum_value_t OVERFLOW_ENUM[] = {
|
||||
{"truncate",
|
||||
external_log_format::json_format_element::overflow_t::TRUNCATE},
|
||||
{"dot-dot", external_log_format::json_format_element::overflow_t::DOTDOT},
|
||||
{"last-word", external_log_format::json_format_element::overflow_t::LASTWORD},
|
||||
{"last-word",
|
||||
external_log_format::json_format_element::overflow_t::LASTWORD},
|
||||
|
||||
json_path_handler_base::ENUM_TERMINATOR,
|
||||
};
|
||||
@ -616,7 +617,8 @@ static const struct json_path_container value_def_handlers = {
|
||||
.with_synopsis("<bool>")
|
||||
.with_description("Indicates whether or not this field should be "
|
||||
"treated as a foreign key for row in another table")
|
||||
.for_field(&external_log_format::value_def::vd_foreign_key),
|
||||
.for_field(&external_log_format::value_def::vd_meta,
|
||||
&logline_value_meta::lvm_foreign_key),
|
||||
|
||||
yajlpp::property_handler("hidden")
|
||||
.with_synopsis("<bool>")
|
||||
|
@ -26,8 +26,23 @@ let hist = func column slice:'1h' top:10 rel <relation> -> (
|
||||
group { tslice } (
|
||||
window (
|
||||
sort {-total}
|
||||
derive {rn = row_number column}
|
||||
)
|
||||
)
|
||||
derive top_value = case [
|
||||
rn < top => column,
|
||||
rn >= top => 'Other',
|
||||
]
|
||||
group { tslice, top_value } (
|
||||
aggregate { total2 = sum total }
|
||||
)
|
||||
group { tslice } (
|
||||
window (
|
||||
sort {-total2}
|
||||
# XXX The `take` here is necessary to workaround a
|
||||
# PRQL issue where the above sort is dropped entirely
|
||||
take top
|
||||
aggregate { v = json.group_object column total }
|
||||
aggregate { v = json.group_object top_value total2 }
|
||||
)
|
||||
)
|
||||
)
|
||||
|
@ -595,7 +595,9 @@ textview_curses::handle_mouse(mouse_event& me)
|
||||
(int) this->lv_left + me.me_x);
|
||||
|
||||
this->set_selection_without_context(mc.mc_line);
|
||||
if (me.me_button == mouse_button_t::BUTTON_LEFT) {
|
||||
if (this->tc_supports_marks
|
||||
&& me.me_button == mouse_button_t::BUTTON_LEFT)
|
||||
{
|
||||
this->textview_value_for_row(mc.mc_line, al);
|
||||
auto line_sf
|
||||
= string_fragment::from_str(al.get_string());
|
||||
|
@ -474,10 +474,8 @@ EXPECTED_FILES = \
|
||||
$(srcdir)/%reldir%/test_prql.sh_06900fac5c2e854b1208320b753fcd43d4ba63a3.out \
|
||||
$(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.err \
|
||||
$(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.out \
|
||||
$(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.err \
|
||||
$(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.out \
|
||||
$(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.err \
|
||||
$(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.out \
|
||||
$(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.err \
|
||||
$(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out \
|
||||
$(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.err \
|
||||
$(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.out \
|
||||
$(srcdir)/%reldir%/test_regex101.sh_182ae9244db314a953af2bee969726e381bc5a32.err \
|
||||
|
@ -1 +1 @@
|
||||
[7m [0mSat Nov 03 08:00:00 1 normal 0 errors 0 warnings 0 marks
|
||||
[7m Sat Nov 03 08:00:00 1 normal 0 errors 0 warnings [0m 0 marks
|
||||
|
@ -1,2 +1,2 @@
|
||||
[7m Sat Nov 03 09:20:0[0m[1m[7m[31m0 1 normal 2 errors [0m[7m[32m 0 warnings [0m 1 marks
|
||||
[7m Sat Nov 03 09:45:0[0m0 1 normal 0 errors 0 warnings 0 marks
|
||||
[7m Sat Nov 03 09:20:00[0m[1m[7m[31m 1 normal 2 errors [0m[7m[32m 0 warnings [0m 1 marks
|
||||
[7m Sat Nov 03 09:45:00[0m 1 normal 0 errors 0 warnings 0 marks
|
||||
|
@ -1 +1 @@
|
||||
[7m [0m[1m[7m[31mS[0mat Nov 03 00:00:00 2 normal 2 errors 0 warnings 0 marks
|
||||
[7m Sat Nov 03 00:00:00 2 normal [0m[1m[7m[31m 2 errors 0 warnings [0m 0 marks
|
||||
|
@ -1 +1 @@
|
||||
[7m [0m[1m[7m[31mS[0mat Nov 03 08:00:00 2 normal 2 errors 0 warnings 0 marks
|
||||
[7m Sat Nov 03 08:00:00 2 normal [0m[1m[7m[31m 2 errors 0 warnings [0m 0 marks
|
||||
|
@ -0,0 +1,4 @@
|
||||
[1m[4m tslice [0m[1m[4m [0m[1m[4m v [0m[1m[4m [0m
|
||||
2011-11-03 00:19:00.000 [7m{"a.fsdn.com":33,"Other":31,"ad.[0m[7mdoubleclick.net":10,"www.goo⋯[0m[7m"\xaf"dia.com"[0m[7m:6,"s0.2m[0m[7mdn.net":[0m[7m5,"googl[0m[7meads.g.[0m[7mdoubl[0m[7mecli[0m[7mck[0m.net":3}
|
||||
2011-11-03 00:20:00.000 [7m{"www.bro-ids.org":24,"[0m[7mwww.google.com[0m[7m":15,"[0m[7msearc[0m[7mh.twi[0m[7mtter[0m[7m.[0m[7mc[0m[7m⋯[0m"\x8b""\xaf"ds.org":2,"ad.doubleclick.net":1,"cont-sjl-1.pandora.com":1}
|
||||
[1m2011-11-03 00:21:00.000[0m[1m [0m[1m[7m{[0m[1m[7m"[0m[1m[7mw[0m[1m[7mw[0m[1m[7mw[0m[1m.google.com":2,"ajax.googleapis.com":1,"api.twitter.com":1,"search.twitter.com":1,"www.bro-ids.org":1} [0m
|
@ -11,3 +11,7 @@ run_cap_test ${lnav_test} -n \
|
||||
run_cap_test ${lnav_test} -n \
|
||||
-c ";from access_log | take abc" \
|
||||
${test_dir}/logfile_access_log.0
|
||||
|
||||
run_cap_test ${lnav_test} -n \
|
||||
-c ";from bro_http_log | stats.hist bro_host slice:'1m'" \
|
||||
${test_dir}/logfile_bro_http.log.0
|
||||
|
Loading…
Reference in New Issue
Block a user