mirror of
https://github.com/tstack/lnav.git
synced 2024-09-11 13:05:51 +03:00
[log_format] change w3c_log to put extra columns into a JSON column
This commit is contained in:
parent
a9a08d3315
commit
4ff2b710d5
10
NEWS
10
NEWS
@ -8,10 +8,12 @@ lnav v0.9.1:
|
||||
following command to open the command prompt with ":filter-in " already
|
||||
filled in:
|
||||
:prompt command : 'filter-in '
|
||||
* Added support for the W3C Extended Log File Format. Note: Since the
|
||||
columns in a W3C log are specified in the file, the name of the format
|
||||
and SQLite table will include a hash ID over of the column names
|
||||
(e.g. w3c_7685df_log).
|
||||
* Added support for the W3C Extended Log File Format with the name
|
||||
"w3c_log". Similarly to the bro log format, the header is used to
|
||||
determine the columns in a particular file. However, since the columns
|
||||
can be different between files, the SQL table only has a well-known set
|
||||
of columns and the remainder are accessible through JSON-objects stored
|
||||
in columns like "cs_headers" and "sc_headers".
|
||||
* Added support for the S3 Access File Format.
|
||||
* To jump to the first search hit above the top line in a view, you can
|
||||
press CTRL+J instead of ENTER in the search prompt. Pressing ENTER
|
||||
|
@ -277,6 +277,7 @@ add_library(diag STATIC
|
||||
lnav_util.cc
|
||||
log_accel.cc
|
||||
log_actions.cc
|
||||
log_data_table.cc
|
||||
log_format.cc
|
||||
log_format_loader.cc
|
||||
log_level.cc
|
||||
|
@ -434,6 +434,7 @@ libdiag_a_SOURCES = \
|
||||
lnav_util.cc \
|
||||
log_accel.cc \
|
||||
log_actions.cc \
|
||||
log_data_table.cc \
|
||||
log_format.cc \
|
||||
log_format_loader.cc \
|
||||
log_level.cc \
|
||||
|
@ -34,16 +34,28 @@
|
||||
|
||||
all_logs_vtab::all_logs_vtab()
|
||||
: log_vtab_impl(intern_string::lookup("all_logs")),
|
||||
alv_value_name(intern_string::lookup("log_format")),
|
||||
alv_msg_name(intern_string::lookup("log_msg_format")),
|
||||
alv_schema_name(intern_string::lookup("log_msg_schema")) {
|
||||
alv_value_meta(intern_string::lookup("log_format"),
|
||||
value_kind_t::VALUE_TEXT,
|
||||
0),
|
||||
alv_msg_meta(intern_string::lookup("log_msg_format"),
|
||||
value_kind_t::VALUE_TEXT,
|
||||
1),
|
||||
alv_schema_meta(intern_string::lookup("log_msg_schema"),
|
||||
value_kind_t::VALUE_TEXT,
|
||||
2) {
|
||||
this->alv_value_meta.lvm_identifier = true;
|
||||
this->alv_msg_meta.lvm_identifier = true;
|
||||
this->alv_schema_meta.lvm_identifier = true;
|
||||
}
|
||||
|
||||
void all_logs_vtab::get_columns(std::vector<vtab_column> &cols) const
|
||||
{
|
||||
cols.emplace_back(this->alv_value_name.get());
|
||||
cols.emplace_back(this->alv_msg_name.get());
|
||||
cols.emplace_back(this->alv_schema_name.get(), SQLITE3_TEXT, "", true);
|
||||
cols.emplace_back(vtab_column(this->alv_value_meta.lvm_name.get())
|
||||
.with_comment("The name of the log file format"));
|
||||
cols.emplace_back(vtab_column(this->alv_msg_meta.lvm_name.get())
|
||||
.with_comment("The message format with variables replaced by hash marks"));
|
||||
cols.emplace_back(this->alv_schema_meta.lvm_name.get(), SQLITE3_TEXT, "", true,
|
||||
"The ID for the message schema");
|
||||
}
|
||||
|
||||
void all_logs_vtab::extract(std::shared_ptr<logfile> lf, uint64_t line_number,
|
||||
@ -51,7 +63,7 @@ void all_logs_vtab::extract(std::shared_ptr<logfile> lf, uint64_t line_number,
|
||||
std::vector<logline_value> &values)
|
||||
{
|
||||
auto format = lf->get_format();
|
||||
values.emplace_back(this->alv_value_name, format->get_name(), 0);
|
||||
values.emplace_back(this->alv_value_meta, format->get_name());
|
||||
|
||||
std::vector<logline_value> sub_values;
|
||||
|
||||
@ -66,14 +78,14 @@ void all_logs_vtab::extract(std::shared_ptr<logfile> lf, uint64_t line_number,
|
||||
|
||||
data_scanner ds(line, body.lr_start, body.lr_end);
|
||||
data_parser dp(&ds);
|
||||
|
||||
std::string str;
|
||||
|
||||
dp.dp_msg_format = &str;
|
||||
dp.parse();
|
||||
|
||||
tmp_shared_buffer tsb(str.c_str());
|
||||
|
||||
values.emplace_back(this->alv_msg_name, tsb.tsb_ref, 1);
|
||||
values.emplace_back(this->alv_msg_meta, tsb.tsb_ref);
|
||||
|
||||
this->alv_schema_manager.invalidate_refs();
|
||||
dp.dp_schema_id.to_string(this->alv_schema_buffer.data());
|
||||
@ -81,7 +93,7 @@ void all_logs_vtab::extract(std::shared_ptr<logfile> lf, uint64_t line_number,
|
||||
schema_ref.share(this->alv_schema_manager,
|
||||
this->alv_schema_buffer.data(),
|
||||
data_parser::schema_id_t::STRING_SIZE - 1);
|
||||
values.emplace_back(this->alv_schema_name, schema_ref, 2);
|
||||
values.emplace_back(this->alv_schema_meta, schema_ref);
|
||||
}
|
||||
|
||||
bool all_logs_vtab::is_valid(log_cursor &lc, logfile_sub_source &lss)
|
||||
|
@ -55,9 +55,9 @@ public:
|
||||
bool next(log_cursor &lc, logfile_sub_source &lss) override;
|
||||
|
||||
private:
|
||||
const intern_string_t alv_value_name;
|
||||
const intern_string_t alv_msg_name;
|
||||
const intern_string_t alv_schema_name;
|
||||
logline_value_meta alv_value_meta;
|
||||
logline_value_meta alv_msg_meta;
|
||||
logline_value_meta alv_schema_meta;
|
||||
shared_buffer alv_schema_manager;
|
||||
std::array<char, data_parser::schema_id_t::STRING_SIZE> alv_schema_buffer{};
|
||||
};
|
||||
|
@ -37,6 +37,7 @@
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "optional.hpp"
|
||||
#include "strnatcmp.h"
|
||||
|
||||
struct string_fragment {
|
||||
@ -134,6 +135,24 @@ struct string_fragment {
|
||||
return *prefix == '\0';
|
||||
}
|
||||
|
||||
string_fragment substr(int begin) {
|
||||
return string_fragment{
|
||||
this->sf_string,
|
||||
this->sf_begin + begin,
|
||||
this->sf_end
|
||||
};
|
||||
}
|
||||
|
||||
nonstd::optional<size_t> find(char ch) const {
|
||||
for (int lpc = this->sf_begin; lpc < this->sf_end; lpc++) {
|
||||
if (this->sf_string[lpc] == ch) {
|
||||
return lpc;
|
||||
}
|
||||
}
|
||||
|
||||
return nonstd::nullopt;
|
||||
}
|
||||
|
||||
const char *to_string(char *buf) const {
|
||||
memcpy(buf, this->data(), this->length());
|
||||
buf[this->length()] = '\0';
|
||||
|
@ -226,31 +226,28 @@ Result<string, string> execute_sql(exec_context &ec, const string &sql, string &
|
||||
}
|
||||
}
|
||||
else if (name[0] == ':' && ec.ec_line_values != nullptr) {
|
||||
vector<logline_value> &lvalues = *ec.ec_line_values;
|
||||
vector<logline_value>::iterator iter;
|
||||
|
||||
for (iter = lvalues.begin(); iter != lvalues.end(); ++iter) {
|
||||
if (strcmp(&name[1], iter->lv_name.get()) != 0) {
|
||||
for (auto& lv : *ec.ec_line_values) {
|
||||
if (lv.lv_meta.lvm_name != &name[1]) {
|
||||
continue;
|
||||
}
|
||||
switch (iter->lv_kind) {
|
||||
case logline_value::VALUE_BOOLEAN:
|
||||
sqlite3_bind_int64(stmt.in(), lpc + 1, iter->lv_value.i);
|
||||
switch (lv.lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
sqlite3_bind_int64(stmt.in(), lpc + 1, lv.lv_value.i);
|
||||
break;
|
||||
case logline_value::VALUE_FLOAT:
|
||||
sqlite3_bind_double(stmt.in(), lpc + 1, iter->lv_value.d);
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
sqlite3_bind_double(stmt.in(), lpc + 1, lv.lv_value.d);
|
||||
break;
|
||||
case logline_value::VALUE_INTEGER:
|
||||
sqlite3_bind_int64(stmt.in(), lpc + 1, iter->lv_value.i);
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
sqlite3_bind_int64(stmt.in(), lpc + 1, lv.lv_value.i);
|
||||
break;
|
||||
case logline_value::VALUE_NULL:
|
||||
case value_kind_t::VALUE_NULL:
|
||||
sqlite3_bind_null(stmt.in(), lpc + 1);
|
||||
break;
|
||||
default:
|
||||
sqlite3_bind_text(stmt.in(),
|
||||
lpc + 1,
|
||||
iter->text_value(),
|
||||
iter->text_length(),
|
||||
lv.text_value(),
|
||||
lv.text_length(),
|
||||
SQLITE_TRANSIENT);
|
||||
break;
|
||||
}
|
||||
|
@ -357,13 +357,16 @@ void field_overlay_source::build_field_lines(const listview_curses &lv)
|
||||
this->fos_unknown_key_size = 0;
|
||||
|
||||
for (auto & ldh_line_value : this->fos_log_helper.ldh_line_values) {
|
||||
int this_key_size = ldh_line_value.lv_name.size();
|
||||
auto& meta = ldh_line_value.lv_meta;
|
||||
int this_key_size = meta.lvm_name.size();
|
||||
|
||||
if (ldh_line_value.lv_kind == logline_value::VALUE_STRUCT) {
|
||||
if (meta.lvm_kind == value_kind_t::VALUE_STRUCT) {
|
||||
this_key_size += 9;
|
||||
}
|
||||
this->fos_known_key_size = max(
|
||||
this->fos_known_key_size, this_key_size);
|
||||
if (!meta.lvm_struct_name.empty()) {
|
||||
this_key_size += meta.lvm_struct_name.size() + 11;
|
||||
}
|
||||
this->fos_known_key_size = max(this->fos_known_key_size, this_key_size);
|
||||
}
|
||||
|
||||
for (auto iter = this->fos_log_helper.ldh_parser->dp_pairs.begin();
|
||||
@ -396,11 +399,16 @@ void field_overlay_source::build_field_lines(const listview_curses &lv)
|
||||
const log_format *last_format = nullptr;
|
||||
|
||||
for (auto & lv : this->fos_log_helper.ldh_line_values) {
|
||||
string format_name = lv.lv_format->get_name().to_string();
|
||||
if (!lv.lv_meta.lvm_format) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto curr_format = lv.lv_meta.lvm_format.value();
|
||||
string format_name = curr_format->get_name().to_string();
|
||||
attr_line_t al;
|
||||
string str, value_str = lv.to_string();
|
||||
|
||||
if (lv.lv_format != last_format) {
|
||||
if (curr_format != last_format) {
|
||||
this->fos_lines.emplace_back(" Known message fields for table " +
|
||||
format_name +
|
||||
":");
|
||||
@ -408,32 +416,48 @@ void field_overlay_source::build_field_lines(const listview_curses &lv)
|
||||
line_range(32, 32 + format_name.length()),
|
||||
&view_curses::VC_STYLE,
|
||||
vc.attrs_for_ident(format_name) | A_BOLD));
|
||||
last_format = lv.lv_format;
|
||||
last_format = curr_format;
|
||||
}
|
||||
|
||||
str = " " + lv.lv_name.to_string();
|
||||
str.append(this->fos_known_key_size - lv.lv_name.size() + 3, ' ');
|
||||
if (lv.lv_meta.lvm_struct_name.empty()) {
|
||||
str = " " + lv.lv_meta.lvm_name.to_string();
|
||||
} else {
|
||||
auto_mem<char, sqlite3_free> jgetter;
|
||||
|
||||
jgetter = sqlite3_mprintf(" jget(%s, '/%q')",
|
||||
lv.lv_meta.lvm_struct_name.get(),
|
||||
lv.lv_meta.lvm_name.get());
|
||||
str = jgetter;
|
||||
}
|
||||
str.append(this->fos_known_key_size - (str.length() - 3), ' ');
|
||||
str += " = " + value_str;
|
||||
|
||||
al.with_string(str)
|
||||
.with_attr(string_attr(
|
||||
line_range(3, 3 + lv.lv_name.size()),
|
||||
&view_curses::VC_STYLE,
|
||||
vc.attrs_for_ident(lv.lv_name.to_string())));
|
||||
al.with_string(str);
|
||||
if (lv.lv_meta.lvm_struct_name.empty()) {
|
||||
al.with_attr(string_attr(
|
||||
line_range(3, 3 + lv.lv_meta.lvm_name.size()),
|
||||
&view_curses::VC_STYLE,
|
||||
vc.attrs_for_ident(lv.lv_meta.lvm_name)));
|
||||
} else {
|
||||
al.with_attr(string_attr(
|
||||
line_range(8, 8 + lv.lv_meta.lvm_struct_name.size()),
|
||||
&view_curses::VC_STYLE,
|
||||
vc.attrs_for_ident(lv.lv_meta.lvm_struct_name)));
|
||||
}
|
||||
|
||||
this->fos_lines.emplace_back(al);
|
||||
this->add_key_line_attrs(this->fos_known_key_size);
|
||||
|
||||
if (lv.lv_kind == logline_value::VALUE_STRUCT) {
|
||||
if (lv.lv_meta.lvm_kind == value_kind_t::VALUE_STRUCT) {
|
||||
json_string js = extract(value_str.c_str());
|
||||
|
||||
al.clear()
|
||||
.append(" extract(")
|
||||
.append(lv.lv_name.get(),
|
||||
.append(lv.lv_meta.lvm_name.get(),
|
||||
&view_curses::VC_STYLE,
|
||||
vc.attrs_for_ident(lv.lv_name.get(), lv.lv_name.size()))
|
||||
vc.attrs_for_ident(lv.lv_meta.lvm_name))
|
||||
.append(")")
|
||||
.append(this->fos_known_key_size - lv.lv_name.size() - 9 + 3, ' ')
|
||||
.append(this->fos_known_key_size - lv.lv_meta.lvm_name.size() - 9 + 3, ' ')
|
||||
.append(" = ")
|
||||
.append((const char *) js.js_content.in(), js.js_len);
|
||||
this->fos_lines.emplace_back(al);
|
||||
|
@ -66,8 +66,14 @@
|
||||
"unit": {
|
||||
"field": "rt_unit",
|
||||
"scaling-factor": {
|
||||
"/msecs": 1000.0,
|
||||
"/micros": 1000000.0
|
||||
"msecs": {
|
||||
"op": "divide",
|
||||
"value": 1000.0
|
||||
},
|
||||
"micros": {
|
||||
"op": "divide",
|
||||
"value": 1000000.0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
@ -193,9 +193,25 @@
|
||||
"title": "/<format_name>/value/<value_name>/unit/scaling-factor",
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"(.*)": {
|
||||
"([^/]+)": {
|
||||
"title": "/<format_name>/value/<value_name>/unit/scaling-factor/<scale>",
|
||||
"type": "number"
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"op": {
|
||||
"title": "/<format_name>/value/<value_name>/unit/scaling-factor/<scale>/<>",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"identity",
|
||||
"multiply",
|
||||
"divide"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"title": "/<format_name>/value/<value_name>/unit/scaling-factor/<scale>/<>",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
|
@ -140,12 +140,17 @@ bool listview_curses::handle_key(int ch)
|
||||
vis_line_t last_line(this->get_inner_height() - 1);
|
||||
vis_line_t tail_bottom(this->get_top_for_last_row());
|
||||
|
||||
if (this->get_top() == last_line)
|
||||
if (this->is_selectable()) {
|
||||
this->set_selection(last_line);
|
||||
} else if (this->get_top() == last_line) {
|
||||
this->set_top(tail_bottom);
|
||||
else if (tail_bottom <= this->get_top())
|
||||
}
|
||||
else if (tail_bottom <= this->get_top()) {
|
||||
this->set_top(last_line);
|
||||
else
|
||||
}
|
||||
else {
|
||||
this->set_top(tail_bottom);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
|
@ -35,6 +35,7 @@
|
||||
#include <termios.h>
|
||||
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <fstream>
|
||||
#include <unordered_map>
|
||||
@ -535,10 +536,16 @@ static void json_write_row(yajl_gen handle, int row)
|
||||
|
||||
switch (hm.hm_column_type) {
|
||||
case SQLITE_FLOAT:
|
||||
case SQLITE_INTEGER:
|
||||
yajl_gen_number(handle, dls.dls_rows[row][col],
|
||||
strlen(dls.dls_rows[row][col]));
|
||||
case SQLITE_INTEGER: {
|
||||
auto len = strlen(dls.dls_rows[row][col]);
|
||||
|
||||
if (len == 0) {
|
||||
obj_map.gen();
|
||||
} else {
|
||||
yajl_gen_number(handle, dls.dls_rows[row][col], len);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case SQLITE_TEXT:
|
||||
switch (hm.hm_sub_type) {
|
||||
case 74: {
|
||||
@ -1005,7 +1012,7 @@ static Result<string, string> com_pipe_to(exec_context &ec, string cmdline, vect
|
||||
setenv("log_time", tmp_str, 1);
|
||||
setenv("log_path", ldh.ldh_file->get_filename().c_str(), 1);
|
||||
for (auto &ldh_line_value : ldh.ldh_line_values) {
|
||||
setenv(ldh_line_value.lv_name.get(),
|
||||
setenv(ldh_line_value.lv_meta.lvm_name.get(),
|
||||
ldh_line_value.to_string().c_str(), 1);
|
||||
}
|
||||
auto iter = ldh.ldh_parser->dp_pairs.begin();
|
||||
@ -3706,7 +3713,7 @@ public:
|
||||
}
|
||||
|
||||
auto format = lf->get_format();
|
||||
const logline_value_stats *stats = format->stats_for_value(this->lsvs_colname);
|
||||
const auto *stats = format->stats_for_value(this->lsvs_colname);
|
||||
|
||||
if (stats == NULL) {
|
||||
continue;
|
||||
@ -3791,11 +3798,11 @@ public:
|
||||
logline_value_cmp(&this->lsvs_colname));
|
||||
|
||||
if (lv_iter != values.end()) {
|
||||
switch (lv_iter->lv_kind) {
|
||||
case logline_value::VALUE_FLOAT:
|
||||
switch (lv_iter->lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
row_out.add_value(sr, lv_iter->lv_value.d, ll->is_marked());
|
||||
break;
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
row_out.add_value(sr, lv_iter->lv_value.i, ll->is_marked());
|
||||
break;
|
||||
default:
|
||||
@ -3844,15 +3851,15 @@ public:
|
||||
logline_value_cmp(&this->lsvs_colname));
|
||||
|
||||
if (lv_iter != values.end()) {
|
||||
switch (lv_iter->lv_kind) {
|
||||
case logline_value::VALUE_FLOAT:
|
||||
switch (lv_iter->lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
if (range_min <= lv_iter->lv_value.d &&
|
||||
lv_iter->lv_value.d <= range_max) {
|
||||
log_tc.toggle_user_mark(&textview_curses::BM_USER,
|
||||
curr_line);
|
||||
}
|
||||
break;
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
if (range_min <= lv_iter->lv_value.i &&
|
||||
lv_iter->lv_value.i <= range_max) {
|
||||
log_tc.toggle_user_mark(&textview_curses::BM_USER,
|
||||
@ -3876,7 +3883,7 @@ public:
|
||||
class db_spectro_value_source : public spectrogram_value_source {
|
||||
public:
|
||||
db_spectro_value_source(string colname)
|
||||
: dsvs_colname(colname),
|
||||
: dsvs_colname(std::move(colname)),
|
||||
dsvs_begin_time(0),
|
||||
dsvs_end_time(0) {
|
||||
this->update_stats();
|
||||
@ -4090,16 +4097,23 @@ static void command_prompt(vector<string> &args)
|
||||
}
|
||||
else {
|
||||
for (auto &ldh_line_value : ldh.ldh_line_values) {
|
||||
const logline_value_stats *stats = ldh_line_value.lv_format->stats_for_value(
|
||||
ldh_line_value.lv_name);
|
||||
auto& meta = ldh_line_value.lv_meta;
|
||||
|
||||
if (!meta.lvm_format) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto *stats = meta.lvm_format.value()->
|
||||
stats_for_value(meta.lvm_name);
|
||||
|
||||
if (stats == nullptr) {
|
||||
continue;
|
||||
}
|
||||
|
||||
lnav_data.ld_rl_view->add_possibility(LNM_COMMAND,
|
||||
"numeric-colname",
|
||||
ldh_line_value.lv_name.to_string());
|
||||
lnav_data.ld_rl_view->add_possibility(
|
||||
LNM_COMMAND,
|
||||
"numeric-colname",
|
||||
meta.lvm_name.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -82,7 +82,7 @@ static string execute_action(log_data_helper &ldh,
|
||||
setenv("LNAV_ACTION_FILE_LINE", env_buffer, 1);
|
||||
snprintf(env_buffer, sizeof(env_buffer), "%d", ldh.ldh_y_offset + 1);
|
||||
setenv("LNAV_ACTION_MSG_LINE", env_buffer, 1);
|
||||
setenv("LNAV_ACTION_VALUE_NAME", lv.lv_name.get(), 1);
|
||||
setenv("LNAV_ACTION_VALUE_NAME", lv.lv_meta.lvm_name.get(), 1);
|
||||
value_line = ldh.ldh_y_offset - ldh.get_value_line(lv) + 1;
|
||||
snprintf(env_buffer, sizeof(env_buffer), "%d", value_line);
|
||||
setenv("LNAV_ACTION_VALUE_LINE", env_buffer, 1);
|
||||
|
@ -114,17 +114,17 @@ public:
|
||||
this->ldh_json_pairs.clear();
|
||||
|
||||
for (const auto& lv : this->ldh_line_values) {
|
||||
this->ldh_namer->cn_builtin_names.emplace_back(lv.lv_name.get());
|
||||
this->ldh_namer->cn_builtin_names.emplace_back(lv.lv_meta.lvm_name.get());
|
||||
}
|
||||
|
||||
for (auto & ldh_line_value : this->ldh_line_values) {
|
||||
switch (ldh_line_value.lv_kind) {
|
||||
case logline_value::VALUE_JSON: {
|
||||
switch (ldh_line_value.lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_JSON: {
|
||||
json_ptr_walk jpw;
|
||||
|
||||
if (jpw.parse(ldh_line_value.lv_sbr.get_data(), ldh_line_value.lv_sbr.length()) == yajl_status_ok &&
|
||||
jpw.complete_parse() == yajl_status_ok) {
|
||||
this->ldh_json_pairs[ldh_line_value.lv_name] = jpw.jpw_values;
|
||||
this->ldh_json_pairs[ldh_line_value.lv_meta.lvm_name] = jpw.jpw_values;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
219
src/log_data_table.cc
Normal file
219
src/log_data_table.cc
Normal file
@ -0,0 +1,219 @@
|
||||
/**
|
||||
* Copyright (c) 2020, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include "log_data_table.hh"
|
||||
|
||||
log_data_table::log_data_table(logfile_sub_source &lss, log_vtab_manager &lvm,
|
||||
content_line_t template_line,
|
||||
intern_string_t table_name)
|
||||
: log_vtab_impl(table_name),
|
||||
ldt_log_source(lss),
|
||||
ldt_template_line(template_line),
|
||||
ldt_instance(-1) {
|
||||
std::shared_ptr<logfile> lf = lss.find(template_line);
|
||||
auto format = lf->get_format();
|
||||
|
||||
this->vi_supports_indexes = false;
|
||||
this->ldt_format_impl = lvm.lookup_impl(format->get_name());
|
||||
this->get_columns_int();
|
||||
}
|
||||
|
||||
void log_data_table::get_columns_int()
|
||||
{
|
||||
static intern_string_t instance_name = intern_string::lookup("log_msg_instance");
|
||||
|
||||
auto& cols = this->ldt_cols;
|
||||
auto& metas = this->ldt_value_metas;
|
||||
content_line_t cl_copy = this->ldt_template_line;
|
||||
std::shared_ptr<logfile> lf = this->ldt_log_source.find(cl_copy);
|
||||
struct line_range body;
|
||||
string_attrs_t sa;
|
||||
std::vector<logline_value> line_values;
|
||||
auto format = lf->get_format();
|
||||
shared_buffer_ref line;
|
||||
|
||||
if (this->ldt_format_impl != nullptr) {
|
||||
this->ldt_format_impl->get_columns(cols);
|
||||
}
|
||||
lf->read_full_message(lf->begin() + cl_copy, line);
|
||||
format->annotate(cl_copy, line, sa, line_values, false);
|
||||
body = find_string_attr_range(sa, &SA_BODY);
|
||||
if (body.lr_end == -1) {
|
||||
this->ldt_schema_id.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
data_scanner ds(line, body.lr_start, body.lr_end);
|
||||
data_parser dp(&ds);
|
||||
column_namer cn;
|
||||
|
||||
dp.parse();
|
||||
|
||||
metas.emplace_back(
|
||||
instance_name, value_kind_t::VALUE_INTEGER, cols.size(), format.get());
|
||||
cols.emplace_back("log_msg_instance", SQLITE_INTEGER);
|
||||
for (auto pair_iter = dp.dp_pairs.begin();
|
||||
pair_iter != dp.dp_pairs.end();
|
||||
++pair_iter) {
|
||||
std::string key_str = dp.get_element_string(
|
||||
pair_iter->e_sub_elements->front());
|
||||
std::string colname = cn.add_column(key_str);
|
||||
int sql_type = SQLITE3_TEXT;
|
||||
value_kind_t kind = value_kind_t::VALUE_TEXT;
|
||||
std::string collator;
|
||||
|
||||
switch (pair_iter->e_sub_elements->back().value_token()) {
|
||||
case DT_IPV4_ADDRESS:
|
||||
case DT_IPV6_ADDRESS:
|
||||
collator = "ipaddress";
|
||||
break;
|
||||
|
||||
case DT_NUMBER:
|
||||
sql_type = SQLITE_FLOAT;
|
||||
kind = value_kind_t::VALUE_FLOAT;
|
||||
break;
|
||||
|
||||
default:
|
||||
collator = "naturalnocase";
|
||||
break;
|
||||
}
|
||||
metas.emplace_back(
|
||||
intern_string::lookup(colname), kind, cols.size(), format.get());
|
||||
cols.emplace_back(colname, sql_type, collator);
|
||||
}
|
||||
this->ldt_schema_id = dp.dp_schema_id;
|
||||
}
|
||||
|
||||
bool log_data_table::next(log_cursor &lc, logfile_sub_source &lss)
|
||||
{
|
||||
if (lc.lc_curr_line == vis_line_t(-1)) {
|
||||
this->ldt_instance = -1;
|
||||
}
|
||||
|
||||
lc.lc_curr_line = lc.lc_curr_line + vis_line_t(1);
|
||||
lc.lc_sub_index = 0;
|
||||
|
||||
if (lc.lc_curr_line == (int)lss.text_line_count()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
content_line_t cl;
|
||||
|
||||
cl = lss.at(lc.lc_curr_line);
|
||||
std::shared_ptr<logfile> lf = lss.find(cl);
|
||||
auto lf_iter = lf->begin() + cl;
|
||||
|
||||
if (!lf_iter->is_message()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (lf_iter->has_schema() &&
|
||||
!lf_iter->match_schema(this->ldt_schema_id)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
string_attrs_t sa;
|
||||
struct line_range body;
|
||||
std::vector<logline_value> line_values;
|
||||
|
||||
lf->read_full_message(lf_iter, this->ldt_current_line);
|
||||
lf->get_format()->annotate(cl,
|
||||
this->ldt_current_line,
|
||||
sa,
|
||||
line_values,
|
||||
false);
|
||||
body = find_string_attr_range(sa, &SA_BODY);
|
||||
if (body.lr_end == -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
data_scanner ds(this->ldt_current_line, body.lr_start, body.lr_end);
|
||||
data_parser dp(&ds);
|
||||
dp.parse();
|
||||
|
||||
lf_iter->set_schema(dp.dp_schema_id);
|
||||
|
||||
/* The cached schema ID in the log line is not complete, so we still */
|
||||
/* need to check for a full match. */
|
||||
if (dp.dp_schema_id != this->ldt_schema_id) {
|
||||
return false;
|
||||
}
|
||||
|
||||
this->ldt_pairs.clear();
|
||||
this->ldt_pairs.swap(dp.dp_pairs, __FILE__, __LINE__);
|
||||
this->ldt_instance += 1;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void log_data_table::extract(std::shared_ptr<logfile> lf, uint64_t line_number,
|
||||
shared_buffer_ref &line,
|
||||
std::vector<logline_value> &values)
|
||||
{
|
||||
auto meta_iter = this->ldt_value_metas.begin();
|
||||
|
||||
this->ldt_format_impl->extract(lf, line_number, line, values);
|
||||
values.emplace_back(*meta_iter, this->ldt_instance);
|
||||
++meta_iter;
|
||||
for (auto &ldt_pair : this->ldt_pairs) {
|
||||
const data_parser::element &pvalue = ldt_pair.get_pair_value();
|
||||
|
||||
switch (pvalue.value_token()) {
|
||||
case DT_NUMBER: {
|
||||
char scan_value[line.length() + 1];
|
||||
double d = 0.0;
|
||||
|
||||
memcpy(scan_value,
|
||||
line.get_data() + pvalue.e_capture.c_begin,
|
||||
pvalue.e_capture.length());
|
||||
scan_value[pvalue.e_capture.length()] = '\0';
|
||||
if (sscanf(scan_value, "%lf", &d) != 1) {
|
||||
d = 0.0;
|
||||
}
|
||||
values.emplace_back(*meta_iter, d);
|
||||
break;
|
||||
}
|
||||
|
||||
default: {
|
||||
values.emplace_back(
|
||||
*meta_iter,
|
||||
line,
|
||||
line_range{
|
||||
pvalue.e_capture.c_begin,
|
||||
pvalue.e_capture.c_end
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
++meta_iter;
|
||||
}
|
||||
}
|
@ -47,192 +47,26 @@ public:
|
||||
log_data_table(logfile_sub_source &lss,
|
||||
log_vtab_manager &lvm,
|
||||
content_line_t template_line,
|
||||
intern_string_t table_name)
|
||||
: log_vtab_impl(table_name),
|
||||
ldt_log_source(lss),
|
||||
ldt_template_line(template_line),
|
||||
ldt_parent_column_count(0),
|
||||
ldt_instance(-1) {
|
||||
std::shared_ptr<logfile> lf = lss.find(template_line);
|
||||
auto format = lf->get_format();
|
||||
intern_string_t table_name);
|
||||
|
||||
this->vi_supports_indexes = false;
|
||||
this->ldt_format_impl = lvm.lookup_impl(format->get_name());
|
||||
this->get_columns_int(this->ldt_cols);
|
||||
};
|
||||
void get_columns_int();
|
||||
|
||||
void get_columns_int(std::vector<vtab_column> &cols)
|
||||
{
|
||||
content_line_t cl_copy = this->ldt_template_line;
|
||||
std::shared_ptr<logfile> lf = this->ldt_log_source.find(cl_copy);
|
||||
struct line_range body;
|
||||
string_attrs_t sa;
|
||||
std::vector<logline_value> line_values;
|
||||
auto format = lf->get_format();
|
||||
shared_buffer_ref line;
|
||||
|
||||
if (this->ldt_format_impl != nullptr) {
|
||||
this->ldt_format_impl->get_columns(cols);
|
||||
}
|
||||
this->ldt_parent_column_count = cols.size();
|
||||
lf->read_full_message(lf->begin() + cl_copy, line);
|
||||
format->annotate(cl_copy, line, sa, line_values, false);
|
||||
body = find_string_attr_range(sa, &SA_BODY);
|
||||
if (body.lr_end == -1) {
|
||||
this->ldt_schema_id.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
data_scanner ds(line, body.lr_start, body.lr_end);
|
||||
data_parser dp(&ds);
|
||||
column_namer cn;
|
||||
|
||||
dp.parse();
|
||||
|
||||
cols.emplace_back("log_msg_instance", SQLITE_INTEGER);
|
||||
for (auto pair_iter = dp.dp_pairs.begin();
|
||||
pair_iter != dp.dp_pairs.end();
|
||||
++pair_iter) {
|
||||
std::string key_str = dp.get_element_string(
|
||||
pair_iter->e_sub_elements->front());
|
||||
std::string colname = cn.add_column(key_str);
|
||||
int sql_type = SQLITE3_TEXT;
|
||||
std::string collator;
|
||||
|
||||
switch (pair_iter->e_sub_elements->back().value_token()) {
|
||||
case DT_IPV4_ADDRESS:
|
||||
case DT_IPV6_ADDRESS:
|
||||
collator = "ipaddress";
|
||||
break;
|
||||
|
||||
case DT_NUMBER:
|
||||
sql_type = SQLITE_FLOAT;
|
||||
break;
|
||||
|
||||
default:
|
||||
collator = "naturalnocase";
|
||||
break;
|
||||
}
|
||||
cols.emplace_back(colname, sql_type, collator);
|
||||
}
|
||||
this->ldt_schema_id = dp.dp_schema_id;
|
||||
};
|
||||
|
||||
void get_columns(std::vector<vtab_column> &cols) const {
|
||||
void get_columns(std::vector<vtab_column> &cols) const override {
|
||||
cols = this->ldt_cols;
|
||||
};
|
||||
|
||||
void get_foreign_keys(std::vector<std::string> &keys_inout) const
|
||||
void get_foreign_keys(std::vector<std::string> &keys_inout) const override
|
||||
{
|
||||
log_vtab_impl::get_foreign_keys(keys_inout);
|
||||
keys_inout.emplace_back("log_msg_instance");
|
||||
};
|
||||
|
||||
bool next(log_cursor &lc, logfile_sub_source &lss)
|
||||
{
|
||||
if (lc.lc_curr_line == vis_line_t(-1)) {
|
||||
this->ldt_instance = -1;
|
||||
}
|
||||
|
||||
lc.lc_curr_line = lc.lc_curr_line + vis_line_t(1);
|
||||
lc.lc_sub_index = 0;
|
||||
|
||||
if (lc.lc_curr_line == (int)lss.text_line_count()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
content_line_t cl;
|
||||
|
||||
cl = lss.at(lc.lc_curr_line);
|
||||
std::shared_ptr<logfile> lf = lss.find(cl);
|
||||
auto lf_iter = lf->begin() + cl;
|
||||
|
||||
if (!lf_iter->is_message()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (lf_iter->has_schema() &&
|
||||
!lf_iter->match_schema(this->ldt_schema_id)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
string_attrs_t sa;
|
||||
struct line_range body;
|
||||
std::vector<logline_value> line_values;
|
||||
|
||||
lf->read_full_message(lf_iter, this->ldt_current_line);
|
||||
lf->get_format()->annotate(cl,
|
||||
this->ldt_current_line,
|
||||
sa,
|
||||
line_values,
|
||||
false);
|
||||
body = find_string_attr_range(sa, &SA_BODY);
|
||||
if (body.lr_end == -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
data_scanner ds(this->ldt_current_line, body.lr_start, body.lr_end);
|
||||
data_parser dp(&ds);
|
||||
dp.parse();
|
||||
|
||||
lf_iter->set_schema(dp.dp_schema_id);
|
||||
|
||||
/* The cached schema ID in the log line is not complete, so we still */
|
||||
/* need to check for a full match. */
|
||||
if (dp.dp_schema_id != this->ldt_schema_id) {
|
||||
return false;
|
||||
}
|
||||
|
||||
this->ldt_pairs.clear();
|
||||
this->ldt_pairs.swap(dp.dp_pairs, __FILE__, __LINE__);
|
||||
this->ldt_instance += 1;
|
||||
|
||||
return true;
|
||||
};
|
||||
bool next(log_cursor &lc, logfile_sub_source &lss) override;
|
||||
|
||||
void extract(std::shared_ptr<logfile> lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref &line,
|
||||
std::vector<logline_value> &values)
|
||||
{
|
||||
static intern_string_t instance_name = intern_string::lookup("log_msg_instance");
|
||||
|
||||
int next_column = this->ldt_parent_column_count;
|
||||
|
||||
this->ldt_format_impl->extract(lf, line_number, line, values);
|
||||
values.emplace_back(instance_name, this->ldt_instance, lf->get_format().get());
|
||||
logline_value &lv = values.back();
|
||||
lv.lv_column = next_column++;
|
||||
for (auto &ldt_pair : this->ldt_pairs) {
|
||||
const data_parser::element &pvalue = ldt_pair.get_pair_value();
|
||||
|
||||
switch (pvalue.value_token()) {
|
||||
case DT_NUMBER: {
|
||||
char scan_value[line.length() + 1];
|
||||
double d = 0.0;
|
||||
|
||||
memcpy(scan_value,
|
||||
line.get_data() + pvalue.e_capture.c_begin,
|
||||
pvalue.e_capture.length());
|
||||
scan_value[pvalue.e_capture.length()] = '\0';
|
||||
if (sscanf(scan_value, "%lf", &d) != 1) {
|
||||
d = 0.0;
|
||||
}
|
||||
values.emplace_back(intern_string::lookup("", 0), d, lf->get_format().get());
|
||||
}
|
||||
break;
|
||||
|
||||
default: {
|
||||
values.emplace_back(intern_string::lookup("", 0),
|
||||
logline_value::VALUE_TEXT, line, false, nullptr,
|
||||
-1, pvalue.e_capture.c_begin, pvalue.e_capture.c_end);
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
values.back().lv_column = next_column++;
|
||||
}
|
||||
};
|
||||
std::vector<logline_value> &values) override;
|
||||
|
||||
private:
|
||||
logfile_sub_source &ldt_log_source;
|
||||
@ -241,9 +75,9 @@ private:
|
||||
shared_buffer_ref ldt_current_line;
|
||||
data_parser::element_list_t ldt_pairs;
|
||||
std::shared_ptr<log_vtab_impl> ldt_format_impl;
|
||||
int ldt_parent_column_count;
|
||||
int64_t ldt_instance;
|
||||
std::vector<vtab_column> ldt_cols;
|
||||
std::vector<logline_value_meta> ldt_value_metas;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
@ -99,57 +99,45 @@ struct line_range logline_value::origin_in_full_msg(const char *msg, ssize_t len
|
||||
return retval;
|
||||
}
|
||||
|
||||
logline_value::logline_value(const intern_string_t name,
|
||||
logline_value::kind_t kind, shared_buffer_ref &sbr,
|
||||
bool ident, const scaling_factor *scaling, int col,
|
||||
int start, int end, bool from_module,
|
||||
const log_format *format)
|
||||
: lv_name(name), lv_kind(kind),
|
||||
lv_identifier(ident), lv_column(col), lv_hidden(false), lv_sub_offset(0),
|
||||
lv_origin(start, end),
|
||||
lv_from_module(from_module),
|
||||
lv_format(format)
|
||||
logline_value::logline_value(logline_value_meta lvm, shared_buffer_ref &sbr,
|
||||
struct line_range origin)
|
||||
: lv_meta(std::move(lvm)), lv_origin(origin)
|
||||
{
|
||||
if (sbr.get_data() == nullptr) {
|
||||
this->lv_kind = kind = VALUE_NULL;
|
||||
this->lv_meta.lvm_kind = value_kind_t::VALUE_NULL;
|
||||
}
|
||||
|
||||
switch (kind) {
|
||||
case VALUE_JSON:
|
||||
case VALUE_STRUCT:
|
||||
case VALUE_TEXT:
|
||||
case VALUE_QUOTED:
|
||||
case VALUE_W3C_QUOTED:
|
||||
case VALUE_TIMESTAMP:
|
||||
this->lv_sbr.subset(sbr, start, end - start);
|
||||
switch (this->lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_JSON:
|
||||
case value_kind_t::VALUE_STRUCT:
|
||||
case value_kind_t::VALUE_TEXT:
|
||||
case value_kind_t::VALUE_QUOTED:
|
||||
case value_kind_t::VALUE_W3C_QUOTED:
|
||||
case value_kind_t::VALUE_TIMESTAMP:
|
||||
this->lv_sbr.subset(sbr, origin.lr_start, origin.length());
|
||||
break;
|
||||
|
||||
case VALUE_NULL:
|
||||
case value_kind_t::VALUE_NULL:
|
||||
break;
|
||||
|
||||
case VALUE_INTEGER:
|
||||
strtonum(this->lv_value.i, sbr.get_data_at(start), end - start);
|
||||
if (scaling != NULL) {
|
||||
scaling->scale(this->lv_value.i);
|
||||
}
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
strtonum(this->lv_value.i, sbr.get_data_at(
|
||||
origin.lr_start), origin.length());
|
||||
break;
|
||||
|
||||
case VALUE_FLOAT: {
|
||||
ssize_t len = end - start;
|
||||
case value_kind_t::VALUE_FLOAT: {
|
||||
ssize_t len = origin.length();
|
||||
char scan_value[len + 1];
|
||||
|
||||
memcpy(scan_value, sbr.get_data_at(start), len);
|
||||
memcpy(scan_value, sbr.get_data_at(origin.lr_start), len);
|
||||
scan_value[len] = '\0';
|
||||
this->lv_value.d = strtod(scan_value, NULL);
|
||||
if (scaling != NULL) {
|
||||
scaling->scale(this->lv_value.d);
|
||||
}
|
||||
this->lv_value.d = strtod(scan_value, nullptr);
|
||||
break;
|
||||
}
|
||||
|
||||
case VALUE_BOOLEAN:
|
||||
if (strncmp(sbr.get_data_at(start), "true", end - start) == 0 ||
|
||||
strncmp(sbr.get_data_at(start), "yes", end - start) == 0) {
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
if (strncmp(sbr.get_data_at(origin.lr_start), "true", origin.length()) == 0 ||
|
||||
strncmp(sbr.get_data_at(origin.lr_start), "yes", origin.length()) == 0) {
|
||||
this->lv_value.i = 1;
|
||||
}
|
||||
else {
|
||||
@ -157,8 +145,8 @@ logline_value::logline_value(const intern_string_t name,
|
||||
}
|
||||
break;
|
||||
|
||||
case VALUE_UNKNOWN:
|
||||
case VALUE__MAX:
|
||||
case value_kind_t::VALUE_UNKNOWN:
|
||||
case value_kind_t::VALUE__MAX:
|
||||
ensure(0);
|
||||
break;
|
||||
}
|
||||
@ -168,28 +156,28 @@ std::string logline_value::to_string() const
|
||||
{
|
||||
char buffer[128];
|
||||
|
||||
switch (this->lv_kind) {
|
||||
case VALUE_NULL:
|
||||
switch (this->lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_NULL:
|
||||
return "null";
|
||||
|
||||
case VALUE_JSON:
|
||||
case VALUE_STRUCT:
|
||||
case VALUE_TEXT:
|
||||
case VALUE_TIMESTAMP:
|
||||
case value_kind_t::VALUE_JSON:
|
||||
case value_kind_t::VALUE_STRUCT:
|
||||
case value_kind_t::VALUE_TEXT:
|
||||
case value_kind_t::VALUE_TIMESTAMP:
|
||||
if (this->lv_sbr.empty()) {
|
||||
return this->lv_intern_string.to_string();
|
||||
}
|
||||
return std::string(this->lv_sbr.get_data(), this->lv_sbr.length());
|
||||
|
||||
case VALUE_QUOTED:
|
||||
case VALUE_W3C_QUOTED:
|
||||
case value_kind_t::VALUE_QUOTED:
|
||||
case value_kind_t::VALUE_W3C_QUOTED:
|
||||
if (this->lv_sbr.length() == 0) {
|
||||
return "";
|
||||
} else {
|
||||
switch (this->lv_sbr.get_data()[0]) {
|
||||
case '\'':
|
||||
case '"': {
|
||||
auto unquote_func = this->lv_kind == VALUE_W3C_QUOTED ?
|
||||
auto unquote_func = this->lv_meta.lvm_kind == value_kind_t::VALUE_W3C_QUOTED ?
|
||||
unquote_w3c : unquote;
|
||||
char unquoted_str[this->lv_sbr.length()];
|
||||
size_t unquoted_len;
|
||||
@ -205,15 +193,15 @@ std::string logline_value::to_string() const
|
||||
}
|
||||
break;
|
||||
|
||||
case VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
snprintf(buffer, sizeof(buffer), "%" PRId64, this->lv_value.i);
|
||||
break;
|
||||
|
||||
case VALUE_FLOAT:
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
snprintf(buffer, sizeof(buffer), "%lf", this->lv_value.d);
|
||||
break;
|
||||
|
||||
case VALUE_BOOLEAN:
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
if (this->lv_value.i) {
|
||||
return "true";
|
||||
}
|
||||
@ -221,8 +209,8 @@ std::string logline_value::to_string() const
|
||||
return "false";
|
||||
}
|
||||
break;
|
||||
case VALUE_UNKNOWN:
|
||||
case VALUE__MAX:
|
||||
case value_kind_t::VALUE_UNKNOWN:
|
||||
case value_kind_t::VALUE__MAX:
|
||||
ensure(0);
|
||||
break;
|
||||
}
|
||||
@ -523,8 +511,8 @@ static int json_array_end(void *ctx)
|
||||
|
||||
sbr.subset(jlu->jlu_shared_buffer, jlu->jlu_sub_start,
|
||||
sub_end - jlu->jlu_sub_start);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, sbr, -1, jlu->jlu_format);
|
||||
jlu->jlu_format->jlf_line_values.back().lv_kind = logline_value::VALUE_JSON;
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(jlu->jlu_format->
|
||||
get_value_meta(field_name, value_kind_t::VALUE_JSON), sbr);
|
||||
}
|
||||
|
||||
return 1;
|
||||
@ -549,7 +537,8 @@ static int rewrite_json_null(yajlpp_parse_context *ypc)
|
||||
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
|
||||
return 1;
|
||||
}
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, jlu->jlu_format);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(jlu->jlu_format->
|
||||
get_value_meta(field_name, value_kind_t::VALUE_NULL));
|
||||
|
||||
return 1;
|
||||
}
|
||||
@ -562,8 +551,9 @@ static int rewrite_json_bool(yajlpp_parse_context *ypc, int val)
|
||||
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
|
||||
return 1;
|
||||
}
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, (bool)val, jlu->jlu_format);
|
||||
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_BOOLEAN),
|
||||
(bool) val);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -575,8 +565,9 @@ static int rewrite_json_int(yajlpp_parse_context *ypc, long long val)
|
||||
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
|
||||
return 1;
|
||||
}
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, (int64_t)val, jlu->jlu_format);
|
||||
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_INTEGER),
|
||||
val);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -588,7 +579,9 @@ static int rewrite_json_double(yajlpp_parse_context *ypc, double val)
|
||||
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
|
||||
return 1;
|
||||
}
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, val, jlu->jlu_format);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_FLOAT),
|
||||
val);
|
||||
|
||||
return 1;
|
||||
}
|
||||
@ -963,19 +956,17 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
|
||||
|
||||
for (size_t lpc = 0; lpc < pat.p_value_by_index.size(); lpc++) {
|
||||
const indexed_value_def &ivd = pat.p_value_by_index[lpc];
|
||||
const struct scaling_factor *scaling = NULL;
|
||||
const struct scaling_factor *scaling = nullptr;
|
||||
pcre_context::capture_t *cap = pc[ivd.ivd_index];
|
||||
const value_def &vd = *ivd.ivd_value_def;
|
||||
|
||||
if (ivd.ivd_unit_field_index >= 0) {
|
||||
pcre_context::iterator unit_cap = pc[ivd.ivd_unit_field_index];
|
||||
|
||||
if (unit_cap != NULL && unit_cap->c_begin != -1) {
|
||||
if (unit_cap != nullptr && unit_cap->c_begin != -1) {
|
||||
intern_string_t unit_val = intern_string::lookup(
|
||||
pi.get_substr_start(unit_cap), unit_cap->length());
|
||||
map<const intern_string_t, scaling_factor>::const_iterator unit_iter;
|
||||
|
||||
unit_iter = vd.vd_unit_scaling.find(unit_val);
|
||||
auto unit_iter = vd.vd_unit_scaling.find(unit_val);
|
||||
if (unit_iter != vd.vd_unit_scaling.end()) {
|
||||
const struct scaling_factor &sf = unit_iter->second;
|
||||
|
||||
@ -985,30 +976,23 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
|
||||
}
|
||||
|
||||
if (cap->is_valid()) {
|
||||
values.emplace_back(vd.vd_name,
|
||||
vd.vd_kind,
|
||||
values.emplace_back(vd.vd_meta,
|
||||
line,
|
||||
vd.vd_identifier,
|
||||
scaling,
|
||||
vd.vd_column,
|
||||
cap->c_begin,
|
||||
cap->c_end,
|
||||
pat.p_module_format,
|
||||
this);
|
||||
line_range{cap->c_begin, cap->c_end});
|
||||
values.back().apply_scaling(scaling);
|
||||
} else {
|
||||
values.emplace_back(vd.vd_name, this);
|
||||
values.emplace_back(vd.vd_meta);
|
||||
}
|
||||
values.back().lv_hidden = vd.vd_hidden || vd.vd_user_hidden;
|
||||
}
|
||||
|
||||
if (annotate_module && module_cap != NULL && body_cap != NULL &&
|
||||
if (annotate_module && module_cap != nullptr && body_cap != nullptr &&
|
||||
body_cap->is_valid()) {
|
||||
intern_string_t mod_name = intern_string::lookup(
|
||||
pi.get_substr_start(module_cap), module_cap->length());
|
||||
auto mod_iter = MODULE_FORMATS.find(mod_name);
|
||||
|
||||
if (mod_iter != MODULE_FORMATS.end() &&
|
||||
mod_iter->second.mf_mod_format != NULL) {
|
||||
mod_iter->second.mf_mod_format != nullptr) {
|
||||
module_format &mf = mod_iter->second;
|
||||
shared_buffer_ref body_ref;
|
||||
|
||||
@ -1016,7 +1000,7 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
|
||||
body_ref.subset(line, body_cap->c_begin, body_cap->length());
|
||||
mf.mf_mod_format->annotate(line_number, body_ref, sa, values, false);
|
||||
for (auto &value : values) {
|
||||
if (!value.lv_from_module) {
|
||||
if (!value.lv_meta.lvm_from_module) {
|
||||
continue;
|
||||
}
|
||||
value.lv_origin.lr_start += body_cap->c_begin;
|
||||
@ -1038,13 +1022,13 @@ void external_log_format::rewrite(exec_context &ec,
|
||||
|
||||
for (iter = values.begin(); iter != values.end(); ++iter) {
|
||||
if (!iter->lv_origin.is_valid()) {
|
||||
log_debug("not rewriting value with invalid origin -- %s", iter->lv_name.get());
|
||||
log_debug("not rewriting value with invalid origin -- %s", iter->lv_meta.lvm_name.get());
|
||||
continue;
|
||||
}
|
||||
|
||||
auto vd_iter = this->elf_value_defs.find(iter->lv_name);
|
||||
auto vd_iter = this->elf_value_defs.find(iter->lv_meta.lvm_name);
|
||||
if (vd_iter == this->elf_value_defs.end()) {
|
||||
log_debug("not rewriting undefined value -- %s", iter->lv_name.get());
|
||||
log_debug("not rewriting undefined value -- %s", iter->lv_meta.lvm_name.get());
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -1129,7 +1113,9 @@ static int rewrite_json_field(yajlpp_parse_context *ypc, const unsigned char *st
|
||||
jlu->jlu_line->get_timeval(), 'T');
|
||||
}
|
||||
tmp_shared_buffer tsb(time_buf);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, tsb.tsb_ref);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_TEXT),
|
||||
tsb.tsb_ref);
|
||||
}
|
||||
else if (jlu->jlu_shared_buffer.contains((const char *)str)) {
|
||||
shared_buffer_ref sbr;
|
||||
@ -1138,25 +1124,32 @@ static int rewrite_json_field(yajlpp_parse_context *ypc, const unsigned char *st
|
||||
(off_t) ((const char *)str - jlu->jlu_line_value),
|
||||
len);
|
||||
if (field_name == jlu->jlu_format->elf_body_field) {
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(body_name, sbr);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(body_name, value_kind_t::VALUE_TEXT),
|
||||
sbr);
|
||||
}
|
||||
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, sbr);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_TEXT), sbr);
|
||||
}
|
||||
else {
|
||||
tmp_shared_buffer tsb((const char *)str, len);
|
||||
|
||||
if (field_name == jlu->jlu_format->elf_body_field) {
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(body_name, tsb.tsb_ref);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(body_name, value_kind_t::VALUE_TEXT),
|
||||
tsb.tsb_ref);
|
||||
}
|
||||
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(field_name, tsb.tsb_ref);
|
||||
jlu->jlu_format->jlf_line_values.emplace_back(
|
||||
jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_TEXT),
|
||||
tsb.tsb_ref);
|
||||
}
|
||||
|
||||
return 1;
|
||||
@ -1227,16 +1220,7 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
|
||||
for (lv_iter = this->jlf_line_values.begin();
|
||||
lv_iter != this->jlf_line_values.end();
|
||||
++lv_iter) {
|
||||
lv_iter->lv_format = this;
|
||||
auto vd_iter = this->elf_value_defs.find(lv_iter->lv_name);
|
||||
if (vd_iter != this->elf_value_defs.end()) {
|
||||
lv_iter->lv_identifier = vd_iter->second->vd_identifier;
|
||||
lv_iter->lv_column = vd_iter->second->vd_column;
|
||||
lv_iter->lv_hidden = vd_iter->second->vd_hidden;
|
||||
lv_iter->lv_user_hidden = vd_iter->second->vd_user_hidden;
|
||||
} else {
|
||||
lv_iter->lv_hidden = this->jlf_hide_extra;
|
||||
}
|
||||
lv_iter->lv_meta.lvm_format = this;
|
||||
}
|
||||
|
||||
int sub_offset = 1 + this->jlf_line_format_init_count;
|
||||
@ -1260,7 +1244,7 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
|
||||
|
||||
lr.lr_start = this->jlf_cached_line.size();
|
||||
|
||||
lv_iter->lv_hidden = lv_iter->lv_user_hidden;
|
||||
lv_iter->lv_meta.lvm_hidden = lv_iter->lv_meta.lvm_user_hidden;
|
||||
if ((int)str.size() > jfe.jfe_max_width) {
|
||||
switch (jfe.jfe_overflow) {
|
||||
case json_format_element::overflow_t::ABBREV: {
|
||||
@ -1303,15 +1287,15 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
|
||||
lr.lr_end = lr.lr_start + nl_pos;
|
||||
}
|
||||
|
||||
if (lv_iter->lv_name == this->lf_timestamp_field) {
|
||||
if (lv_iter->lv_meta.lvm_name == this->lf_timestamp_field) {
|
||||
this->jlf_line_attrs.emplace_back(
|
||||
lr, &logline::L_TIMESTAMP);
|
||||
}
|
||||
else if (lv_iter->lv_name == this->elf_body_field) {
|
||||
else if (lv_iter->lv_meta.lvm_name == this->elf_body_field) {
|
||||
this->jlf_line_attrs.emplace_back(
|
||||
lr, &SA_BODY);
|
||||
}
|
||||
else if (lv_iter->lv_name == this->elf_opid_field) {
|
||||
else if (lv_iter->lv_meta.lvm_name == this->elf_opid_field) {
|
||||
this->jlf_line_attrs.emplace_back(
|
||||
lr, &logline::L_OPID);
|
||||
}
|
||||
@ -1389,7 +1373,7 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
|
||||
"body", -1);
|
||||
logline_value &lv = this->jlf_line_values[lpc];
|
||||
|
||||
if (lv.lv_hidden || used_values[lpc] || body_name == lv.lv_name) {
|
||||
if (lv.lv_meta.lvm_hidden || used_values[lpc] || body_name == lv.lv_meta.lvm_name) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -1397,7 +1381,7 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
|
||||
size_t curr_pos = 0, nl_pos, line_len = -1;
|
||||
|
||||
lv.lv_sub_offset = sub_offset;
|
||||
lv.lv_origin.lr_start = 2 + lv.lv_name.size() + 2;
|
||||
lv.lv_origin.lr_start = 2 + lv.lv_meta.lvm_name.size() + 2;
|
||||
do {
|
||||
nl_pos = str.find('\n', curr_pos);
|
||||
if (nl_pos != std::string::npos) {
|
||||
@ -1407,8 +1391,8 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
|
||||
line_len = str.size() - curr_pos;
|
||||
}
|
||||
this->json_append_to_cache(" ", 2);
|
||||
this->json_append_to_cache(lv.lv_name.get(),
|
||||
lv.lv_name.size());
|
||||
this->json_append_to_cache(lv.lv_meta.lvm_name.get(),
|
||||
lv.lv_meta.lvm_name.size());
|
||||
this->json_append_to_cache(": ", 2);
|
||||
this->json_append_to_cache(
|
||||
&str.c_str()[curr_pos], line_len);
|
||||
@ -1466,29 +1450,41 @@ void external_log_format::build(std::vector<std::string> &errors) {
|
||||
if (!this->lf_timestamp_field.empty()) {
|
||||
auto &vd = this->elf_value_defs[this->lf_timestamp_field];
|
||||
if (vd.get() == nullptr) {
|
||||
vd = make_shared<external_log_format::value_def>();
|
||||
vd = make_shared<external_log_format::value_def>(
|
||||
this->lf_timestamp_field,
|
||||
value_kind_t::VALUE_TEXT,
|
||||
-1,
|
||||
this);
|
||||
}
|
||||
vd->vd_name = this->lf_timestamp_field;
|
||||
vd->vd_kind = logline_value::VALUE_TEXT;
|
||||
vd->vd_meta.lvm_name = this->lf_timestamp_field;
|
||||
vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
|
||||
vd->vd_internal = true;
|
||||
}
|
||||
if (!this->elf_level_field.empty() && this->elf_value_defs.
|
||||
find(this->elf_level_field) == this->elf_value_defs.end()) {
|
||||
auto &vd = this->elf_value_defs[this->elf_level_field];
|
||||
if (vd.get() == nullptr) {
|
||||
vd = make_shared<external_log_format::value_def>();
|
||||
vd = make_shared<external_log_format::value_def>(
|
||||
this->elf_level_field,
|
||||
value_kind_t::VALUE_TEXT,
|
||||
-1,
|
||||
this);
|
||||
}
|
||||
vd->vd_name = this->elf_level_field;
|
||||
vd->vd_kind = logline_value::VALUE_TEXT;
|
||||
vd->vd_meta.lvm_name = this->elf_level_field;
|
||||
vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
|
||||
vd->vd_internal = true;
|
||||
}
|
||||
if (!this->elf_body_field.empty()) {
|
||||
auto &vd = this->elf_value_defs[this->elf_body_field];
|
||||
if (vd.get() == nullptr) {
|
||||
vd = make_shared<external_log_format::value_def>();
|
||||
vd = make_shared<external_log_format::value_def>(
|
||||
this->elf_body_field,
|
||||
value_kind_t::VALUE_TEXT,
|
||||
-1,
|
||||
this);
|
||||
}
|
||||
vd->vd_name = this->elf_body_field;
|
||||
vd->vd_kind = logline_value::VALUE_TEXT;
|
||||
vd->vd_meta.lvm_name = this->elf_body_field;
|
||||
vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
|
||||
vd->vd_internal = true;
|
||||
}
|
||||
|
||||
@ -1568,8 +1564,8 @@ void external_log_format::build(std::vector<std::string> &errors) {
|
||||
else {
|
||||
ivd.ivd_unit_field_index = -1;
|
||||
}
|
||||
if (!vd->vd_internal && vd->vd_column == -1) {
|
||||
vd->vd_column = this->elf_column_count++;
|
||||
if (!vd->vd_internal && vd->vd_meta.lvm_column == -1) {
|
||||
vd->vd_meta.lvm_column = this->elf_column_count++;
|
||||
}
|
||||
ivd.ivd_value_def = vd;
|
||||
pat.p_value_by_index.push_back(ivd);
|
||||
@ -1582,10 +1578,10 @@ void external_log_format::build(std::vector<std::string> &errors) {
|
||||
auto &ivd = pat.p_value_by_index[lpc];
|
||||
auto vd = ivd.ivd_value_def;
|
||||
|
||||
if (!vd->vd_foreign_key && !vd->vd_identifier) {
|
||||
switch (vd->vd_kind) {
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case logline_value::VALUE_FLOAT:
|
||||
if (!vd->vd_foreign_key && !vd->vd_meta.lvm_identifier) {
|
||||
switch (vd->vd_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
pat.p_numeric_value_indexes.push_back(lpc);
|
||||
break;
|
||||
default:
|
||||
@ -1657,12 +1653,12 @@ void external_log_format::build(std::vector<std::string> &errors) {
|
||||
std::vector<std::string>::iterator act_iter;
|
||||
|
||||
if (!vd->vd_internal &&
|
||||
vd->vd_column == -1) {
|
||||
vd->vd_column = this->elf_column_count++;
|
||||
vd->vd_meta.lvm_column == -1) {
|
||||
vd->vd_meta.lvm_column = this->elf_column_count++;
|
||||
}
|
||||
|
||||
if (vd->vd_kind == logline_value::VALUE_UNKNOWN) {
|
||||
vd->vd_kind = logline_value::VALUE_TEXT;
|
||||
if (vd->vd_meta.lvm_kind == value_kind_t::VALUE_UNKNOWN) {
|
||||
vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
|
||||
}
|
||||
|
||||
for (act_iter = vd->vd_action_list.begin();
|
||||
@ -1672,7 +1668,7 @@ void external_log_format::build(std::vector<std::string> &errors) {
|
||||
this->lf_action_defs.end()) {
|
||||
errors.push_back("error:" +
|
||||
this->elf_name.to_string() + ":" +
|
||||
vd->vd_name.get() +
|
||||
vd->vd_meta.lvm_name.get() +
|
||||
": cannot find action -- " + (*act_iter));
|
||||
}
|
||||
}
|
||||
@ -1821,13 +1817,13 @@ void external_log_format::build(std::vector<std::string> &errors) {
|
||||
}
|
||||
|
||||
for (auto &elf_value_def : this->elf_value_defs) {
|
||||
if (elf_value_def.second->vd_foreign_key || elf_value_def.second->vd_identifier) {
|
||||
if (elf_value_def.second->vd_foreign_key || elf_value_def.second->vd_meta.lvm_identifier) {
|
||||
continue;
|
||||
}
|
||||
|
||||
switch (elf_value_def.second->vd_kind) {
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case logline_value::VALUE_FLOAT:
|
||||
switch (elf_value_def.second->vd_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
elf_value_def.second->vd_values_index = this->elf_numeric_value_defs.size();
|
||||
this->elf_numeric_value_defs.push_back(elf_value_def.second);
|
||||
break;
|
||||
@ -1860,9 +1856,9 @@ void external_log_format::build(std::vector<std::string> &errors) {
|
||||
case JLF_VARIABLE: {
|
||||
auto vd_iter = this->elf_value_defs.find(jfe.jfe_value);
|
||||
if (jfe.jfe_value == ts) {
|
||||
this->elf_value_defs[this->lf_timestamp_field]->vd_hidden = true;
|
||||
this->elf_value_defs[this->lf_timestamp_field]->vd_meta.lvm_hidden = true;
|
||||
} else if (jfe.jfe_value == level_field) {
|
||||
this->elf_value_defs[this->elf_level_field]->vd_hidden = true;
|
||||
this->elf_value_defs[this->elf_level_field]->vd_meta.lvm_hidden = true;
|
||||
} else if (vd_iter == this->elf_value_defs.end()) {
|
||||
char index_str[32];
|
||||
|
||||
@ -2032,19 +2028,19 @@ public:
|
||||
|
||||
cols.resize(elf.elf_column_count);
|
||||
for (const auto &vd : elf.elf_value_def_order) {
|
||||
pair<int, unsigned int> type_pair = log_vtab_impl::logline_value_to_sqlite_type(vd->vd_kind);
|
||||
auto type_pair = log_vtab_impl::logline_value_to_sqlite_type(vd->vd_meta.lvm_kind);
|
||||
|
||||
if (vd->vd_column == -1) {
|
||||
if (vd->vd_meta.lvm_column == -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
require(0 <= vd->vd_column && vd->vd_column < elf.elf_column_count);
|
||||
require(0 <= vd->vd_meta.lvm_column && vd->vd_meta.lvm_column < elf.elf_column_count);
|
||||
|
||||
cols[vd->vd_column].vc_name = vd->vd_name.get();
|
||||
cols[vd->vd_column].vc_type = type_pair.first;
|
||||
cols[vd->vd_column].vc_subtype = type_pair.second;
|
||||
cols[vd->vd_column].vc_collator = vd->vd_collate;
|
||||
cols[vd->vd_column].vc_comment = vd->vd_description;
|
||||
cols[vd->vd_meta.lvm_column].vc_name = vd->vd_meta.lvm_name.get();
|
||||
cols[vd->vd_meta.lvm_column].vc_type = type_pair.first;
|
||||
cols[vd->vd_meta.lvm_column].vc_subtype = type_pair.second;
|
||||
cols[vd->vd_meta.lvm_column].vc_collator = vd->vd_collate;
|
||||
cols[vd->vd_meta.lvm_column].vc_comment = vd->vd_description;
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -93,55 +93,95 @@ struct scaling_factor {
|
||||
double sf_value;
|
||||
};
|
||||
|
||||
enum class value_kind_t : int {
|
||||
VALUE_UNKNOWN = -1,
|
||||
VALUE_NULL,
|
||||
VALUE_TEXT,
|
||||
VALUE_INTEGER,
|
||||
VALUE_FLOAT,
|
||||
VALUE_BOOLEAN,
|
||||
VALUE_JSON,
|
||||
VALUE_STRUCT,
|
||||
VALUE_QUOTED,
|
||||
VALUE_W3C_QUOTED,
|
||||
VALUE_TIMESTAMP,
|
||||
|
||||
VALUE__MAX
|
||||
};
|
||||
|
||||
struct logline_value_meta {
|
||||
logline_value_meta(
|
||||
intern_string_t name,
|
||||
value_kind_t kind,
|
||||
int col = -1,
|
||||
const nonstd::optional<log_format *>& format = nonstd::nullopt)
|
||||
: lvm_name(name), lvm_kind(kind), lvm_column(col), lvm_format(format)
|
||||
{};
|
||||
|
||||
bool is_hidden() const {
|
||||
return this->lvm_hidden || this->lvm_user_hidden;
|
||||
}
|
||||
|
||||
logline_value_meta& with_struct_name(intern_string_t name) {
|
||||
this->lvm_struct_name = name;
|
||||
return *this;
|
||||
}
|
||||
|
||||
intern_string_t lvm_name;
|
||||
value_kind_t lvm_kind;
|
||||
int lvm_column{-1};
|
||||
bool lvm_identifier{false};
|
||||
bool lvm_hidden{false};
|
||||
bool lvm_user_hidden{false};
|
||||
bool lvm_from_module{false};
|
||||
intern_string_t lvm_struct_name;
|
||||
nonstd::optional<log_format *> lvm_format;
|
||||
};
|
||||
|
||||
class logline_value {
|
||||
public:
|
||||
enum kind_t {
|
||||
VALUE_UNKNOWN = -1,
|
||||
VALUE_NULL,
|
||||
VALUE_TEXT,
|
||||
VALUE_INTEGER,
|
||||
VALUE_FLOAT,
|
||||
VALUE_BOOLEAN,
|
||||
VALUE_JSON,
|
||||
VALUE_STRUCT,
|
||||
VALUE_QUOTED,
|
||||
VALUE_W3C_QUOTED,
|
||||
VALUE_TIMESTAMP,
|
||||
|
||||
VALUE__MAX
|
||||
logline_value(logline_value_meta lvm)
|
||||
: lv_meta(std::move(lvm)) {
|
||||
this->lv_meta.lvm_kind = value_kind_t::VALUE_NULL;
|
||||
};
|
||||
|
||||
logline_value(const intern_string_t name, const log_format *format)
|
||||
: lv_name(name), lv_kind(VALUE_NULL), lv_identifier(), lv_column(-1),
|
||||
lv_hidden(false), lv_sub_offset(0), lv_from_module(false), lv_format(format) { };
|
||||
logline_value(const intern_string_t name, bool b, const log_format *format)
|
||||
: lv_name(name),
|
||||
lv_kind(VALUE_BOOLEAN),
|
||||
lv_value((int64_t)(b ? 1 : 0)),
|
||||
lv_identifier(),
|
||||
lv_column(-1),
|
||||
lv_hidden(false), lv_sub_offset(0),
|
||||
lv_from_module(false), lv_format(format) { };
|
||||
logline_value(const intern_string_t name, int64_t i, const log_format *format)
|
||||
: lv_name(name), lv_kind(VALUE_INTEGER), lv_value(i), lv_identifier(), lv_column(-1),
|
||||
lv_hidden(false), lv_sub_offset(0), lv_from_module(false), lv_format(format) { };
|
||||
logline_value(const intern_string_t name, double i, const log_format *format)
|
||||
: lv_name(name), lv_kind(VALUE_FLOAT), lv_value(i), lv_identifier(), lv_column(-1),
|
||||
lv_hidden(false), lv_sub_offset(0), lv_from_module(false), lv_format(format) { };
|
||||
logline_value(const intern_string_t name, shared_buffer_ref &sbr, int column = -1, const log_format *format = nullptr)
|
||||
: lv_name(name), lv_kind(VALUE_TEXT), lv_sbr(sbr),
|
||||
lv_identifier(), lv_column(column),
|
||||
lv_hidden(false), lv_sub_offset(0), lv_from_module(false), lv_format(format) {
|
||||
logline_value(logline_value_meta lvm, bool b)
|
||||
: lv_meta(std::move(lvm)),
|
||||
lv_value((int64_t)(b ? 1 : 0)) {
|
||||
this->lv_meta.lvm_kind = value_kind_t::VALUE_BOOLEAN;
|
||||
}
|
||||
logline_value(logline_value_meta lvm, int64_t i)
|
||||
: lv_meta(std::move(lvm)), lv_value(i) {
|
||||
this->lv_meta.lvm_kind = value_kind_t::VALUE_INTEGER;
|
||||
};
|
||||
logline_value(const intern_string_t name, const intern_string_t val, int column = -1)
|
||||
: lv_name(name), lv_kind(VALUE_TEXT), lv_intern_string(val), lv_identifier(),
|
||||
lv_column(column), lv_hidden(false), lv_sub_offset(0), lv_from_module(false), lv_format(NULL) {
|
||||
logline_value(logline_value_meta lvm, double i)
|
||||
: lv_meta(std::move(lvm)), lv_value(i) {
|
||||
this->lv_meta.lvm_kind = value_kind_t::VALUE_FLOAT;
|
||||
};
|
||||
logline_value(logline_value_meta lvm, shared_buffer_ref &sbr)
|
||||
: lv_meta(std::move(lvm)), lv_sbr(sbr) {
|
||||
};
|
||||
logline_value(logline_value_meta lvm, const intern_string_t val)
|
||||
: lv_meta(std::move(lvm)), lv_intern_string(val) {
|
||||
|
||||
};
|
||||
logline_value(const intern_string_t name, kind_t kind, shared_buffer_ref &sbr,
|
||||
bool ident, const scaling_factor *scaling,
|
||||
int col, int start, int end, bool from_module=false,
|
||||
const log_format *format=NULL);
|
||||
logline_value(logline_value_meta lvm, shared_buffer_ref &sbr,
|
||||
struct line_range origin);
|
||||
|
||||
void apply_scaling(const scaling_factor *sf) {
|
||||
if (sf != nullptr) {
|
||||
switch (this->lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
sf->scale(this->lv_value.i);
|
||||
break;
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
sf->scale(this->lv_value.d);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string to_string() const;
|
||||
|
||||
@ -164,8 +204,7 @@ public:
|
||||
|
||||
struct line_range origin_in_full_msg(const char *msg, ssize_t len) const;
|
||||
|
||||
intern_string_t lv_name;
|
||||
kind_t lv_kind;
|
||||
logline_value_meta lv_meta;
|
||||
union value_u {
|
||||
int64_t i;
|
||||
double d;
|
||||
@ -175,15 +214,9 @@ public:
|
||||
value_u(double d) : d(d) { };
|
||||
} lv_value;
|
||||
shared_buffer_ref lv_sbr;
|
||||
int lv_sub_offset{0};
|
||||
intern_string_t lv_intern_string;
|
||||
bool lv_identifier;
|
||||
int lv_column;
|
||||
bool lv_hidden;
|
||||
bool lv_user_hidden;
|
||||
int lv_sub_offset;
|
||||
struct line_range lv_origin;
|
||||
bool lv_from_module;
|
||||
const log_format *lv_format;
|
||||
};
|
||||
|
||||
struct logline_value_stats {
|
||||
@ -237,19 +270,19 @@ struct logline_value_stats {
|
||||
};
|
||||
|
||||
struct logline_value_cmp {
|
||||
logline_value_cmp(const intern_string_t *name = NULL, int col = -1)
|
||||
explicit logline_value_cmp(const intern_string_t *name = nullptr, int col = -1)
|
||||
: lvc_name(name), lvc_column(col) {
|
||||
|
||||
};
|
||||
|
||||
bool operator()(const logline_value &lv) {
|
||||
bool operator()(const logline_value &lv) const {
|
||||
bool retval = true;
|
||||
|
||||
if (this->lvc_name != NULL) {
|
||||
retval = retval && ((*this->lvc_name) == lv.lv_name);
|
||||
if (this->lvc_name != nullptr) {
|
||||
retval = retval && ((*this->lvc_name) == lv.lv_meta.lvm_name);
|
||||
}
|
||||
if (this->lvc_column != -1) {
|
||||
retval = retval && (this->lvc_column == lv.lv_column);
|
||||
retval = retval && (this->lvc_column == lv.lv_meta.lvm_column);
|
||||
}
|
||||
|
||||
return retval;
|
||||
|
@ -50,17 +50,15 @@ public:
|
||||
};
|
||||
|
||||
struct value_def {
|
||||
intern_string_t vd_name;
|
||||
logline_value::kind_t vd_kind{logline_value::VALUE_UNKNOWN};
|
||||
value_def(intern_string_t name, value_kind_t kind, int col, log_format *format)
|
||||
: vd_meta(name, kind, col, format) {};
|
||||
|
||||
logline_value_meta vd_meta;
|
||||
std::string vd_collate;
|
||||
bool vd_identifier{false};
|
||||
bool vd_foreign_key{false};
|
||||
intern_string_t vd_unit_field;
|
||||
std::map<const intern_string_t, scaling_factor> vd_unit_scaling;
|
||||
int vd_column{-1};
|
||||
ssize_t vd_values_index{-1};
|
||||
bool vd_hidden{false};
|
||||
bool vd_user_hidden{false};
|
||||
bool vd_internal{false};
|
||||
std::vector<std::string> vd_action_list;
|
||||
std::string vd_rewriter;
|
||||
@ -163,7 +161,7 @@ public:
|
||||
return false;
|
||||
}
|
||||
|
||||
vd_iter->second->vd_user_hidden = val;
|
||||
vd_iter->second->vd_meta.lvm_user_hidden = val;
|
||||
return true;
|
||||
};
|
||||
|
||||
@ -175,7 +173,7 @@ public:
|
||||
for (size_t lpc = 0; lpc < this->elf_numeric_value_defs.size(); lpc++) {
|
||||
value_def &vd = *this->elf_numeric_value_defs[lpc];
|
||||
|
||||
if (vd.vd_name == name) {
|
||||
if (vd.vd_meta.lvm_name == name) {
|
||||
retval = &this->lf_value_stats[lpc];
|
||||
break;
|
||||
}
|
||||
@ -191,7 +189,7 @@ public:
|
||||
const std::vector<std::string> *get_actions(const logline_value &lv) const {
|
||||
const std::vector<std::string> *retval = nullptr;
|
||||
|
||||
const auto iter = this->elf_value_defs.find(lv.lv_name);
|
||||
const auto iter = this->elf_value_defs.find(lv.lv_meta.lvm_name);
|
||||
if (iter != this->elf_value_defs.end()) {
|
||||
retval = &iter->second->vd_action_list;
|
||||
}
|
||||
@ -282,7 +280,7 @@ public:
|
||||
return (this->jlf_hide_extra || !top_level) ? 0 : line_count;
|
||||
}
|
||||
|
||||
if (iter->second->vd_hidden) {
|
||||
if (iter->second->vd_meta.lvm_hidden) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -412,6 +410,23 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
logline_value_meta get_value_meta(intern_string_t field_name,
|
||||
value_kind_t kind) {
|
||||
auto iter = this->elf_value_defs.find(field_name);
|
||||
|
||||
if (iter == this->elf_value_defs.end()) {
|
||||
auto retval = logline_value_meta(field_name, kind, -1, this);
|
||||
|
||||
retval.lvm_hidden = this->jlf_hide_extra;
|
||||
return retval;
|
||||
}
|
||||
|
||||
auto lvm = iter->second->vd_meta;
|
||||
|
||||
lvm.lvm_kind = kind;
|
||||
return lvm;
|
||||
}
|
||||
|
||||
bool jlf_hide_extra;
|
||||
std::vector<json_format_element> jlf_line_format;
|
||||
int jlf_line_format_init_count{0};
|
||||
|
@ -34,12 +34,14 @@
|
||||
#include <stdio.h>
|
||||
|
||||
#include <utility>
|
||||
#include <algorithm>
|
||||
|
||||
#include "pcrepp/pcrepp.hh"
|
||||
#include "sql_util.hh"
|
||||
#include "log_format.hh"
|
||||
#include "log_vtab_impl.hh"
|
||||
#include "lnav_util.hh"
|
||||
#include "base/opt_util.hh"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -244,7 +246,7 @@ string from_escaped_string(const char *str, size_t len)
|
||||
return retval;
|
||||
}
|
||||
|
||||
const char *
|
||||
nonstd::optional<const char *>
|
||||
lnav_strnstr(const char *s, const char *find, size_t slen)
|
||||
{
|
||||
char c, sc;
|
||||
@ -254,13 +256,15 @@ lnav_strnstr(const char *s, const char *find, size_t slen)
|
||||
len = strlen(find);
|
||||
do {
|
||||
do {
|
||||
if (slen < 1 || (sc = *s) == '\0')
|
||||
return (nullptr);
|
||||
if (slen < 1 || (sc = *s) == '\0') {
|
||||
return nonstd::nullopt;
|
||||
}
|
||||
--slen;
|
||||
++s;
|
||||
} while (sc != c);
|
||||
if (len > slen)
|
||||
return (nullptr);
|
||||
if (len > slen) {
|
||||
return nonstd::nullopt;
|
||||
}
|
||||
} while (strncmp(s, find, len) != 0);
|
||||
s--;
|
||||
}
|
||||
@ -273,7 +277,7 @@ struct separated_string {
|
||||
const char *ss_separator;
|
||||
size_t ss_separator_len;
|
||||
|
||||
explicit separated_string(const char *str = nullptr, size_t len = -1)
|
||||
separated_string(const char *str, size_t len)
|
||||
: ss_str(str), ss_len(len), ss_separator(",") {
|
||||
this->ss_separator_len = strlen(this->ss_separator);
|
||||
};
|
||||
@ -297,14 +301,14 @@ struct separated_string {
|
||||
|
||||
void update() {
|
||||
const separated_string &ss = this->i_parent;
|
||||
const char *next_field;
|
||||
|
||||
next_field = lnav_strnstr(this->i_pos, ss.ss_separator,
|
||||
ss.ss_len - (this->i_pos - ss.ss_str));
|
||||
if (next_field == nullptr) {
|
||||
this->i_next_pos = ss.ss_str + ss.ss_len;
|
||||
auto next_field = lnav_strnstr(
|
||||
this->i_pos,
|
||||
ss.ss_separator,
|
||||
ss.ss_len - (this->i_pos - ss.ss_str));
|
||||
if (next_field) {
|
||||
this->i_next_pos = next_field.value() + ss.ss_separator_len;
|
||||
} else {
|
||||
this->i_next_pos = next_field + ss.ss_separator_len;
|
||||
this->i_next_pos = ss.ss_str + ss.ss_len;
|
||||
}
|
||||
};
|
||||
|
||||
@ -355,24 +359,20 @@ class bro_log_format : public log_format {
|
||||
public:
|
||||
|
||||
struct field_def {
|
||||
const intern_string_t fd_name;
|
||||
logline_value::kind_t fd_kind;
|
||||
bool fd_identifier;
|
||||
logline_value_meta fd_meta;
|
||||
std::string fd_collator;
|
||||
int fd_numeric_index;
|
||||
|
||||
explicit field_def(const intern_string_t name)
|
||||
: fd_name(name),
|
||||
fd_kind(logline_value::VALUE_TEXT),
|
||||
fd_identifier(false),
|
||||
explicit field_def(const intern_string_t name, int col, log_format *format)
|
||||
: fd_meta(name, value_kind_t::VALUE_TEXT, col, format),
|
||||
fd_numeric_index(-1) {
|
||||
};
|
||||
|
||||
field_def &with_kind(logline_value::kind_t kind,
|
||||
field_def &with_kind(value_kind_t kind,
|
||||
bool identifier = false,
|
||||
const std::string &collator = "") {
|
||||
this->fd_kind = kind;
|
||||
this->fd_identifier = identifier;
|
||||
this->fd_meta.lvm_kind = kind;
|
||||
this->fd_meta.lvm_identifier = identifier;
|
||||
this->fd_collator = collator;
|
||||
return *this;
|
||||
};
|
||||
@ -425,9 +425,9 @@ public:
|
||||
break;
|
||||
}
|
||||
|
||||
const field_def &fd = this->blf_field_defs[iter.index()];
|
||||
const auto &fd = this->blf_field_defs[iter.index()];
|
||||
|
||||
if (TS == fd.fd_name) {
|
||||
if (TS == fd.fd_meta.lvm_name) {
|
||||
string_fragment sf = *iter;
|
||||
|
||||
if (this->lf_date_time.scan(sf.data(),
|
||||
@ -438,22 +438,22 @@ public:
|
||||
this->lf_timestamp_flags = tm.et_flags;
|
||||
found_ts = true;
|
||||
}
|
||||
} else if (STATUS_CODE == fd.fd_name) {
|
||||
} else if (STATUS_CODE == fd.fd_meta.lvm_name) {
|
||||
string_fragment sf = *iter;
|
||||
|
||||
if (!sf.empty() && sf[0] >= '4') {
|
||||
level = LEVEL_ERROR;
|
||||
}
|
||||
} else if (UID == fd.fd_name) {
|
||||
} else if (UID == fd.fd_meta.lvm_name) {
|
||||
string_fragment sf = *iter;
|
||||
|
||||
opid = hash_str(sf.data(), sf.length());
|
||||
}
|
||||
|
||||
if (fd.fd_numeric_index >= 0) {
|
||||
switch (fd.fd_kind) {
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case logline_value::VALUE_FLOAT: {
|
||||
switch (fd.fd_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_FLOAT: {
|
||||
string_fragment sf = *iter;
|
||||
char field_copy[sf.length() + 1];
|
||||
double val;
|
||||
@ -548,7 +548,10 @@ public:
|
||||
this->blf_format_name = intern_string::lookup(full_name);
|
||||
} else if (directive == "#fields") {
|
||||
do {
|
||||
this->blf_field_defs.emplace_back(intern_string::lookup("bro_" + sql_safe_ident(*iter)));
|
||||
this->blf_field_defs.emplace_back(
|
||||
intern_string::lookup("bro_" + sql_safe_ident(*iter)),
|
||||
this->blf_field_defs.size(),
|
||||
this);
|
||||
++iter;
|
||||
} while (iter != ss.end());
|
||||
} else if (directive == "#types") {
|
||||
@ -576,26 +579,26 @@ public:
|
||||
|
||||
do {
|
||||
string_fragment field_type = *iter;
|
||||
field_def &fd = this->blf_field_defs[iter.index() - 1];
|
||||
auto &fd = this->blf_field_defs[iter.index() - 1];
|
||||
|
||||
if (field_type == "time") {
|
||||
fd.with_kind(logline_value::VALUE_TIMESTAMP);
|
||||
fd.with_kind(value_kind_t::VALUE_TIMESTAMP);
|
||||
} else if (field_type == "string") {
|
||||
bool ident = binary_search(begin(KNOWN_IDS), end(KNOWN_IDS), fd.fd_name);
|
||||
fd.with_kind(logline_value::VALUE_TEXT, ident);
|
||||
bool ident = binary_search(begin(KNOWN_IDS), end(KNOWN_IDS), fd.fd_meta.lvm_name);
|
||||
fd.with_kind(value_kind_t::VALUE_TEXT, ident);
|
||||
} else if (field_type == "count") {
|
||||
bool ident = binary_search(begin(KNOWN_IDS), end(KNOWN_IDS), fd.fd_name);
|
||||
fd.with_kind(logline_value::VALUE_INTEGER, ident)
|
||||
bool ident = binary_search(begin(KNOWN_IDS), end(KNOWN_IDS), fd.fd_meta.lvm_name);
|
||||
fd.with_kind(value_kind_t::VALUE_INTEGER, ident)
|
||||
.with_numeric_index(numeric_count);
|
||||
numeric_count += 1;
|
||||
} else if (field_type == "bool") {
|
||||
fd.with_kind(logline_value::VALUE_BOOLEAN);
|
||||
fd.with_kind(value_kind_t::VALUE_BOOLEAN);
|
||||
} else if (field_type == "addr") {
|
||||
fd.with_kind(logline_value::VALUE_TEXT, true, "ipaddress");
|
||||
fd.with_kind(value_kind_t::VALUE_TEXT, true, "ipaddress");
|
||||
} else if (field_type == "port") {
|
||||
fd.with_kind(logline_value::VALUE_INTEGER, true);
|
||||
fd.with_kind(value_kind_t::VALUE_INTEGER, true);
|
||||
} else if (field_type == "interval") {
|
||||
fd.with_kind(logline_value::VALUE_FLOAT)
|
||||
fd.with_kind(value_kind_t::VALUE_FLOAT)
|
||||
.with_numeric_index(numeric_count);
|
||||
numeric_count += 1;
|
||||
}
|
||||
@ -636,36 +639,27 @@ public:
|
||||
|
||||
const field_def &fd = this->blf_field_defs[iter.index()];
|
||||
string_fragment sf = *iter;
|
||||
logline_value::kind_t kind = fd.fd_kind;
|
||||
auto kind = fd.fd_meta.lvm_kind;
|
||||
|
||||
if (sf == this->blf_empty_field) {
|
||||
sf.clear();
|
||||
} else if (sf == this->blf_unset_field) {
|
||||
sf.invalidate();
|
||||
kind = logline_value::VALUE_NULL;
|
||||
kind = value_kind_t::VALUE_NULL;
|
||||
}
|
||||
|
||||
auto lr = line_range(sf.sf_begin, sf.sf_end);
|
||||
|
||||
if (fd.fd_name == TS) {
|
||||
if (fd.fd_meta.lvm_name == TS) {
|
||||
sa.emplace_back(lr, &logline::L_TIMESTAMP);
|
||||
} else if (fd.fd_name == UID) {
|
||||
} else if (fd.fd_meta.lvm_name == UID) {
|
||||
sa.emplace_back(lr, &logline::L_OPID);
|
||||
}
|
||||
|
||||
if (lr.is_valid()) {
|
||||
values.emplace_back(fd.fd_name,
|
||||
kind,
|
||||
sbr,
|
||||
fd.fd_identifier,
|
||||
nullptr,
|
||||
iter.index(),
|
||||
lr.lr_start,
|
||||
lr.lr_end,
|
||||
false,
|
||||
this);
|
||||
values.emplace_back(fd.fd_meta, sbr, lr);
|
||||
} else {
|
||||
values.emplace_back(fd.fd_name, this);
|
||||
values.emplace_back(fd.fd_meta);
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -674,7 +668,7 @@ public:
|
||||
const logline_value_stats *retval = nullptr;
|
||||
|
||||
for (size_t lpc = 0; lpc < this->blf_field_defs.size(); lpc++) {
|
||||
if (this->blf_field_defs[lpc].fd_name == name) {
|
||||
if (this->blf_field_defs[lpc].fd_meta.lvm_name == name) {
|
||||
if (this->blf_field_defs[lpc].fd_numeric_index < 0) {
|
||||
break;
|
||||
}
|
||||
@ -699,9 +693,9 @@ public:
|
||||
|
||||
void get_columns(vector<vtab_column> &cols) const {
|
||||
for (const auto &fd : this->blt_format.blf_field_defs) {
|
||||
std::pair<int, unsigned int> type_pair = log_vtab_impl::logline_value_to_sqlite_type(fd.fd_kind);
|
||||
std::pair<int, unsigned int> type_pair = log_vtab_impl::logline_value_to_sqlite_type(fd.fd_meta.lvm_kind);
|
||||
|
||||
cols.emplace_back(fd.fd_name.to_string(), type_pair.first, fd.fd_collator, false, "", type_pair.second);
|
||||
cols.emplace_back(fd.fd_meta.lvm_name.to_string(), type_pair.first, fd.fd_collator, false, "", type_pair.second);
|
||||
}
|
||||
};
|
||||
|
||||
@ -709,8 +703,8 @@ public:
|
||||
this->log_vtab_impl::get_foreign_keys(keys_inout);
|
||||
|
||||
for (const auto &fd : this->blt_format.blf_field_defs) {
|
||||
if (fd.fd_identifier) {
|
||||
keys_inout.push_back(fd.fd_name.to_string());
|
||||
if (fd.fd_meta.lvm_identifier) {
|
||||
keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -855,35 +849,36 @@ public:
|
||||
|
||||
struct field_def {
|
||||
const intern_string_t fd_name;
|
||||
const intern_string_t fd_sql_name;
|
||||
logline_value::kind_t fd_kind;
|
||||
bool fd_identifier;
|
||||
logline_value_meta fd_meta;
|
||||
std::string fd_collator;
|
||||
int fd_numeric_index;
|
||||
|
||||
explicit field_def(const intern_string_t name)
|
||||
: fd_name(name),
|
||||
fd_sql_name(intern_string::lookup(sql_safe_ident(name.to_string_fragment()))),
|
||||
fd_kind(logline_value::VALUE_TEXT),
|
||||
fd_identifier(false),
|
||||
fd_meta(intern_string::lookup(sql_safe_ident(name.to_string_fragment())),
|
||||
value_kind_t::VALUE_TEXT),
|
||||
fd_numeric_index(-1) {
|
||||
};
|
||||
|
||||
field_def(const char *name, logline_value::kind_t kind, bool ident = false, std::string coll = "")
|
||||
: fd_name(intern_string::lookup(name)),
|
||||
fd_sql_name(intern_string::lookup(sql_safe_ident(string_fragment(name)))),
|
||||
fd_kind(kind),
|
||||
fd_identifier(ident),
|
||||
fd_collator(std::move(coll)),
|
||||
fd_numeric_index(-1) {
|
||||
|
||||
field_def(const intern_string_t name, logline_value_meta meta)
|
||||
: fd_name(name), fd_meta(meta), fd_numeric_index(-1) {
|
||||
}
|
||||
|
||||
field_def &with_kind(logline_value::kind_t kind,
|
||||
field_def(int col, const char *name, value_kind_t kind, bool ident = false, std::string coll = "")
|
||||
: fd_name(intern_string::lookup(name)),
|
||||
fd_meta(intern_string::lookup(sql_safe_ident(string_fragment(name))),
|
||||
kind,
|
||||
col),
|
||||
fd_collator(std::move(coll)),
|
||||
fd_numeric_index(-1) {
|
||||
this->fd_meta.lvm_identifier = ident;
|
||||
}
|
||||
|
||||
field_def &with_kind(value_kind_t kind,
|
||||
bool identifier = false,
|
||||
const std::string &collator = "") {
|
||||
this->fd_kind = kind;
|
||||
this->fd_identifier = identifier;
|
||||
this->fd_meta.lvm_kind = kind;
|
||||
this->fd_meta.lvm_identifier = identifier;
|
||||
this->fd_collator = collator;
|
||||
return *this;
|
||||
};
|
||||
@ -894,6 +889,19 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
struct field_to_struct_t {
|
||||
field_to_struct_t(const char *prefix, const char *struct_name)
|
||||
: fs_prefix(prefix),
|
||||
fs_struct_name(intern_string::lookup(struct_name)) {
|
||||
}
|
||||
|
||||
const char *fs_prefix;
|
||||
intern_string_t fs_struct_name;
|
||||
};
|
||||
|
||||
static const std::vector<field_def> KNOWN_FIELDS;
|
||||
const static std::vector<field_to_struct_t> KNOWN_STRUCT_FIELDS;
|
||||
|
||||
w3c_log_format() {
|
||||
this->lf_is_self_describing = true;
|
||||
this->lf_time_ordered = false;
|
||||
@ -987,9 +995,9 @@ public:
|
||||
}
|
||||
|
||||
if (fd.fd_numeric_index >= 0) {
|
||||
switch (fd.fd_kind) {
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case logline_value::VALUE_FLOAT: {
|
||||
switch (fd.fd_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_FLOAT: {
|
||||
char field_copy[sf.length() + 1];
|
||||
double val;
|
||||
|
||||
@ -1030,86 +1038,9 @@ public:
|
||||
std::vector<logline> &dst,
|
||||
const line_info &li,
|
||||
shared_buffer_ref &sbr) override {
|
||||
static const field_def KNOWN_FIELDS[] = {
|
||||
{
|
||||
"cs-method",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"c-ip",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
"ipaddress",
|
||||
},
|
||||
{
|
||||
"cs-bytes",
|
||||
logline_value::kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"cs-host",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"cs-uri-stem",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
"naturalnocase",
|
||||
},
|
||||
{
|
||||
"cs-uri-query",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"cs-username",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"cs-version",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"s-ip",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
"ipaddress",
|
||||
},
|
||||
{
|
||||
"s-port",
|
||||
logline_value::kind_t::VALUE_INTEGER,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"s-computername",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"s-sitename",
|
||||
logline_value::kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"sc-bytes",
|
||||
logline_value::kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"sc-status",
|
||||
logline_value::kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"time-taken",
|
||||
logline_value::kind_t::VALUE_FLOAT,
|
||||
false,
|
||||
},
|
||||
};
|
||||
static auto W3C_LOG_NAME = intern_string::lookup("w3c_log");
|
||||
static auto X_FIELDS_NAME = intern_string::lookup("x_fields");
|
||||
static auto X_FIELDS_IDX = 0;
|
||||
|
||||
if (!this->wlf_format_name.empty()) {
|
||||
return this->scan_int(dst, li, sbr);
|
||||
@ -1157,13 +1088,11 @@ public:
|
||||
this->wlf_time_scanner.set_base_time(tv.tv_sec);
|
||||
}
|
||||
} else if (directive == "#Fields:") {
|
||||
hasher id_hash;
|
||||
int numeric_count = 0;
|
||||
|
||||
do {
|
||||
string_fragment sf = *iter;
|
||||
|
||||
id_hash.update(sf);
|
||||
sf.trim(")");
|
||||
auto field_iter = std::find_if(begin(KNOWN_FIELDS),
|
||||
end(KNOWN_FIELDS),
|
||||
@ -1172,14 +1101,43 @@ public:
|
||||
});
|
||||
if (field_iter != end(KNOWN_FIELDS)) {
|
||||
this->wlf_field_defs.emplace_back(*field_iter);
|
||||
} else {
|
||||
} else if (sf == "date" || sf == "time") {
|
||||
this->wlf_field_defs.emplace_back(
|
||||
intern_string::lookup(sf));
|
||||
} else {
|
||||
const auto fs_iter = std::find_if(
|
||||
begin(KNOWN_STRUCT_FIELDS),
|
||||
end(KNOWN_STRUCT_FIELDS),
|
||||
[&sf](auto elem) {
|
||||
return sf.startswith(elem.fs_prefix);
|
||||
});
|
||||
if (fs_iter != end(KNOWN_STRUCT_FIELDS)) {
|
||||
auto field_name = intern_string::lookup(sf.substr(3));
|
||||
this->wlf_field_defs.emplace_back(
|
||||
field_name, logline_value_meta(
|
||||
field_name,
|
||||
value_kind_t::VALUE_TEXT,
|
||||
KNOWN_FIELDS.size() + 1 +
|
||||
std::distance(begin(KNOWN_STRUCT_FIELDS), fs_iter),
|
||||
this)
|
||||
.with_struct_name(fs_iter->fs_struct_name));
|
||||
} else {
|
||||
auto field_name = intern_string::lookup(sf);
|
||||
this->wlf_field_defs.emplace_back(
|
||||
field_name,
|
||||
logline_value_meta(field_name,
|
||||
value_kind_t::VALUE_TEXT,
|
||||
KNOWN_FIELDS.size() +
|
||||
X_FIELDS_IDX,
|
||||
this)
|
||||
.with_struct_name(X_FIELDS_NAME));
|
||||
}
|
||||
}
|
||||
auto& fd = this->wlf_field_defs.back();
|
||||
switch (fd.fd_kind) {
|
||||
case logline_value::kind_t::VALUE_FLOAT:
|
||||
case logline_value::kind_t::VALUE_INTEGER:
|
||||
fd.fd_meta.lvm_format = nonstd::make_optional(this);
|
||||
switch (fd.fd_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
fd.with_numeric_index(numeric_count);
|
||||
numeric_count += 1;
|
||||
break;
|
||||
@ -1190,8 +1148,7 @@ public:
|
||||
++iter;
|
||||
} while (iter != ss.end());
|
||||
|
||||
this->wlf_format_name = intern_string::lookup(fmt::format(
|
||||
"w3c_{}_log", id_hash.to_string().substr(0, 6)));
|
||||
this->wlf_format_name = W3C_LOG_NAME;
|
||||
this->lf_value_stats.resize(numeric_count);
|
||||
}
|
||||
}
|
||||
@ -1223,30 +1180,26 @@ public:
|
||||
}
|
||||
|
||||
const field_def &fd = this->wlf_field_defs[iter.index()];
|
||||
logline_value::kind_t kind = fd.fd_kind;
|
||||
|
||||
if (sf == "-") {
|
||||
sf.invalidate();
|
||||
kind = logline_value::VALUE_NULL;
|
||||
}
|
||||
|
||||
auto lr = line_range(sf.sf_begin, sf.sf_end);
|
||||
|
||||
if (lr.is_valid()) {
|
||||
values.emplace_back(fd.fd_sql_name,
|
||||
sf.startswith("\"") ?
|
||||
logline_value::kind_t::VALUE_W3C_QUOTED :
|
||||
kind,
|
||||
sbr,
|
||||
fd.fd_identifier,
|
||||
nullptr,
|
||||
iter.index(),
|
||||
lr.lr_start,
|
||||
lr.lr_end,
|
||||
false,
|
||||
this);
|
||||
values.emplace_back(fd.fd_meta, sbr, lr);
|
||||
if (sf.startswith("\"")) {
|
||||
auto& meta = values.back().lv_meta;
|
||||
|
||||
if (meta.lvm_kind == value_kind_t::VALUE_TEXT) {
|
||||
meta.lvm_kind = value_kind_t::VALUE_W3C_QUOTED;
|
||||
} else {
|
||||
meta.lvm_kind = value_kind_t::VALUE_NULL;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
values.emplace_back(fd.fd_sql_name, this);
|
||||
values.emplace_back(fd.fd_meta);
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -1255,7 +1208,7 @@ public:
|
||||
const logline_value_stats *retval = nullptr;
|
||||
|
||||
for (const auto & wlf_field_def : this->wlf_field_defs) {
|
||||
if (wlf_field_def.fd_sql_name == name) {
|
||||
if (wlf_field_def.fd_meta.lvm_name == name) {
|
||||
if (wlf_field_def.fd_numeric_index < 0) {
|
||||
break;
|
||||
}
|
||||
@ -1279,19 +1232,31 @@ public:
|
||||
}
|
||||
|
||||
void get_columns(vector<vtab_column> &cols) const override {
|
||||
for (const auto &fd : this->wlt_format.wlf_field_defs) {
|
||||
std::pair<int, unsigned int> type_pair = log_vtab_impl::logline_value_to_sqlite_type(fd.fd_kind);
|
||||
for (const auto &fd : KNOWN_FIELDS) {
|
||||
auto type_pair = log_vtab_impl::logline_value_to_sqlite_type(
|
||||
fd.fd_meta.lvm_kind);
|
||||
|
||||
cols.emplace_back(fd.fd_sql_name.to_string(), type_pair.first, fd.fd_collator, false, "", type_pair.second);
|
||||
cols.emplace_back(fd.fd_meta.lvm_name.to_string(),
|
||||
type_pair.first,
|
||||
fd.fd_collator,
|
||||
false,
|
||||
"",
|
||||
type_pair.second);
|
||||
}
|
||||
cols.emplace_back("x_fields");
|
||||
cols.back().with_comment(
|
||||
"A JSON-object that contains fields that are not first-class columns");
|
||||
for (const auto& fs : KNOWN_STRUCT_FIELDS) {
|
||||
cols.emplace_back(fs.fs_struct_name.to_string());
|
||||
}
|
||||
};
|
||||
|
||||
void get_foreign_keys(std::vector<std::string> &keys_inout) const override {
|
||||
this->log_vtab_impl::get_foreign_keys(keys_inout);
|
||||
|
||||
for (const auto &fd : this->wlt_format.wlf_field_defs) {
|
||||
if (fd.fd_identifier) {
|
||||
keys_inout.push_back(fd.fd_sql_name.to_string());
|
||||
for (const auto &fd : KNOWN_FIELDS) {
|
||||
if (fd.fd_meta.lvm_identifier) {
|
||||
keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1332,6 +1297,116 @@ public:
|
||||
vector<field_def> wlf_field_defs;
|
||||
};
|
||||
|
||||
static int KNOWN_FIELD_INDEX = 0;
|
||||
const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"cs-method",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"c-ip",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
"ipaddress",
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"cs-bytes",
|
||||
value_kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"cs-host",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"cs-uri-stem",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
"naturalnocase",
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"cs-uri-query",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
false,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"cs-username",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
false,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"cs-version",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"s-ip",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
"ipaddress",
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"s-port",
|
||||
value_kind_t::VALUE_INTEGER,
|
||||
true,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"s-computername",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"s-sitename",
|
||||
value_kind_t::VALUE_TEXT,
|
||||
true,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"sc-bytes",
|
||||
value_kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"sc-status",
|
||||
value_kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"sc-substatus",
|
||||
value_kind_t::VALUE_INTEGER,
|
||||
false,
|
||||
},
|
||||
{
|
||||
KNOWN_FIELD_INDEX++,
|
||||
"time-taken",
|
||||
value_kind_t::VALUE_FLOAT,
|
||||
false,
|
||||
},
|
||||
};
|
||||
|
||||
const std::vector<w3c_log_format::field_to_struct_t> w3c_log_format::KNOWN_STRUCT_FIELDS = {
|
||||
{"cs(", "cs_headers"},
|
||||
{"sc(", "sc_headers"},
|
||||
{"rs(", "rs_headers"},
|
||||
{"sr(", "sr_headers"},
|
||||
};
|
||||
|
||||
log_format::register_root_format<bro_log_format> bro_log_instance;
|
||||
log_format::register_root_format<w3c_log_format> w3c_log_instance;
|
||||
log_format::register_root_format<generic_log_format> generic_log_instance;
|
||||
|
@ -120,8 +120,8 @@ static external_log_format::value_def *value_def_provider(const yajlpp_provider_
|
||||
shared_ptr<external_log_format::value_def> retval;
|
||||
|
||||
if (iter == elf->elf_value_defs.end()) {
|
||||
retval = make_shared<external_log_format::value_def>();
|
||||
retval->vd_name = value_name;
|
||||
retval = make_shared<external_log_format::value_def>(
|
||||
value_name, value_kind_t::VALUE_TEXT, -1, elf);
|
||||
elf->elf_value_defs[value_name] = retval;
|
||||
elf->elf_value_def_order.emplace_back(retval);
|
||||
} else {
|
||||
@ -133,18 +133,9 @@ static external_log_format::value_def *value_def_provider(const yajlpp_provider_
|
||||
|
||||
static scaling_factor *scaling_factor_provider(const yajlpp_provider_context &ypc, external_log_format::value_def *value_def)
|
||||
{
|
||||
string scale_spec = ypc.get_substr(0);
|
||||
|
||||
const intern_string_t scale_name = intern_string::lookup(scale_spec.substr(1));
|
||||
auto scale_name = ypc.get_substr_i(0);
|
||||
scaling_factor &retval = value_def->vd_unit_scaling[scale_name];
|
||||
|
||||
if (scale_spec[0] == '/') {
|
||||
retval.sf_op = scale_op_t::SO_DIVIDE;
|
||||
}
|
||||
else if (scale_spec[0] == '*') {
|
||||
retval.sf_op = scale_op_t::SO_MULTIPLY;
|
||||
}
|
||||
|
||||
return &retval;
|
||||
}
|
||||
|
||||
@ -366,7 +357,7 @@ static struct json_path_container pattern_handlers = {
|
||||
.with_description(
|
||||
"If true, this pattern will only be used to parse message bodies "
|
||||
"of container formats, like syslog")
|
||||
.FOR_FIELD(external_log_format::pattern, p_module_format)
|
||||
.for_field(&external_log_format::pattern::p_module_format)
|
||||
};
|
||||
|
||||
static const json_path_handler_base::enum_value_t ALIGN_ENUM[] = {
|
||||
@ -443,23 +434,40 @@ static struct json_path_container line_format_handlers = {
|
||||
};
|
||||
|
||||
static const json_path_handler_base::enum_value_t KIND_ENUM[] = {
|
||||
{"string", logline_value::VALUE_TEXT},
|
||||
{"integer", logline_value::VALUE_INTEGER},
|
||||
{"float", logline_value::VALUE_FLOAT},
|
||||
{"boolean", logline_value::VALUE_BOOLEAN},
|
||||
{"json", logline_value::VALUE_JSON},
|
||||
{"struct", logline_value::VALUE_STRUCT},
|
||||
{"quoted", logline_value::VALUE_QUOTED},
|
||||
{"string", value_kind_t::VALUE_TEXT},
|
||||
{"integer", value_kind_t::VALUE_INTEGER},
|
||||
{"float", value_kind_t::VALUE_FLOAT},
|
||||
{"boolean", value_kind_t::VALUE_BOOLEAN},
|
||||
{"json", value_kind_t::VALUE_JSON},
|
||||
{"struct", value_kind_t::VALUE_STRUCT},
|
||||
{"quoted", value_kind_t::VALUE_QUOTED},
|
||||
|
||||
json_path_handler_base::ENUM_TERMINATOR
|
||||
};
|
||||
|
||||
static struct json_path_container scale_handlers = {
|
||||
yajlpp::pattern_property_handler("(?<scale>.*)")
|
||||
.with_synopsis("[*,/]<unit>")
|
||||
static const json_path_handler_base::enum_value_t SCALE_OP_ENUM[] = {
|
||||
{"identity", scale_op_t::SO_IDENTITY},
|
||||
{"multiply", scale_op_t::SO_MULTIPLY},
|
||||
{"divide", scale_op_t::SO_DIVIDE},
|
||||
|
||||
json_path_handler_base::ENUM_TERMINATOR
|
||||
};
|
||||
|
||||
static struct json_path_container scaling_factor_handlers = {
|
||||
yajlpp::pattern_property_handler("op")
|
||||
.with_enum_values(SCALE_OP_ENUM)
|
||||
.FOR_FIELD(scaling_factor, sf_op),
|
||||
|
||||
yajlpp::pattern_property_handler("value")
|
||||
.FOR_FIELD(scaling_factor, sf_value)
|
||||
};
|
||||
|
||||
static struct json_path_container scale_handlers = {
|
||||
yajlpp::pattern_property_handler("(?<scale>[^/]+)")
|
||||
.with_obj_provider(scaling_factor_provider)
|
||||
.with_children(scaling_factor_handlers),
|
||||
};
|
||||
|
||||
static struct json_path_container unit_handlers = {
|
||||
yajlpp::property_handler("field")
|
||||
.with_synopsis("<field-name>")
|
||||
@ -468,7 +476,6 @@ static struct json_path_container unit_handlers = {
|
||||
|
||||
yajlpp::property_handler("scaling-factor")
|
||||
.with_description("Transforms the numeric value by the given factor")
|
||||
.with_obj_provider(scaling_factor_provider)
|
||||
.with_children(scale_handlers),
|
||||
};
|
||||
|
||||
@ -477,7 +484,8 @@ static struct json_path_container value_def_handlers = {
|
||||
.with_synopsis("string|integer|float|boolean|json|quoted")
|
||||
.with_description("The type of data in the field")
|
||||
.with_enum_values(KIND_ENUM)
|
||||
.FOR_FIELD(external_log_format::value_def, vd_kind),
|
||||
.for_field(&external_log_format::value_def::vd_meta,
|
||||
&logline_value_meta::lvm_kind),
|
||||
|
||||
yajlpp::property_handler("collate")
|
||||
.with_synopsis("<function>")
|
||||
@ -491,7 +499,8 @@ static struct json_path_container value_def_handlers = {
|
||||
yajlpp::property_handler("identifier")
|
||||
.with_synopsis("<bool>")
|
||||
.with_description("Indicates whether or not this field contains an identifier that should be highlighted")
|
||||
.FOR_FIELD(external_log_format::value_def, vd_identifier),
|
||||
.for_field(&external_log_format::value_def::vd_meta,
|
||||
&logline_value_meta::lvm_identifier),
|
||||
|
||||
yajlpp::property_handler("foreign-key")
|
||||
.with_synopsis("<bool>")
|
||||
@ -501,7 +510,8 @@ static struct json_path_container value_def_handlers = {
|
||||
yajlpp::property_handler("hidden")
|
||||
.with_synopsis("<bool>")
|
||||
.with_description("Indicates whether or not this field should be hidden")
|
||||
.FOR_FIELD(external_log_format::value_def, vd_hidden),
|
||||
.for_field(&external_log_format::value_def::vd_meta,
|
||||
&logline_value_meta::lvm_hidden),
|
||||
|
||||
yajlpp::property_handler("action-list#")
|
||||
.with_synopsis("<string>")
|
||||
@ -538,12 +548,12 @@ static struct json_path_container highlighter_def_handlers = {
|
||||
yajlpp::property_handler("underline")
|
||||
.with_synopsis("<enabled>")
|
||||
.with_description("Highlight this pattern with an underline.")
|
||||
.FOR_FIELD(external_log_format::highlighter_def, hd_underline),
|
||||
.for_field(&external_log_format::highlighter_def::hd_underline),
|
||||
|
||||
yajlpp::property_handler("blink")
|
||||
.with_synopsis("<enabled>")
|
||||
.with_description("Highlight this pattern by blinking.")
|
||||
.FOR_FIELD(external_log_format::highlighter_def, hd_blink)
|
||||
.for_field(&external_log_format::highlighter_def::hd_blink)
|
||||
};
|
||||
|
||||
static const json_path_handler_base::enum_value_t LEVEL_ENUM[] = {
|
||||
@ -681,7 +691,7 @@ struct json_path_container format_handlers = {
|
||||
yajlpp::property_handler("ordered-by-time")
|
||||
.with_synopsis("<bool>")
|
||||
.with_description("Indicates that the order of messages in the file is time-based.")
|
||||
.FOR_FIELD(log_format, lf_time_ordered),
|
||||
.for_field(&log_format::lf_time_ordered),
|
||||
yajlpp::property_handler("level")
|
||||
.with_description("The map of level names to patterns or integer values")
|
||||
.with_children(level_handlers),
|
||||
@ -937,10 +947,6 @@ void load_formats(const std::vector<ghc::filesystem::path> &extra_paths,
|
||||
load_from_path(extra_path, errors);
|
||||
}
|
||||
|
||||
if (!errors.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t mod_counter = 0;
|
||||
|
||||
vector<std::shared_ptr<external_log_format>> alpha_ordered_formats;
|
||||
@ -972,6 +978,10 @@ void load_formats(const std::vector<ghc::filesystem::path> &extra_paths,
|
||||
}
|
||||
}
|
||||
|
||||
if (!errors.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto& graph_ordered_formats = external_log_format::GRAPH_ORDERED_FORMATS;
|
||||
|
||||
while (!alpha_ordered_formats.empty()) {
|
||||
|
@ -35,6 +35,8 @@
|
||||
|
||||
const static std::string LOG_MSG_INSTANCE = "log_msg_instance";
|
||||
static auto instance_name = intern_string::lookup("log_msg_instance");
|
||||
static auto instance_meta = logline_value_meta(
|
||||
instance_name, value_kind_t::VALUE_INTEGER, 0);
|
||||
static auto empty = intern_string::lookup("", 0);
|
||||
|
||||
log_search_table::log_search_table(const char *regex,
|
||||
@ -58,6 +60,7 @@ void log_search_table::get_columns_int(std::vector<vtab_column> &cols)
|
||||
std::string colname;
|
||||
int sqlite_type = SQLITE3_TEXT;
|
||||
|
||||
colname = cn.add_column(this->lst_regex.name_for_capture(lpc));
|
||||
if (this->lst_regex.captures().size() ==
|
||||
(size_t) this->lst_regex.get_capture_count()) {
|
||||
auto iter = this->lst_regex.cap_begin() + lpc;
|
||||
@ -66,20 +69,25 @@ void log_search_table::get_columns_int(std::vector<vtab_column> &cols)
|
||||
sqlite_type = guess_type_from_pcre(cap_re, collator);
|
||||
switch (sqlite_type) {
|
||||
case SQLITE_FLOAT:
|
||||
this->lst_column_types.push_back(
|
||||
logline_value::VALUE_FLOAT);
|
||||
this->lst_column_metas.emplace_back(
|
||||
intern_string::lookup(colname),
|
||||
value_kind_t::VALUE_FLOAT,
|
||||
cols.size());
|
||||
break;
|
||||
case SQLITE_INTEGER:
|
||||
this->lst_column_types.push_back(
|
||||
logline_value::VALUE_INTEGER);
|
||||
this->lst_column_metas.emplace_back(
|
||||
intern_string::lookup(colname),
|
||||
value_kind_t::VALUE_INTEGER,
|
||||
cols.size());
|
||||
break;
|
||||
default:
|
||||
this->lst_column_types.push_back(
|
||||
logline_value::VALUE_TEXT);
|
||||
this->lst_column_metas.emplace_back(
|
||||
intern_string::lookup(colname),
|
||||
value_kind_t::VALUE_TEXT,
|
||||
cols.size());
|
||||
break;
|
||||
}
|
||||
}
|
||||
colname = cn.add_column(this->lst_regex.name_for_capture(lpc));
|
||||
cols.emplace_back(colname, sqlite_type, collator);
|
||||
}
|
||||
}
|
||||
@ -136,14 +144,10 @@ log_search_table::extract(std::shared_ptr<logfile> lf, uint64_t line_number,
|
||||
shared_buffer_ref &line,
|
||||
std::vector<logline_value> &values)
|
||||
{
|
||||
int next_column = 0;
|
||||
|
||||
values.emplace_back(instance_name, this->lst_instance, lf->get_format().get());
|
||||
values.back().lv_column = next_column++;
|
||||
values.emplace_back(instance_meta, this->lst_instance);
|
||||
for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) {
|
||||
auto cap = this->lst_match_context[lpc];
|
||||
values.emplace_back(empty, this->lst_column_types[lpc], line,
|
||||
false, nullptr, -1, cap->c_begin, cap->c_end);
|
||||
values.back().lv_column = next_column++;
|
||||
values.emplace_back(this->lst_column_metas[lpc], line,
|
||||
line_range{cap->c_begin, cap->c_end});
|
||||
}
|
||||
}
|
||||
|
@ -62,7 +62,7 @@ public:
|
||||
pcrepp lst_regex;
|
||||
shared_buffer_ref lst_current_line;
|
||||
pcre_context_static<128> lst_match_context;
|
||||
std::vector<logline_value::kind_t> lst_column_types;
|
||||
std::vector<logline_value_meta> lst_column_metas;
|
||||
int64_t lst_instance;
|
||||
std::vector<vtab_column> lst_cols;
|
||||
};
|
||||
|
@ -127,33 +127,33 @@ std::string log_vtab_impl::get_table_statement()
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
pair<int, unsigned int> log_vtab_impl::logline_value_to_sqlite_type(logline_value::kind_t kind)
|
||||
pair<int, unsigned int> log_vtab_impl::logline_value_to_sqlite_type(value_kind_t kind)
|
||||
{
|
||||
int type = 0;
|
||||
unsigned int subtype = 0;
|
||||
|
||||
switch (kind) {
|
||||
case logline_value::VALUE_JSON:
|
||||
case value_kind_t::VALUE_JSON:
|
||||
type = SQLITE3_TEXT;
|
||||
subtype = 74;
|
||||
break;
|
||||
case logline_value::VALUE_NULL:
|
||||
case logline_value::VALUE_TEXT:
|
||||
case logline_value::VALUE_STRUCT:
|
||||
case logline_value::VALUE_QUOTED:
|
||||
case logline_value::VALUE_W3C_QUOTED:
|
||||
case logline_value::VALUE_TIMESTAMP:
|
||||
case value_kind_t::VALUE_NULL:
|
||||
case value_kind_t::VALUE_TEXT:
|
||||
case value_kind_t::VALUE_STRUCT:
|
||||
case value_kind_t::VALUE_QUOTED:
|
||||
case value_kind_t::VALUE_W3C_QUOTED:
|
||||
case value_kind_t::VALUE_TIMESTAMP:
|
||||
type = SQLITE3_TEXT;
|
||||
break;
|
||||
case logline_value::VALUE_FLOAT:
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
type = SQLITE_FLOAT;
|
||||
break;
|
||||
case logline_value::VALUE_BOOLEAN:
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
type = SQLITE_INTEGER;
|
||||
break;
|
||||
case logline_value::VALUE_UNKNOWN:
|
||||
case logline_value::VALUE__MAX:
|
||||
case value_kind_t::VALUE_UNKNOWN:
|
||||
case value_kind_t::VALUE__MAX:
|
||||
ensure(0);
|
||||
break;
|
||||
}
|
||||
@ -573,76 +573,121 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
|
||||
logline_value_cmp(NULL, sub_col));
|
||||
|
||||
if (lv_iter != vc->line_values.end()) {
|
||||
switch (lv_iter->lv_kind) {
|
||||
case logline_value::VALUE_NULL:
|
||||
sqlite3_result_null(ctx);
|
||||
break;
|
||||
case logline_value::VALUE_JSON: {
|
||||
if (!lv_iter->lv_meta.lvm_struct_name.empty()) {
|
||||
yajlpp_gen gen;
|
||||
yajl_gen_config(gen, yajl_gen_beautify, false);
|
||||
|
||||
{
|
||||
yajlpp_map root(gen);
|
||||
|
||||
for (auto &lv_struct : vc->line_values) {
|
||||
if (lv_struct.lv_meta.lvm_column != sub_col) {
|
||||
continue;
|
||||
}
|
||||
|
||||
root.gen(lv_struct.lv_meta.lvm_name);
|
||||
switch (lv_struct.lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_NULL:
|
||||
root.gen();
|
||||
break;
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
root.gen((bool) lv_struct.lv_value.i);
|
||||
break;
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
root.gen(lv_struct.lv_value.i);
|
||||
break;
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
root.gen(lv_struct.lv_value.d);
|
||||
break;
|
||||
default:
|
||||
root.gen(lv_struct.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto sf = gen.to_string_fragment();
|
||||
sqlite3_result_text(ctx,
|
||||
lv_iter->text_value(),
|
||||
lv_iter->text_length(),
|
||||
sf.data(),
|
||||
sf.length(),
|
||||
SQLITE_TRANSIENT);
|
||||
sqlite3_result_subtype(ctx, 74);
|
||||
break;
|
||||
}
|
||||
case logline_value::VALUE_STRUCT:
|
||||
case logline_value::VALUE_TEXT:
|
||||
case logline_value::VALUE_TIMESTAMP: {
|
||||
sqlite3_result_text(ctx,
|
||||
lv_iter->text_value(),
|
||||
lv_iter->text_length(),
|
||||
SQLITE_TRANSIENT);
|
||||
break;
|
||||
}
|
||||
case logline_value::VALUE_W3C_QUOTED:
|
||||
case logline_value::VALUE_QUOTED:
|
||||
if (lv_iter->lv_sbr.length() == 0) {
|
||||
sqlite3_result_text(ctx, "", 0, SQLITE_STATIC);
|
||||
}
|
||||
else {
|
||||
const char *text_value = lv_iter->lv_sbr.get_data();
|
||||
size_t text_len = lv_iter->lv_sbr.length();
|
||||
|
||||
switch (text_value[0]) {
|
||||
case '\'':
|
||||
case '"': {
|
||||
char *val = (char *)sqlite3_malloc(text_len);
|
||||
|
||||
if (val == nullptr) {
|
||||
sqlite3_result_error_nomem(ctx);
|
||||
}
|
||||
else {
|
||||
auto unquote_func =
|
||||
lv_iter->lv_kind == logline_value::VALUE_W3C_QUOTED ?
|
||||
unquote_w3c : unquote;
|
||||
|
||||
size_t unquoted_len = unquote_func(val, text_value, text_len);
|
||||
sqlite3_result_text(ctx, val, unquoted_len, sqlite3_free);
|
||||
}
|
||||
} else {
|
||||
switch (lv_iter->lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_NULL:
|
||||
sqlite3_result_null(ctx);
|
||||
break;
|
||||
case value_kind_t::VALUE_JSON: {
|
||||
sqlite3_result_text(ctx,
|
||||
lv_iter->text_value(),
|
||||
lv_iter->text_length(),
|
||||
SQLITE_TRANSIENT);
|
||||
sqlite3_result_subtype(ctx, 74);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
sqlite3_result_text(ctx, text_value,
|
||||
lv_iter->lv_sbr.length(), SQLITE_TRANSIENT);
|
||||
case value_kind_t::VALUE_STRUCT:
|
||||
case value_kind_t::VALUE_TEXT:
|
||||
case value_kind_t::VALUE_TIMESTAMP: {
|
||||
sqlite3_result_text(ctx,
|
||||
lv_iter->text_value(),
|
||||
lv_iter->text_length(),
|
||||
SQLITE_TRANSIENT);
|
||||
break;
|
||||
}
|
||||
}
|
||||
case value_kind_t::VALUE_W3C_QUOTED:
|
||||
case value_kind_t::VALUE_QUOTED:
|
||||
if (lv_iter->lv_sbr.empty()) {
|
||||
sqlite3_result_text(ctx, "", 0, SQLITE_STATIC);
|
||||
} else {
|
||||
const char *text_value = lv_iter->lv_sbr.get_data();
|
||||
size_t text_len = lv_iter->lv_sbr.length();
|
||||
|
||||
switch (text_value[0]) {
|
||||
case '\'':
|
||||
case '"': {
|
||||
char *val = (char *) sqlite3_malloc(
|
||||
text_len);
|
||||
|
||||
if (val == nullptr) {
|
||||
sqlite3_result_error_nomem(ctx);
|
||||
} else {
|
||||
auto unquote_func =
|
||||
lv_iter->lv_meta.lvm_kind ==
|
||||
value_kind_t::VALUE_W3C_QUOTED ?
|
||||
unquote_w3c : unquote;
|
||||
|
||||
size_t unquoted_len = unquote_func(
|
||||
val, text_value, text_len);
|
||||
sqlite3_result_text(ctx, val,
|
||||
unquoted_len,
|
||||
sqlite3_free);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
sqlite3_result_text(ctx, text_value,
|
||||
lv_iter->lv_sbr.length(),
|
||||
SQLITE_TRANSIENT);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
sqlite3_result_int64(ctx, lv_iter->lv_value.i);
|
||||
break;
|
||||
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
sqlite3_result_double(ctx, lv_iter->lv_value.d);
|
||||
break;
|
||||
|
||||
case value_kind_t::VALUE_UNKNOWN:
|
||||
case value_kind_t::VALUE__MAX:
|
||||
require(0);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case logline_value::VALUE_BOOLEAN:
|
||||
case logline_value::VALUE_INTEGER:
|
||||
sqlite3_result_int64(ctx, lv_iter->lv_value.i);
|
||||
break;
|
||||
|
||||
case logline_value::VALUE_FLOAT:
|
||||
sqlite3_result_double(ctx, lv_iter->lv_value.d);
|
||||
break;
|
||||
|
||||
case logline_value::VALUE_UNKNOWN:
|
||||
case logline_value::VALUE__MAX:
|
||||
require(0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
|
@ -89,6 +89,11 @@ public:
|
||||
vc_subtype(subtype) {
|
||||
};
|
||||
|
||||
vtab_column& with_comment(const std::string comment) {
|
||||
this->vc_comment = comment;
|
||||
return *this;
|
||||
}
|
||||
|
||||
std::string vc_name;
|
||||
int vc_type;
|
||||
std::string vc_collator;
|
||||
@ -97,7 +102,7 @@ public:
|
||||
int vc_subtype;
|
||||
};
|
||||
|
||||
static std::pair<int, unsigned int> logline_value_to_sqlite_type(logline_value::kind_t kind);
|
||||
static std::pair<int, unsigned int> logline_value_to_sqlite_type(value_kind_t kind);
|
||||
|
||||
log_vtab_impl(const intern_string_t name) : vi_supports_indexes(true), vi_name(name) {
|
||||
this->vi_attrs.resize(128);
|
||||
|
@ -388,12 +388,12 @@ void logfile_sub_source::text_attrs_for_line(textview_curses &lv,
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line_value.lv_hidden) {
|
||||
if (line_value.lv_meta.is_hidden()) {
|
||||
value_out.emplace_back(
|
||||
line_value.lv_origin, &SA_HIDDEN);
|
||||
}
|
||||
|
||||
if (!line_value.lv_identifier || !line_value.lv_origin.is_valid()) {
|
||||
if (!line_value.lv_meta.lvm_identifier || !line_value.lv_origin.is_valid()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -1160,21 +1160,21 @@ bool logfile_sub_source::eval_sql_filter(sqlite3_stmt *stmt, iterator ld, logfil
|
||||
continue;
|
||||
}
|
||||
for (auto& lv : values) {
|
||||
if (lv.lv_name != &name[1]) {
|
||||
if (lv.lv_meta.lvm_name != &name[1]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
switch (lv.lv_kind) {
|
||||
case logline_value::VALUE_BOOLEAN:
|
||||
switch (lv.lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
|
||||
break;
|
||||
case logline_value::VALUE_FLOAT:
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
sqlite3_bind_double(stmt, lpc + 1, lv.lv_value.d);
|
||||
break;
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
|
||||
break;
|
||||
case logline_value::VALUE_NULL:
|
||||
case value_kind_t::VALUE_NULL:
|
||||
sqlite3_bind_null(stmt, lpc + 1);
|
||||
break;
|
||||
default:
|
||||
|
@ -584,7 +584,7 @@ void readline_shlex_highlighter(attr_line_t &al, int x)
|
||||
case shlex_token_t::ST_QUOTED_VARIABLE_REF: {
|
||||
int extra = token == shlex_token_t::ST_VARIABLE_REF ? 0 : 1;
|
||||
string ident = str.substr(cap.c_begin + 1 + extra, cap.length() - 1 - extra * 2);
|
||||
int attrs = vc.attrs_for_ident(ident.c_str(), ident.size());
|
||||
int attrs = vc.attrs_for_ident(ident);
|
||||
|
||||
al.with_attr(string_attr(
|
||||
line_range(cap.c_begin, cap.c_begin + 1 + extra),
|
||||
|
@ -258,21 +258,21 @@ void add_filter_expr_possibilities(readline_curses *rlc, int context, const std:
|
||||
lf->read_full_message(ll, sbr);
|
||||
format->annotate(cl, sbr, sa, values);
|
||||
for (auto& lv : values) {
|
||||
if (!lv.lv_identifier) {
|
||||
if (!lv.lv_meta.lvm_identifier) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto_mem<char> ident(sqlite3_free);
|
||||
|
||||
ident = sql_quote_ident(lv.lv_name.get());
|
||||
ident = sql_quote_ident(lv.lv_meta.lvm_name.get());
|
||||
auto bound_name = fmt::format(":{}", ident);
|
||||
rlc->add_possibility(context, type, bound_name);
|
||||
switch (lv.lv_kind) {
|
||||
case logline_value::VALUE_BOOLEAN:
|
||||
case logline_value::VALUE_FLOAT:
|
||||
case logline_value::VALUE_NULL:
|
||||
switch (lv.lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
case value_kind_t::VALUE_NULL:
|
||||
break;
|
||||
case logline_value::VALUE_INTEGER:
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
rlc->add_possibility(
|
||||
context, type,
|
||||
std::to_string(lv.lv_value.i));
|
||||
|
@ -1413,7 +1413,7 @@ static void save_session_with_id(const std::string session_id)
|
||||
}
|
||||
|
||||
for (const auto& vd : elf->elf_value_defs) {
|
||||
if (!vd.second->vd_user_hidden) {
|
||||
if (!vd.second->vd_meta.lvm_user_hidden) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -1533,7 +1533,7 @@ void reset_session()
|
||||
}
|
||||
|
||||
for (const auto &vd : elf->elf_value_defs) {
|
||||
vd.second->vd_user_hidden = false;
|
||||
vd.second->vd_meta.lvm_user_hidden = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -283,6 +283,10 @@ public:
|
||||
|
||||
attr_t attrs_for_ident(const char *str, size_t len) const;
|
||||
|
||||
attr_t attrs_for_ident(intern_string_t str) const {
|
||||
return this->attrs_for_ident(str.get(), str.size());
|
||||
}
|
||||
|
||||
attr_t attrs_for_ident(const std::string &str) const {
|
||||
return this->attrs_for_ident(str.c_str(), str.length());
|
||||
};
|
||||
|
@ -222,6 +222,9 @@ struct json_path_handler_base {
|
||||
bool jph_is_array;
|
||||
bool jph_is_pattern_property{false};
|
||||
std::vector<std::string> jph_examples;
|
||||
|
||||
std::function<int(yajlpp_parse_context *, int)> jph_bool_cb;
|
||||
std::function<int(yajlpp_parse_context *, const unsigned char *str, size_t len)> jph_str_cb;
|
||||
};
|
||||
|
||||
struct json_path_handler;
|
||||
|
@ -248,13 +248,12 @@ struct json_path_handler : public json_path_handler_base {
|
||||
return 1;
|
||||
};
|
||||
|
||||
template<typename T, bool T::*BOOL>
|
||||
static int bool_field_cb(yajlpp_parse_context *ypc, int val) {
|
||||
auto obj = (T *) ypc->ypc_obj_stack.top();
|
||||
return ypc->ypc_current_handler->jph_bool_cb(ypc, val);
|
||||
};
|
||||
|
||||
obj->*BOOL = static_cast<bool>(val);
|
||||
|
||||
return 1;
|
||||
static int str_field_cb2(yajlpp_parse_context *ypc, const unsigned char *str, size_t len) {
|
||||
return ypc->ypc_current_handler->jph_str_cb(ypc, str, len);
|
||||
};
|
||||
|
||||
template<typename T, typename NUM_T, NUM_T T::*NUM>
|
||||
@ -421,12 +420,123 @@ struct json_path_handler : public json_path_handler_base {
|
||||
|
||||
template<typename T, typename BOOL_T, bool T::*BOOL>
|
||||
json_path_handler &for_field() {
|
||||
this->add_cb(bool_field_cb<T, BOOL>);
|
||||
this->add_cb(bool_field_cb);
|
||||
this->jph_bool_cb = [&](yajlpp_parse_context *ypc, int val) {
|
||||
auto obj = (T *) ypc->ypc_obj_stack.top();
|
||||
|
||||
obj->*BOOL = static_cast<bool>(val);
|
||||
|
||||
return 1;
|
||||
};
|
||||
this->jph_gen_callback = field_gen<T, bool, BOOL>;
|
||||
|
||||
return *this;
|
||||
};
|
||||
|
||||
template<typename T, typename U>
|
||||
static inline U& get_field(T& input, U (T::*field)) {
|
||||
return input.*field;
|
||||
}
|
||||
|
||||
template<typename T, typename U, typename... V>
|
||||
static inline auto get_field(T& input, U (T::*field), V... args)
|
||||
-> decltype(get_field(input.*field, args...)) {
|
||||
return get_field(input.*field, args...);
|
||||
}
|
||||
|
||||
template<typename T, typename U, typename... V>
|
||||
static inline auto get_field(void *input, U (T::*field), V... args)
|
||||
-> decltype(get_field(*((T *) input), field, args...)) {
|
||||
return get_field(*((T *) input), field, args...);
|
||||
}
|
||||
|
||||
template<typename R, typename T, typename... Args>
|
||||
struct LastIs {
|
||||
static constexpr bool value = LastIs<R, Args...>::value;
|
||||
};
|
||||
|
||||
template<typename R, typename T>
|
||||
struct LastIs<R, T> {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template<typename R, typename T>
|
||||
struct LastIs<R, R T::*> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
template<typename T, typename... Args>
|
||||
struct LastIsEnum {
|
||||
static constexpr bool value = LastIsEnum<Args...>::value;
|
||||
};
|
||||
|
||||
template<typename T, typename U>
|
||||
struct LastIsEnum<U T::*> {
|
||||
static constexpr bool value = std::is_enum<U>::value;
|
||||
};
|
||||
|
||||
template<
|
||||
typename... Args,
|
||||
std::enable_if_t<LastIs<bool, Args...>::value, bool> = true
|
||||
>
|
||||
json_path_handler &for_field(Args... args) {
|
||||
this->add_cb(bool_field_cb);
|
||||
this->jph_bool_cb = [args...](yajlpp_parse_context *ypc, int val) {
|
||||
auto obj = ypc->ypc_obj_stack.top();
|
||||
|
||||
json_path_handler::get_field(obj, args...) = static_cast<bool>(val);
|
||||
|
||||
return 1;
|
||||
};
|
||||
return *this;
|
||||
}
|
||||
|
||||
template<
|
||||
typename... Args,
|
||||
std::enable_if_t<LastIsEnum<Args...>::value, bool> = true
|
||||
>
|
||||
json_path_handler &for_field(Args... args) {
|
||||
this->add_cb(str_field_cb2);
|
||||
this->jph_str_cb = [args...](yajlpp_parse_context *ypc,
|
||||
const unsigned char *str, size_t len) {
|
||||
auto obj = ypc->ypc_obj_stack.top();
|
||||
auto handler = ypc->ypc_current_handler;
|
||||
auto res = handler->to_enum_value(string_fragment(str, 0, len));
|
||||
|
||||
if (res) {
|
||||
json_path_handler::get_field(obj, args...) =
|
||||
(decltype(json_path_handler::get_field(obj,
|
||||
args...))) res.value();
|
||||
} else {
|
||||
ypc->report_error(lnav_log_level_t::ERROR,
|
||||
"error:%s:line %d\n "
|
||||
"Invalid value, '%.*s', for option:",
|
||||
ypc->ypc_source.c_str(),
|
||||
ypc->get_line_number(),
|
||||
len,
|
||||
str);
|
||||
|
||||
ypc->report_error(lnav_log_level_t::ERROR,
|
||||
" %s %s -- %s\n",
|
||||
&ypc->ypc_path[0],
|
||||
handler->jph_synopsis,
|
||||
handler->jph_description);
|
||||
ypc->report_error(lnav_log_level_t::ERROR,
|
||||
" Allowed values: ");
|
||||
for (int lpc = 0; handler->jph_enum_values[lpc].first; lpc++) {
|
||||
const json_path_handler::enum_value_t &ev = handler->jph_enum_values[lpc];
|
||||
|
||||
ypc->report_error(lnav_log_level_t::ERROR, " %s\n",
|
||||
ev.first);
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
};
|
||||
|
||||
return *this;
|
||||
};
|
||||
|
||||
|
||||
template<typename T, typename NUM_T, NUM_T T::*NUM>
|
||||
json_path_handler &for_field(typename std::enable_if<std::is_integral<NUM_T>::value &&
|
||||
|
@ -371,6 +371,7 @@ DISTCLEANFILES = \
|
||||
logfile_stdin.log \
|
||||
logfile_stdin.0.log \
|
||||
logfile_syslog_test.0 \
|
||||
logfile_syslog_test.2 \
|
||||
logfile_syslog_fr_test.0 \
|
||||
logfile_syslog_with_mixed_times_test.0 \
|
||||
test-logs.tgz \
|
||||
|
@ -1,9 +1,10 @@
|
||||
{
|
||||
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
|
||||
"bad_sample_log": {
|
||||
"title": "invalid sample test",
|
||||
"regex": {
|
||||
"std": {
|
||||
"pattern": "^(?<timestamp>\\d+): (?<body>.*)$"
|
||||
"pattern": "^(?<timestamp>\\d+): (?<pid>\\w+) (?<body>.*)$"
|
||||
},
|
||||
"semi": {
|
||||
"pattern": "^(?<timestamp>\\d+); (?<body>\\w+)$"
|
||||
@ -12,6 +13,11 @@
|
||||
"timestamp-format" : [
|
||||
"%i"
|
||||
],
|
||||
"value": {
|
||||
"pid": {
|
||||
"kind": "foo"
|
||||
}
|
||||
},
|
||||
"sample": [
|
||||
{
|
||||
"line": "1428634687123; foo bar"
|
||||
|
@ -25,14 +25,31 @@ warning: description <string> -- A description of the field
|
||||
error:format.json:4:invalid json -- parse error: object key and value must be separated by a colon (':')
|
||||
ar_log": { "abc" } }
|
||||
(right here) ------^
|
||||
error:foobar_log: no regexes specified for format
|
||||
error:foobar_log:no sample logs provided, all formats must have samples
|
||||
error:invalid_key_log: no regexes specified for format
|
||||
error:invalid_key_log:no sample logs provided, all formats must have samples
|
||||
EOF
|
||||
|
||||
run_test ${lnav_test} -C \
|
||||
-I ${test_dir}/bad-config
|
||||
|
||||
sed -i "" -e "s|/.*/init.sql|init.sql|g" `test_err_filename`
|
||||
sed -i "" -e "s|/.*/init.sql|init.sql|g" \
|
||||
-e "s|/.*/format|format|g" \
|
||||
`test_err_filename`
|
||||
|
||||
check_error_output "invalid format not detected?" <<EOF
|
||||
error:format.json:line 18
|
||||
Invalid value, 'foo', for option:
|
||||
/bad_sample_log/value/pid/kind string|integer|float|boolean|json|quoted -- The type of data in the field
|
||||
Allowed values:
|
||||
string
|
||||
integer
|
||||
float
|
||||
boolean
|
||||
json
|
||||
struct
|
||||
quoted
|
||||
error:bad_regex_log.regex[std]:missing )
|
||||
error:bad_regex_log.regex[std]:^(?<timestamp>\d+: (?<body>.*)$
|
||||
error:bad_regex_log.regex[std]: ^
|
||||
@ -45,7 +62,7 @@ error:bad_sample_log:invalid sample -- 1428634687123; foo bar
|
||||
error:bad_sample_log:partial sample matched -- 1428634687123; foo
|
||||
error: against pattern bad_sample_log/regex/semi -- ^(?<timestamp>\d+); (?<body>\w+)$
|
||||
error:bad_sample_log:partial sample matched -- 1428634687123
|
||||
error: against pattern bad_sample_log/regex/std -- ^(?<timestamp>\d+): (?<body>.*)$
|
||||
error: against pattern bad_sample_log/regex/std -- ^(?<timestamp>\d+): (?<pid>\w+) (?<body>.*)$
|
||||
error:no_sample_log:no sample logs provided, all formats must have samples
|
||||
error:init.sql:2:near "TALE": syntax error
|
||||
EOF
|
||||
|
@ -143,9 +143,9 @@ run_test ./drive_logfile -f bro_conn_log ${srcdir}/logfile_bro_conn.log.0
|
||||
|
||||
on_error_fail_with "Didn't infer bro_conn_log log format?"
|
||||
|
||||
run_test ./drive_logfile -f w3c_7685df_log ${srcdir}/logfile_w3c.0
|
||||
run_test ./drive_logfile -f w3c_log ${srcdir}/logfile_w3c.0
|
||||
|
||||
on_error_fail_with "Didn't infer w3c_7685df_log log format?"
|
||||
on_error_fail_with "Didn't infer w3c_log log format?"
|
||||
|
||||
|
||||
run_test ./drive_logfile ${srcdir}/logfile_empty.0
|
||||
@ -153,7 +153,7 @@ run_test ./drive_logfile ${srcdir}/logfile_empty.0
|
||||
on_error_fail_with "Didn't handle empty log?"
|
||||
|
||||
|
||||
run_test ./drive_logfile -t -f w3c_2957b3_log ${srcdir}/logfile_w3c.2
|
||||
run_test ./drive_logfile -t -f w3c_log ${srcdir}/logfile_w3c.2
|
||||
|
||||
check_output "w3c timestamp interpreted incorrectly?" <<EOF
|
||||
Oct 09 16:44:49 2000 -- 000
|
||||
@ -172,7 +172,7 @@ Oct 10 16:44:49 2000 -- 000
|
||||
Oct 10 16:48:05 2000 -- 000
|
||||
EOF
|
||||
|
||||
run_test ./drive_logfile -t -f w3c_5bd538_log ${srcdir}/logfile_w3c.4
|
||||
run_test ./drive_logfile -t -f w3c_log ${srcdir}/logfile_w3c.4
|
||||
|
||||
check_output "quoted w3c timestamp interpreted incorrectly?" <<EOF
|
||||
Jun 28 07:26:35 2017 -- 000
|
||||
|
@ -2,15 +2,69 @@
|
||||
|
||||
lnav_test="${top_builddir}/src/lnav-test"
|
||||
|
||||
cp ${srcdir}/logfile_syslog.2 logfile_syslog_test.2
|
||||
touch -t 201511030923 logfile_syslog_test.2
|
||||
run_test ${lnav_test} -n \
|
||||
-c ";SELECT sc_substatus FROM w3c_e28cf8_log" \
|
||||
-c ";SELECT *, log_msg_schema FROM all_logs" \
|
||||
-c ":write-csv-to -" \
|
||||
logfile_syslog_test.2
|
||||
|
||||
check_output "all_logs does not work?" <<EOF
|
||||
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,log_format,log_msg_format,log_msg_schema
|
||||
0,<NULL>,2015-11-03 09:23:38.000,0,info,0,<NULL>,<NULL>,<NULL>,syslog_log, # is up,f7ca05240a1c3f67c85b9db38bf90171
|
||||
1,<NULL>,2015-11-03 09:23:38.000,0,info,0,<NULL>,<NULL>,<NULL>,syslog_log, # is up,f7ca05240a1c3f67c85b9db38bf90171
|
||||
2,<NULL>,2015-11-03 09:23:38.000,0,info,0,<NULL>,<NULL>,<NULL>,syslog_log, # is down,a2ea6285505d1a8947504667581daf1b
|
||||
EOF
|
||||
|
||||
|
||||
run_test ${lnav_test} -n \
|
||||
-c ";SELECT sc_substatus FROM w3c_log" \
|
||||
-c ":write-json-to -" \
|
||||
${test_dir}/logfile_w3c.3
|
||||
|
||||
check_output "w3c quoted strings are not handled correctly?" <<EOF
|
||||
sc_substatus
|
||||
0
|
||||
0
|
||||
"garbage" w/ spaces
|
||||
[
|
||||
{
|
||||
"sc_substatus": 0
|
||||
},
|
||||
{
|
||||
"sc_substatus": 0
|
||||
},
|
||||
{
|
||||
"sc_substatus": null
|
||||
}
|
||||
]
|
||||
EOF
|
||||
|
||||
run_test ${lnav_test} -n \
|
||||
-c ";SELECT cs_headers FROM w3c_log" \
|
||||
-c ":write-json-to -" \
|
||||
${test_dir}/logfile_w3c.3
|
||||
|
||||
check_output "w3c headers are not captured?" <<EOF
|
||||
[
|
||||
{
|
||||
"cs_headers": {
|
||||
"User-Agent": "Mozilla/5.0 (Linux; Android 4.4.4; SM-G900V Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.59 Mobile Safari/537.36",
|
||||
"Referer": "http://example.com/Search/SearchResults.pg?informationRecipient.languageCode.c=en",
|
||||
"Host": "xzy.example.com"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cs_headers": {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
|
||||
"Referer": null,
|
||||
"Host": "example.hello.com"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cs_headers": {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
|
||||
"Referer": null,
|
||||
"Host": "hello.example.com"
|
||||
}
|
||||
}
|
||||
]
|
||||
EOF
|
||||
|
||||
run_test ${lnav_test} -n \
|
||||
@ -156,7 +210,21 @@ run_test ${lnav_test} -n \
|
||||
|
||||
check_output "uwsgi not working?" <<EOF
|
||||
log_line log_part log_time log_idle_msecs log_level log_mark log_comment log_tags log_filters c_ip cs_bytes cs_method cs_uri_query cs_uri_stem cs_username cs_vars cs_version s_app s_core s_pid s_req s_runtime s_switches s_worker_reqs sc_bytes sc_header_bytes sc_headers sc_status
|
||||
0 <NULL> 2016-03-13 22:49:12.000 0 info 0 <NULL> <NULL> <NULL> 127.0.0.1 696 POST <NULL> /update_metrics 38 HTTP/1.1 0 3 88185 1 129.0 1 1 47 378 9 200
|
||||
0 <NULL> 2016-03-13 22:49:12.000 0 info 0 <NULL> <NULL> <NULL> 127.0.0.1 696 POST <NULL> /update_metrics 38 HTTP/1.1 0 3 88185 1 0.129 1 1 47 378 9 200
|
||||
EOF
|
||||
|
||||
run_test ${lnav_test} -n \
|
||||
-c ";SELECT s_runtime FROM uwsgi_log LIMIT 5" \
|
||||
-c ':write-csv-to -' \
|
||||
${test_dir}/logfile_uwsgi.0
|
||||
|
||||
check_output "uwsgi scaling not working?" <<EOF
|
||||
s_runtime
|
||||
0.129
|
||||
0.035
|
||||
6.8e-05
|
||||
0.016
|
||||
0.01
|
||||
EOF
|
||||
|
||||
run_test env TZ=UTC ${lnav_test} -n \
|
||||
|
Loading…
Reference in New Issue
Block a user