mirror of
https://github.com/tstack/lnav.git
synced 2024-09-11 13:05:51 +03:00
[config] externalize file format detection
This commit is contained in:
parent
ff0b112728
commit
2e09202ff5
9
NEWS.md
9
NEWS.md
@ -14,6 +14,15 @@ Features:
|
||||
internally, they will not interfere with timestamps that
|
||||
are in the commands output.
|
||||
* Added a `:cd` command to change **lnav**'s current directory.
|
||||
* Added support for automatically converting files that are
|
||||
in a format not natively supported by **lnav**. The
|
||||
`/tuning/file-format` section allows you to define a format
|
||||
that is detected by examining the file header. The format
|
||||
must also provide a converter script that can be run to
|
||||
convert the file. You can then define a log format that
|
||||
matches the MIME type defined by the file format and can
|
||||
consume the converted form of the file. The built-in PCAP
|
||||
support in **lnav** is implemented using this mechanism.
|
||||
|
||||
Bug Fixes:
|
||||
* When piping data into **lnav**'s stdin, the input used to
|
||||
|
@ -73,6 +73,59 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"file-format": {
|
||||
"description": "sdfjdls",
|
||||
"title": "/tuning/file-format",
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"(\\w+~1[\\w\\.\\-]+)": {
|
||||
"description": "File format definitions, keyed by their MIME type",
|
||||
"title": "/tuning/file-format/<mime_type>",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {
|
||||
"title": "/tuning/file-format/<mime_type>/title",
|
||||
"description": "The display name for this file format",
|
||||
"type": "string"
|
||||
},
|
||||
"header": {
|
||||
"description": "File header detection definitions",
|
||||
"title": "/tuning/file-format/<mime_type>/header",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expr": {
|
||||
"description": "The expressions used to check if a file header matches this file format",
|
||||
"title": "/tuning/file-format/<mime_type>/header/expr",
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"(\\w+)": {
|
||||
"title": "/tuning/file-format/<mime_type>/header/expr/<header_expr_name>",
|
||||
"description": "SQLite expression",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"size": {
|
||||
"title": "/tuning/file-format/<mime_type>/header/size",
|
||||
"description": "The minimum size required for this header type",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"converter": {
|
||||
"title": "/tuning/file-format/<mime_type>/converter",
|
||||
"description": "The script used to convert the file",
|
||||
"type": "string",
|
||||
"pattern": "[\\w\\.\\-]+"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"logfile": {
|
||||
"description": "Settings related to log files",
|
||||
"title": "/tuning/logfile",
|
||||
|
@ -81,9 +81,7 @@
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"application/vnd.tcpdump.pcap"
|
||||
]
|
||||
"pattern": "^\\w/[\\w\\.]+"
|
||||
}
|
||||
},
|
||||
"level-field": {
|
||||
|
@ -260,6 +260,10 @@ command.
|
||||
|
||||
.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/clipboard
|
||||
|
||||
.. _tuning_file_format:
|
||||
|
||||
.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/file-format
|
||||
|
||||
.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/piper
|
||||
|
||||
.. jsonschema:: ../schemas/config-v1.schema.json#/definitions/clip-commands
|
||||
|
@ -133,6 +133,12 @@ object with the following fields:
|
||||
|
||||
:json: True if each log line is JSON-encoded.
|
||||
|
||||
:mime-types: An array of MIME types that this log format should only be
|
||||
used with. These MIME types refer to file formats that are defined
|
||||
using the `Automatic File Conversion`_ feature. This property should
|
||||
not be defined for log files that do not require conversion and can be
|
||||
naturally parsed using regexes or are JSON-lines.
|
||||
|
||||
:line-format: An array that specifies the text format for JSON-encoded
|
||||
log messages. Log files that are JSON-encoded will have each message
|
||||
converted from the raw JSON encoding into this format. Each element
|
||||
@ -559,3 +565,29 @@ will win.
|
||||
|
||||
.. [#] The maximum number of lines to check can be configured. See the
|
||||
:ref:`tuning` section for more details.
|
||||
|
||||
Automatic File Conversion
|
||||
-------------------------
|
||||
|
||||
File formats that are not naturally understood by **lnav** can be
|
||||
automatically detected and converted to a usable form using the
|
||||
:ref:`tuning_file_format` configuration options. For example,
|
||||
PCAP files can be detected and converted to a JSON-lines form
|
||||
using :code:`tshark`. The conversion process works as follows:
|
||||
|
||||
#. The first 1024 bytes of the file are read, if available.
|
||||
#. This header is converted into a hex string.
|
||||
#. For each file-format, every "header expression" is evaluated
|
||||
to see if there is a match. The header expressions are
|
||||
SQLite expressions where the following variables are defined:
|
||||
|
||||
:\:header: A string containing the header as a hex string.
|
||||
:\:filepath: The path to the file.
|
||||
#. If a match is found, the converter script defined in the
|
||||
file format will be invoked and passed the format MIME type
|
||||
and path to the file as arguments. The script should write
|
||||
the converted form of the input file on its standard output.
|
||||
Any errors should be written to the standard error.
|
||||
#. The MIME type will be associated with the original file and
|
||||
only log formats that have the corresponding type will be
|
||||
used to interpret the file.
|
||||
|
@ -351,6 +351,7 @@ add_library(
|
||||
extension-functions.cc
|
||||
field_overlay_source.cc
|
||||
file_collection.cc
|
||||
file_converter_manager.cc
|
||||
file_format.cc
|
||||
file_vtab.cc
|
||||
files_sub_source.cc
|
||||
@ -391,7 +392,7 @@ add_library(
|
||||
data_scanner.cc
|
||||
data_scanner_re.cc
|
||||
data_parser.cc
|
||||
pcap_manager.cc
|
||||
file_converter_manager.cc
|
||||
plain_text_source.cc
|
||||
pretty_printer.cc
|
||||
pugixml/pugixml.cpp
|
||||
@ -462,7 +463,9 @@ add_library(
|
||||
elem_to_json.hh
|
||||
field_overlay_source.hh
|
||||
file_collection.hh
|
||||
file_converter_manager.hh
|
||||
file_format.hh
|
||||
file_format.cfg.hh
|
||||
files_sub_source.hh
|
||||
filter_observer.hh
|
||||
filter_status_source.hh
|
||||
@ -502,7 +505,7 @@ add_library(
|
||||
md2attr_line.hh
|
||||
md4cpp.hh
|
||||
optional.hpp
|
||||
pcap_manager.hh
|
||||
file_converter_manager.hh
|
||||
piper.looper.hh
|
||||
piper.looper.cfg.hh
|
||||
plain_text_source.hh
|
||||
|
@ -201,7 +201,9 @@ noinst_HEADERS = \
|
||||
environ_vtab.hh \
|
||||
field_overlay_source.hh \
|
||||
file_collection.hh \
|
||||
file_converter_manager.hh \
|
||||
file_format.hh \
|
||||
file_format.cfg.hh \
|
||||
file_vtab.cfg.hh \
|
||||
files_sub_source.hh \
|
||||
filter_observer.hh \
|
||||
@ -258,7 +260,6 @@ noinst_HEADERS = \
|
||||
md2attr_line.hh \
|
||||
md4cpp.hh \
|
||||
optional.hpp \
|
||||
pcap_manager.hh \
|
||||
piper.looper.hh \
|
||||
piper.looper.cfg.hh \
|
||||
plain_text_source.hh \
|
||||
@ -391,6 +392,7 @@ libdiag_a_SOURCES = \
|
||||
extension-functions.cc \
|
||||
field_overlay_source.cc \
|
||||
file_collection.cc \
|
||||
file_converter_manager.cc \
|
||||
file_format.cc \
|
||||
files_sub_source.cc \
|
||||
filter_observer.cc \
|
||||
@ -428,7 +430,6 @@ libdiag_a_SOURCES = \
|
||||
md4cpp.cc \
|
||||
network-extension-functions.cc \
|
||||
data_parser.cc \
|
||||
pcap_manager.cc \
|
||||
piper.looper.cc \
|
||||
plain_text_source.cc \
|
||||
pollable.cc \
|
||||
|
@ -50,7 +50,8 @@ column_namer::existing_name(const string_fragment& in_name) const
|
||||
auto upped = toupper(in_name.to_string());
|
||||
|
||||
if (std::binary_search(
|
||||
std::begin(sql_keywords), std::end(sql_keywords), upped)) {
|
||||
std::begin(sql_keywords), std::end(sql_keywords), upped))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
@ -99,10 +100,13 @@ column_namer::add_column(const string_fragment& in_name)
|
||||
this->cn_name_counters[counter_name] = num;
|
||||
}
|
||||
|
||||
log_debug(
|
||||
"column name already exists: %.*s", retval.length(), retval.data());
|
||||
fmt::format_to(
|
||||
std::back_inserter(buf), FMT_STRING("{}_{}"), base_name, num);
|
||||
log_debug("column name already exists (%.*s), trying (%.*s)",
|
||||
retval.length(),
|
||||
retval.data(),
|
||||
buf.size(),
|
||||
buf.data());
|
||||
retval = string_fragment::from_memory_buffer(buf);
|
||||
num += 1;
|
||||
}
|
||||
|
@ -41,9 +41,9 @@
|
||||
#include "base/opt_util.hh"
|
||||
#include "base/string_util.hh"
|
||||
#include "config.h"
|
||||
#include "file_converter_manager.hh"
|
||||
#include "lnav_util.hh"
|
||||
#include "logfile.hh"
|
||||
#include "pcap_manager.hh"
|
||||
#include "service_tags.hh"
|
||||
#include "tailer/tailer.looper.hh"
|
||||
|
||||
@ -134,7 +134,6 @@ file_collection::regenerate_unique_file_names()
|
||||
switch (pair.second.ofd_format) {
|
||||
case file_format_t::UNKNOWN:
|
||||
case file_format_t::ARCHIVE:
|
||||
case file_format_t::PCAP:
|
||||
case file_format_t::SQLITE_DB: {
|
||||
auto bn = ghc::filesystem::path(pair.first).filename().string();
|
||||
if (bn.length() > this->fc_largest_path_length) {
|
||||
@ -336,63 +335,6 @@ file_collection::watch_logfile(const std::string& filename,
|
||||
retval.fc_other_files[filename].ofd_format = ff;
|
||||
break;
|
||||
|
||||
case file_format_t::PCAP: {
|
||||
auto res = pcap_manager::convert(filename);
|
||||
|
||||
if (res.isOk()) {
|
||||
auto convert_res = res.unwrap();
|
||||
retval.fc_child_pollers.emplace_back(child_poller{
|
||||
std::move(convert_res.cr_child),
|
||||
[filename,
|
||||
st,
|
||||
error_queue = convert_res.cr_error_queue](
|
||||
auto& fc, auto& child) {
|
||||
if (child.was_normal_exit()
|
||||
&& child.exit_status() == EXIT_SUCCESS)
|
||||
{
|
||||
log_info("pcap[%d] exited normally",
|
||||
child.in());
|
||||
return;
|
||||
}
|
||||
log_error("pcap[%d] exited with %d",
|
||||
child.in(),
|
||||
child.status());
|
||||
fc.fc_name_to_errors.emplace(
|
||||
filename,
|
||||
file_error_info{
|
||||
st.st_mtime,
|
||||
fmt::format(
|
||||
FMT_STRING("{}"),
|
||||
fmt::join(*error_queue, "\n")),
|
||||
});
|
||||
},
|
||||
});
|
||||
loo.with_stat_for_temp(st);
|
||||
auto open_res
|
||||
= logfile::open(convert_res.cr_destination, loo);
|
||||
if (open_res.isOk()) {
|
||||
retval.fc_files.push_back(open_res.unwrap());
|
||||
} else {
|
||||
log_error("failed to open: %s -- %s",
|
||||
filename.c_str(),
|
||||
open_res.unwrapErr().c_str());
|
||||
retval.fc_name_to_errors.emplace(
|
||||
filename,
|
||||
file_error_info{
|
||||
st.st_mtime,
|
||||
open_res.unwrapErr(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
retval.fc_name_to_errors.emplace(filename,
|
||||
file_error_info{
|
||||
st.st_mtime,
|
||||
res.unwrapErr(),
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case file_format_t::ARCHIVE: {
|
||||
nonstd::optional<
|
||||
std::list<archive_manager::extract_progress>::iterator>
|
||||
@ -462,10 +404,64 @@ file_collection::watch_logfile(const std::string& filename,
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
default: {
|
||||
auto filename_to_open = filename;
|
||||
|
||||
if (!loo.loo_temp_file) {
|
||||
auto eff = detect_mime_type(filename);
|
||||
|
||||
if (eff) {
|
||||
auto cr = file_converter_manager::convert(
|
||||
eff.value(), filename);
|
||||
|
||||
if (cr.isErr()) {
|
||||
retval.fc_name_to_errors.emplace(
|
||||
filename,
|
||||
file_error_info{
|
||||
st.st_mtime,
|
||||
cr.unwrapErr(),
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
auto convert_res = cr.unwrap();
|
||||
retval.fc_child_pollers.emplace_back(child_poller{
|
||||
std::move(convert_res.cr_child),
|
||||
[filename,
|
||||
st,
|
||||
error_queue = convert_res.cr_error_queue](
|
||||
auto& fc, auto& child) {
|
||||
if (child.was_normal_exit()
|
||||
&& child.exit_status() == EXIT_SUCCESS)
|
||||
{
|
||||
log_info(
|
||||
"converter[%d] exited normally",
|
||||
child.in());
|
||||
return;
|
||||
}
|
||||
log_error("converter[%d] exited with %d",
|
||||
child.in(),
|
||||
child.status());
|
||||
fc.fc_name_to_errors.emplace(
|
||||
filename,
|
||||
file_error_info{
|
||||
st.st_mtime,
|
||||
fmt::format(
|
||||
FMT_STRING("{}"),
|
||||
fmt::join(*error_queue, "\n")),
|
||||
});
|
||||
},
|
||||
});
|
||||
loo.with_filename(filename);
|
||||
loo.with_stat_for_temp(st);
|
||||
loo.loo_mime_type = eff->eff_mime_type;
|
||||
filename_to_open = convert_res.cr_destination;
|
||||
}
|
||||
}
|
||||
|
||||
log_info("loading new file: filename=%s", filename.c_str());
|
||||
|
||||
auto open_res = logfile::open(filename, loo);
|
||||
auto open_res = logfile::open(filename_to_open, loo);
|
||||
if (open_res.isOk()) {
|
||||
retval.fc_files.push_back(open_res.unwrap());
|
||||
} else {
|
||||
@ -477,6 +473,7 @@ file_collection::watch_logfile(const std::string& filename,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return retval;
|
||||
|
@ -25,15 +25,13 @@
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* @file pcap_manager.cc
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "pcap_manager.hh"
|
||||
#include "file_converter_manager.hh"
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
@ -42,16 +40,16 @@
|
||||
#include "config.h"
|
||||
#include "line_buffer.hh"
|
||||
|
||||
namespace pcap_manager {
|
||||
namespace file_converter_manager {
|
||||
|
||||
Result<convert_result, std::string>
|
||||
convert(const std::string& filename)
|
||||
convert(const external_file_format& eff, const std::string& filename)
|
||||
{
|
||||
log_info("attempting to convert pcap file -- %s", filename.c_str());
|
||||
log_info("attempting to convert file -- %s", filename.c_str());
|
||||
|
||||
ghc::filesystem::create_directories(lnav::paths::workdir());
|
||||
auto outfile = TRY(lnav::filesystem::open_temp_file(lnav::paths::workdir()
|
||||
/ "pcap.XXXXXX"));
|
||||
/ "conversion.XXXXXX"));
|
||||
auto err_pipe = TRY(auto_pipe::for_child_fd(STDERR_FILENO));
|
||||
auto child = TRY(lnav::pid::from_fork());
|
||||
|
||||
@ -62,28 +60,35 @@ convert(const std::string& filename)
|
||||
dup2(dev_null, STDIN_FILENO);
|
||||
dup2(outfile.second.get(), STDOUT_FILENO);
|
||||
outfile.second.reset();
|
||||
setenv("TZ", "UTC", 1);
|
||||
|
||||
auto new_path = lnav::filesystem::build_path({
|
||||
eff.eff_source_path.parent_path(),
|
||||
lnav::paths::dotlnav() / "formats/default",
|
||||
});
|
||||
setenv("PATH", new_path.c_str(), 1);
|
||||
log_info("invoking converter: %s (PATH=%s)",
|
||||
eff.eff_converter.c_str(),
|
||||
new_path.c_str());
|
||||
auto mime_str = eff.eff_mime_type.to_string();
|
||||
|
||||
const char* args[] = {
|
||||
"tshark",
|
||||
"-T",
|
||||
"ek",
|
||||
"-P",
|
||||
"-V",
|
||||
"-t",
|
||||
"ad",
|
||||
"-r",
|
||||
eff.eff_converter.c_str(),
|
||||
mime_str.c_str(),
|
||||
filename.c_str(),
|
||||
nullptr,
|
||||
};
|
||||
|
||||
execvp("tshark", (char**) args);
|
||||
setenv("TZ", "UTC", 1);
|
||||
execvp(eff.eff_converter.c_str(), (char**) args);
|
||||
if (errno == ENOENT) {
|
||||
fprintf(stderr,
|
||||
"pcap support requires 'tshark' v3+ to be installed\n");
|
||||
"cannot find converter: %s\n",
|
||||
eff.eff_converter.c_str());
|
||||
} else {
|
||||
fprintf(
|
||||
stderr, "failed to execute 'tshark' -- %s\n", strerror(errno));
|
||||
fprintf(stderr,
|
||||
"failed to execute converter: %s -- %s\n",
|
||||
eff.eff_converter.c_str(),
|
||||
strerror(errno));
|
||||
}
|
||||
_exit(EXIT_FAILURE);
|
||||
}
|
||||
@ -138,4 +143,4 @@ convert(const std::string& filename)
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace pcap_manager
|
||||
} // namespace file_converter_manager
|
@ -25,12 +25,10 @@
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* @file pcap_manager.hh
|
||||
*/
|
||||
|
||||
#ifndef lnav_pcap_manager_hh
|
||||
#define lnav_pcap_manager_hh
|
||||
#ifndef lnav_file_converter_manager_hh
|
||||
#define lnav_file_converter_manager_hh
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@ -38,9 +36,10 @@
|
||||
#include "base/auto_fd.hh"
|
||||
#include "base/auto_pid.hh"
|
||||
#include "base/result.h"
|
||||
#include "file_format.hh"
|
||||
#include "ghc/filesystem.hpp"
|
||||
|
||||
namespace pcap_manager {
|
||||
namespace file_converter_manager {
|
||||
|
||||
struct convert_result {
|
||||
auto_pid<process_state::running> cr_child;
|
||||
@ -48,8 +47,9 @@ struct convert_result {
|
||||
std::shared_ptr<std::vector<std::string>> cr_error_queue;
|
||||
};
|
||||
|
||||
Result<convert_result, std::string> convert(const std::string& filename);
|
||||
Result<convert_result, std::string> convert(const external_file_format& eff,
|
||||
const std::string& filename);
|
||||
|
||||
} // namespace pcap_manager
|
||||
} // namespace file_converter_manager
|
||||
|
||||
#endif
|
@ -29,65 +29,22 @@
|
||||
* @file file_format.hh
|
||||
*/
|
||||
|
||||
#include <unordered_map>
|
||||
|
||||
#include "file_format.hh"
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include "archive_manager.hh"
|
||||
#include "base/auto_fd.hh"
|
||||
#include "base/fs_util.hh"
|
||||
#include "base/injector.hh"
|
||||
#include "base/intern_string.hh"
|
||||
#include "base/lnav_log.hh"
|
||||
#include "config.h"
|
||||
|
||||
static bool
|
||||
is_pcap_header(uint8_t* buffer)
|
||||
{
|
||||
size_t offset = 0;
|
||||
if (buffer[0] == 0x0a && buffer[1] == 0x0d && buffer[2] == 0x0d
|
||||
&& buffer[3] == 0x0a)
|
||||
{
|
||||
offset += sizeof(uint32_t) * 2;
|
||||
if (buffer[offset + 0] == 0x1a && buffer[offset + 1] == 0x2b
|
||||
&& buffer[offset + 2] == 0x3c && buffer[offset + 3] == 0x4d)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (buffer[offset + 0] == 0x4d && buffer[offset + 1] == 0x3c
|
||||
&& buffer[offset + 2] == 0x2b && buffer[offset + 3] == 0x1a)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (buffer[0] == 0xa1 && buffer[1] == 0xb2 && buffer[2] == 0xc3
|
||||
&& buffer[3] == 0xd4)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (buffer[0] == 0xd4 && buffer[1] == 0xc3 && buffer[2] == 0xb2
|
||||
&& buffer[3] == 0xa1)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (buffer[0] == 0xa1 && buffer[1] == 0xb2 && buffer[2] == 0x3c
|
||||
&& buffer[3] == 0x4d)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (buffer[0] == 0x4d && buffer[1] == 0x3c && buffer[2] == 0xb2
|
||||
&& buffer[3] == 0xa1)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
#include "lnav_config.hh"
|
||||
#include "readline_highlighters.hh"
|
||||
#include "safe/safe.h"
|
||||
#include "sql_util.hh"
|
||||
#include "sqlite-extension-func.hh"
|
||||
|
||||
file_format_t
|
||||
detect_file_format(const ghc::filesystem::path& filename)
|
||||
@ -109,11 +66,237 @@ detect_file_format(const ghc::filesystem::path& filename)
|
||||
|
||||
if (header_frag.startswith(SQLITE3_HEADER)) {
|
||||
retval = file_format_t::SQLITE_DB;
|
||||
} else if (rc > 24 && is_pcap_header(buffer)) {
|
||||
retval = file_format_t::PCAP;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
mime_type
|
||||
mime_type::from_str(const std::string& str)
|
||||
{
|
||||
auto slash_index = str.find('/');
|
||||
|
||||
if (slash_index == std::string::npos) {
|
||||
return {"application", str};
|
||||
}
|
||||
|
||||
return {str.substr(0, slash_index), str.substr(slash_index + 1)};
|
||||
}
|
||||
|
||||
struct compiled_header_expr {
|
||||
auto_mem<sqlite3_stmt> che_stmt{sqlite3_finalize};
|
||||
bool che_enabled{true};
|
||||
};
|
||||
|
||||
struct file_format_expressions : public lnav_config_listener {
|
||||
void reload_config(error_reporter& reporter) override
|
||||
{
|
||||
log_debug("reloading file-format header expressions");
|
||||
|
||||
safe::WriteAccess<safe::Safe<inner>> in(instance);
|
||||
|
||||
if (in->e_db.in() == nullptr) {
|
||||
if (sqlite3_open(":memory:", in->e_db.out()) != SQLITE_OK) {
|
||||
log_error("unable to open memory DB");
|
||||
return;
|
||||
}
|
||||
register_sqlite_funcs(in->e_db.in(), sqlite_registration_funcs);
|
||||
}
|
||||
|
||||
in->e_header_exprs.clear();
|
||||
const auto& cfg = injector::get<const lnav::file_formats::config&>();
|
||||
for (const auto& fpair : cfg.c_defs) {
|
||||
for (const auto& hpair : fpair.second.fd_header.h_exprs.he_exprs) {
|
||||
auto stmt_str = fmt::format(FMT_STRING("SELECT 1 WHERE {}"),
|
||||
hpair.second);
|
||||
compiled_header_expr che;
|
||||
|
||||
log_info("preparing file-format header expression: %s",
|
||||
stmt_str.c_str());
|
||||
auto retcode = sqlite3_prepare_v2(in->e_db.in(),
|
||||
stmt_str.c_str(),
|
||||
stmt_str.size(),
|
||||
che.che_stmt.out(),
|
||||
nullptr);
|
||||
if (retcode != SQLITE_OK) {
|
||||
auto sql_al
|
||||
= attr_line_t(hpair.second)
|
||||
.with_attr_for_all(SA_PREFORMATTED.value())
|
||||
.with_attr_for_all(
|
||||
VC_ROLE.value(role_t::VCR_QUOTED_CODE));
|
||||
readline_sqlite_highlighter(sql_al, -1);
|
||||
intern_string_t watch_expr_path
|
||||
= intern_string::lookup(fmt::format(
|
||||
FMT_STRING(
|
||||
"/tuning/file-formats/{}/header/expr/{}"),
|
||||
json_ptr::encode_str(fpair.first.c_str()),
|
||||
hpair.first));
|
||||
auto snippet = lnav::console::snippet::from(
|
||||
source_location(watch_expr_path), sql_al);
|
||||
|
||||
auto um = lnav::console::user_message::error(
|
||||
"SQL expression is invalid")
|
||||
.with_reason(sqlite3_errmsg(in->e_db.in()))
|
||||
.with_snippet(snippet);
|
||||
|
||||
reporter(&hpair.second, um);
|
||||
continue;
|
||||
}
|
||||
|
||||
in->e_header_exprs[fpair.first][hpair.first] = std::move(che);
|
||||
}
|
||||
|
||||
if (fpair.second.fd_header.h_exprs.he_exprs.empty()) {
|
||||
auto um
|
||||
= lnav::console::user_message::error(
|
||||
"At least one header expression is required for "
|
||||
"a file format")
|
||||
.with_reason(
|
||||
"Header expressions are used to detect a format");
|
||||
reporter(&fpair.second.fd_header.h_exprs, um);
|
||||
}
|
||||
if (fpair.second.fd_converter.pp_value.empty()) {
|
||||
auto um = lnav::console::user_message::error(
|
||||
"A converter is required for a file format")
|
||||
.with_reason(
|
||||
"The converter script transforms the file "
|
||||
"into a format that can be consumed by lnav");
|
||||
reporter(&fpair.second.fd_converter, um);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void unload_config() override
|
||||
{
|
||||
safe::WriteAccess<safe::Safe<inner>> in(instance);
|
||||
|
||||
in->e_header_exprs.clear();
|
||||
}
|
||||
|
||||
struct inner {
|
||||
auto_sqlite3 e_db;
|
||||
std::map<std::string, std::map<std::string, compiled_header_expr>>
|
||||
e_header_exprs;
|
||||
};
|
||||
|
||||
safe::Safe<inner> instance;
|
||||
};
|
||||
|
||||
static file_format_expressions format_exprs;
|
||||
|
||||
nonstd::optional<external_file_format>
|
||||
detect_mime_type(const ghc::filesystem::path& filename)
|
||||
{
|
||||
uint8_t buffer[1024];
|
||||
size_t buffer_size = 0;
|
||||
|
||||
{
|
||||
auto_fd fd;
|
||||
|
||||
if ((fd = lnav::filesystem::openp(filename, O_RDONLY)) == -1) {
|
||||
return nonstd::nullopt;
|
||||
}
|
||||
|
||||
ssize_t rc;
|
||||
|
||||
if ((rc = read(fd, buffer, sizeof(buffer))) == -1) {
|
||||
return nonstd::nullopt;
|
||||
}
|
||||
buffer_size = rc;
|
||||
}
|
||||
|
||||
auto hexbuf = auto_buffer::alloc(buffer_size * 2);
|
||||
|
||||
for (int lpc = 0; lpc < buffer_size; lpc++) {
|
||||
fmt::format_to(
|
||||
std::back_inserter(hexbuf), FMT_STRING("{:02x}"), buffer[lpc]);
|
||||
}
|
||||
|
||||
safe::WriteAccess<safe::Safe<file_format_expressions::inner>> in(
|
||||
format_exprs.instance);
|
||||
|
||||
const auto& cfg = injector::get<const lnav::file_formats::config&>();
|
||||
for (const auto& fpair : cfg.c_defs) {
|
||||
if (buffer_size < fpair.second.fd_header.h_size) {
|
||||
log_debug("file content too small (%d) for header detection: %s",
|
||||
buffer_size,
|
||||
fpair.first.c_str());
|
||||
continue;
|
||||
}
|
||||
for (const auto& hpair : fpair.second.fd_header.h_exprs.he_exprs) {
|
||||
auto& he = in->e_header_exprs[fpair.first][hpair.first];
|
||||
|
||||
if (!he.che_enabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* stmt = he.che_stmt.in();
|
||||
|
||||
if (stmt == nullptr) {
|
||||
continue;
|
||||
}
|
||||
sqlite3_reset(stmt);
|
||||
auto count = sqlite3_bind_parameter_count(stmt);
|
||||
for (int lpc = 0; lpc < count; lpc++) {
|
||||
const auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1);
|
||||
|
||||
if (name[0] == '$') {
|
||||
const char* env_value;
|
||||
|
||||
if ((env_value = getenv(&name[1])) != nullptr) {
|
||||
sqlite3_bind_text(
|
||||
stmt, lpc + 1, env_value, -1, SQLITE_STATIC);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":header") == 0) {
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
hexbuf.in(),
|
||||
hexbuf.size(),
|
||||
SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":filepath") == 0) {
|
||||
sqlite3_bind_text(
|
||||
stmt, lpc + 1, filename.c_str(), -1, SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
auto step_res = sqlite3_step(stmt);
|
||||
|
||||
switch (step_res) {
|
||||
case SQLITE_OK:
|
||||
case SQLITE_DONE:
|
||||
continue;
|
||||
case SQLITE_ROW:
|
||||
break;
|
||||
default: {
|
||||
log_error(
|
||||
"failed to execute file-format header expression: "
|
||||
"%s:%s -- %s",
|
||||
fpair.first.c_str(),
|
||||
hpair.first.c_str(),
|
||||
sqlite3_errmsg(in->e_db));
|
||||
he.che_enabled = false;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
log_info("detected MIME type for: %s -- %s (header-expr: %s)",
|
||||
filename.c_str(),
|
||||
fpair.first.c_str(),
|
||||
hpair.first.c_str());
|
||||
return external_file_format{
|
||||
mime_type::from_str(fpair.first),
|
||||
fpair.second.fd_converter.pp_value,
|
||||
fpair.second.fd_converter.pp_location.sl_source.to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return nonstd::nullopt;
|
||||
}
|
||||
|
63
src/file_format.cfg.hh
Normal file
63
src/file_format.cfg.hh
Normal file
@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Copyright (c) 2023, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef lnav_file_format_cfg_hh
|
||||
#define lnav_file_format_cfg_hh
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "yajlpp/yajlpp.hh"
|
||||
|
||||
namespace lnav {
|
||||
namespace file_formats {
|
||||
|
||||
struct header_exprs {
|
||||
std::map<std::string, std::string> he_exprs;
|
||||
};
|
||||
|
||||
struct header {
|
||||
header_exprs h_exprs;
|
||||
size_t h_size{32};
|
||||
};
|
||||
|
||||
struct format_def {
|
||||
std::string fd_title;
|
||||
header fd_header;
|
||||
positioned_property<std::string> fd_converter;
|
||||
};
|
||||
|
||||
struct config {
|
||||
std::map<std::string, format_def> c_defs;
|
||||
};
|
||||
|
||||
} // namespace file_formats
|
||||
} // namespace lnav
|
||||
|
||||
#endif
|
@ -34,17 +34,45 @@
|
||||
|
||||
#include "fmt/format.h"
|
||||
#include "ghc/filesystem.hpp"
|
||||
#include "optional.hpp"
|
||||
|
||||
enum class file_format_t : int {
|
||||
UNKNOWN,
|
||||
SQLITE_DB,
|
||||
ARCHIVE,
|
||||
PCAP,
|
||||
REMOTE,
|
||||
};
|
||||
|
||||
struct mime_type {
|
||||
static mime_type from_str(const std::string& str);
|
||||
|
||||
std::string mt_type;
|
||||
std::string mt_subtype;
|
||||
|
||||
bool operator<(const mime_type& other) const
|
||||
{
|
||||
return this->mt_type < other.mt_type
|
||||
&& this->mt_subtype < other.mt_subtype;
|
||||
}
|
||||
|
||||
std::string to_string() const
|
||||
{
|
||||
return fmt::format(
|
||||
FMT_STRING("{}/{}"), this->mt_type, this->mt_subtype);
|
||||
}
|
||||
};
|
||||
|
||||
struct external_file_format {
|
||||
mime_type eff_mime_type;
|
||||
std::string eff_converter;
|
||||
ghc::filesystem::path eff_source_path;
|
||||
};
|
||||
|
||||
file_format_t detect_file_format(const ghc::filesystem::path& filename);
|
||||
|
||||
nonstd::optional<external_file_format> detect_mime_type(
|
||||
const ghc::filesystem::path& filename);
|
||||
|
||||
namespace fmt {
|
||||
template<>
|
||||
struct formatter<file_format_t> : formatter<string_view> {
|
||||
@ -59,9 +87,6 @@ struct formatter<file_format_t> : formatter<string_view> {
|
||||
case file_format_t::ARCHIVE:
|
||||
name = "\U0001F5C4 Archive";
|
||||
break;
|
||||
case file_format_t::PCAP:
|
||||
name = "\U0001F5A5 Pcap";
|
||||
break;
|
||||
case file_format_t::REMOTE:
|
||||
name = "\U0001F5A5 Remote";
|
||||
break;
|
||||
|
@ -348,8 +348,9 @@ listview_curses::do_update()
|
||||
}
|
||||
}
|
||||
|
||||
if (this->lv_selectable && this->lv_selection >= 0
|
||||
&& (row > this->lv_tail_space) && (blank_rows < this->lv_tail_space)
|
||||
if (this->lv_selectable && !this->lv_sync_selection_and_top
|
||||
&& this->lv_selection >= 0 && (row > this->lv_tail_space)
|
||||
&& (blank_rows < this->lv_tail_space)
|
||||
&& ((row - this->lv_tail_space) < this->lv_selection))
|
||||
{
|
||||
this->shift_top(this->lv_selection - row + this->lv_tail_space);
|
||||
|
@ -2449,6 +2449,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
|
||||
}
|
||||
|
||||
load_config(lnav_data.ld_config_paths, config_errors);
|
||||
|
||||
if (!config_errors.empty()) {
|
||||
if (print_user_msgs(config_errors, mode_flags) != EXIT_SUCCESS) {
|
||||
return EXIT_FAILURE;
|
||||
|
@ -81,6 +81,9 @@ lnav_config_listener* lnav_config_listener::LISTENER_LIST;
|
||||
static auto a = injector::bind<archive_manager::config>::to_instance(
|
||||
+[]() { return &lnav_config.lc_archive_manager; });
|
||||
|
||||
static auto ff = injector::bind<lnav::file_formats::config>::to_instance(
|
||||
+[]() { return &lnav_config.lc_file_formats; });
|
||||
|
||||
static auto fvc = injector::bind<file_vtab::config>::to_instance(
|
||||
+[]() { return &lnav_config.lc_file_vtab; });
|
||||
|
||||
@ -1040,6 +1043,55 @@ static const struct json_path_container ui_handlers = {
|
||||
.with_children(keymap_defs_handlers),
|
||||
};
|
||||
|
||||
static const struct json_path_container header_expr_handlers = {
|
||||
yajlpp::pattern_property_handler("(?<header_expr_name>\\w+)")
|
||||
.with_description("SQLite expression")
|
||||
.for_field(&lnav::file_formats::header_exprs::he_exprs),
|
||||
};
|
||||
|
||||
static const struct json_path_container header_handlers = {
|
||||
yajlpp::property_handler("expr")
|
||||
.with_description("The expressions used to check if a file header "
|
||||
"matches this file format")
|
||||
.for_child(&lnav::file_formats::header::h_exprs)
|
||||
.with_children(header_expr_handlers),
|
||||
yajlpp::property_handler("size")
|
||||
.with_description("The minimum size required for this header type")
|
||||
.for_field(&lnav::file_formats::header::h_size),
|
||||
};
|
||||
|
||||
static const struct json_path_container format_def_handlers = {
|
||||
yajlpp::property_handler("title")
|
||||
.with_description("The display name for this file format")
|
||||
.for_field(&lnav::file_formats::format_def::fd_title),
|
||||
yajlpp::property_handler("header")
|
||||
.with_description("File header detection definitions")
|
||||
.for_child(&lnav::file_formats::format_def::fd_header)
|
||||
.with_children(header_handlers),
|
||||
yajlpp::property_handler("converter")
|
||||
.with_description("The script used to convert the file")
|
||||
.with_pattern(R"([\w\.\-]+)")
|
||||
.for_field(&lnav::file_formats::format_def::fd_converter),
|
||||
};
|
||||
|
||||
static const struct json_path_container format_defs_handlers = {
|
||||
yajlpp::pattern_property_handler(R"((?<mime_type>\w+~1[\w\.\-]+))")
|
||||
.with_description("File format definitions, keyed by their MIME type")
|
||||
.with_obj_provider<lnav::file_formats::format_def, _lnav_config>(
|
||||
[](const yajlpp_provider_context& ypc, _lnav_config* root) {
|
||||
auto& retval
|
||||
= root->lc_file_formats.c_defs[ypc.get_substr("mime_type")];
|
||||
return &retval;
|
||||
})
|
||||
.with_path_provider<_lnav_config>(
|
||||
[](struct _lnav_config* cfg, std::vector<std::string>& paths_out) {
|
||||
for (const auto& iter : cfg->lc_file_formats.c_defs) {
|
||||
paths_out.emplace_back(iter.first);
|
||||
}
|
||||
})
|
||||
.with_children(format_def_handlers),
|
||||
};
|
||||
|
||||
static const struct json_path_container archive_handlers = {
|
||||
yajlpp::property_handler("min-free-space")
|
||||
.with_synopsis("<bytes>")
|
||||
@ -1267,6 +1319,9 @@ static const struct json_path_container tuning_handlers = {
|
||||
yajlpp::property_handler("file-vtab")
|
||||
.with_description("Settings related to the lnav_file virtual-table")
|
||||
.with_children(file_vtab_handlers),
|
||||
yajlpp::property_handler("file-format")
|
||||
.with_description("sdfjdls")
|
||||
.with_children(format_defs_handlers),
|
||||
yajlpp::property_handler("logfile")
|
||||
.with_description("Settings related to log files")
|
||||
.with_children(logfile_handlers),
|
||||
@ -1654,28 +1709,31 @@ reload_config(std::vector<lnav::console::user_message>& errors)
|
||||
auto cb = [&cfg_value, &errors, &errmsg](
|
||||
const json_path_handler_base& jph,
|
||||
const std::string& path,
|
||||
void* mem) {
|
||||
const void* mem) {
|
||||
if (mem != cfg_value) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto loc_iter
|
||||
= lnav_config_locations.find(intern_string::lookup(path));
|
||||
if (loc_iter == lnav_config_locations.end()) {
|
||||
return;
|
||||
auto has_loc = loc_iter != lnav_config_locations.end();
|
||||
auto um
|
||||
= lnav::console::user_message::error(
|
||||
attr_line_t()
|
||||
.append(has_loc ? "invalid value for property "
|
||||
: "missing value for property ")
|
||||
.append_quoted(lnav::roles::symbol(path)))
|
||||
.with_reason(errmsg)
|
||||
.with_help(jph.get_help_text(path));
|
||||
|
||||
if (has_loc) {
|
||||
um.with_snippet(
|
||||
lnav::console::snippet::from(loc_iter->second.sl_source,
|
||||
"")
|
||||
.with_line(loc_iter->second.sl_line_number));
|
||||
}
|
||||
|
||||
errors.emplace_back(
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t()
|
||||
.append("invalid value for property ")
|
||||
.append_quoted(lnav::roles::symbol(path)))
|
||||
.with_reason(errmsg)
|
||||
.with_snippet(
|
||||
lnav::console::snippet::from(
|
||||
loc_iter->second.sl_source, "")
|
||||
.with_line(loc_iter->second.sl_line_number))
|
||||
.with_help(jph.get_help_text(path)));
|
||||
errors.emplace_back(um);
|
||||
};
|
||||
|
||||
for (const auto& jph : lnav_config_handlers.jpc_children) {
|
||||
|
@ -43,6 +43,7 @@
|
||||
#include "base/file_range.hh"
|
||||
#include "base/lnav.console.hh"
|
||||
#include "base/result.h"
|
||||
#include "file_format.cfg.hh"
|
||||
#include "file_vtab.cfg.hh"
|
||||
#include "ghc/filesystem.hpp"
|
||||
#include "lnav_config_fwd.hh"
|
||||
@ -111,6 +112,7 @@ struct _lnav_config {
|
||||
|
||||
archive_manager::config lc_archive_manager;
|
||||
lnav::piper::config lc_piper;
|
||||
lnav::file_formats::config lc_file_formats;
|
||||
file_vtab::config lc_file_vtab;
|
||||
lnav::logfile::config lc_logfile;
|
||||
tailer::config lc_tailer;
|
||||
|
@ -2950,13 +2950,15 @@ external_log_format::match_name(const std::string& filename)
|
||||
}
|
||||
|
||||
bool
|
||||
external_log_format::match_mime_type(const file_format_t ff) const
|
||||
external_log_format::match_mime_type(const mime_type& mt) const
|
||||
{
|
||||
if (ff == file_format_t::UNKNOWN && this->elf_mime_types.empty()) {
|
||||
if (mt.mt_type == "text" && mt.mt_subtype == "plain"
|
||||
&& this->elf_mime_types.empty())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return this->elf_mime_types.count(ff) == 1;
|
||||
return this->elf_mime_types.count(mt) == 1;
|
||||
}
|
||||
|
||||
auto
|
||||
|
@ -352,9 +352,9 @@ public:
|
||||
|
||||
virtual bool match_name(const std::string& filename) { return true; }
|
||||
|
||||
virtual bool match_mime_type(const file_format_t ff) const
|
||||
virtual bool match_mime_type(const mime_type& mt) const
|
||||
{
|
||||
if (ff == file_format_t::UNKNOWN) {
|
||||
if (mt.mt_type == "text" && mt.mt_subtype == "plain") {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
@ -141,7 +141,7 @@ public:
|
||||
|
||||
bool match_name(const std::string& filename) override;
|
||||
|
||||
bool match_mime_type(const file_format_t ff) const override;
|
||||
bool match_mime_type(const mime_type& mt) const override;
|
||||
|
||||
scan_result_t scan(logfile& lf,
|
||||
std::vector<logline>& dst,
|
||||
@ -313,7 +313,7 @@ public:
|
||||
std::vector<ghc::filesystem::path> elf_format_source_order;
|
||||
std::map<intern_string_t, int> elf_format_sources;
|
||||
std::list<intern_string_t> elf_collision;
|
||||
std::set<file_format_t> elf_mime_types;
|
||||
std::set<mime_type> elf_mime_types;
|
||||
factory_container<lnav::pcre2pp::code> elf_filename_pcre;
|
||||
std::map<std::string, std::shared_ptr<pattern>> elf_patterns;
|
||||
std::vector<std::shared_ptr<pattern>> elf_pattern_order;
|
||||
|
@ -282,10 +282,7 @@ read_format_field(yajlpp_parse_context* ypc,
|
||||
elf->elf_module_id_field = intern_string::lookup(value);
|
||||
elf->elf_container = true;
|
||||
} else if (field_name == "mime-types") {
|
||||
auto value_opt = ypc->ypc_current_handler->to_enum_value(value);
|
||||
if (value_opt) {
|
||||
elf->elf_mime_types.insert((file_format_t) *value_opt);
|
||||
}
|
||||
elf->elf_mime_types.insert(mime_type::from_str(value));
|
||||
}
|
||||
|
||||
return 1;
|
||||
@ -820,15 +817,6 @@ static const struct json_path_container search_table_handlers = {
|
||||
.with_children(search_table_def_handlers),
|
||||
};
|
||||
|
||||
static const json_path_handler_base::enum_value_t MIME_TYPE_ENUM[] = {
|
||||
{
|
||||
"application/vnd.tcpdump.pcap",
|
||||
file_format_t::PCAP,
|
||||
},
|
||||
|
||||
json_path_handler_base::ENUM_TERMINATOR,
|
||||
};
|
||||
|
||||
const struct json_path_container format_handlers = {
|
||||
yajlpp::property_handler("regex")
|
||||
.with_description(
|
||||
@ -858,8 +846,9 @@ const struct json_path_container format_handlers = {
|
||||
"log files with a matching name")
|
||||
.for_field(&external_log_format::elf_filename_pcre),
|
||||
json_path_handler("mime-types#", read_format_field)
|
||||
.with_description("A list of mime-types this format should be used for")
|
||||
.with_enum_values(MIME_TYPE_ENUM),
|
||||
.with_pattern(R"(^\w/[\w\.]+)")
|
||||
.with_description(
|
||||
"A list of mime-types this format should be used for"),
|
||||
json_path_handler("level-field")
|
||||
.with_description(
|
||||
"The name of the level field in the log message pattern")
|
||||
|
@ -259,7 +259,10 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
||||
this->lf_mismatched_formats.insert(curr->get_name());
|
||||
continue;
|
||||
}
|
||||
if (!curr->match_mime_type(this->lf_options.loo_file_format)) {
|
||||
if (this->lf_options.loo_mime_type
|
||||
&& !curr->match_mime_type(
|
||||
this->lf_options.loo_mime_type.value()))
|
||||
{
|
||||
if (li.li_file_range.fr_offset == 0) {
|
||||
log_debug("(%s) does not match file format: %s",
|
||||
curr->get_name().get(),
|
||||
|
@ -66,6 +66,7 @@ struct logfile_open_options_base {
|
||||
ssize_t loo_visible_size_limit{-1};
|
||||
bool loo_tail{true};
|
||||
file_format_t loo_file_format{file_format_t::UNKNOWN};
|
||||
nonstd::optional<mime_type> loo_mime_type;
|
||||
nonstd::optional<lnav::piper::running_handle> loo_piper;
|
||||
};
|
||||
|
||||
|
@ -420,7 +420,7 @@ add_config_possibilities()
|
||||
std::set<std::string> visited;
|
||||
auto cb = [rc, &visited](const json_path_handler_base& jph,
|
||||
const std::string& path,
|
||||
void* mem) {
|
||||
const void* mem) {
|
||||
if (jph.jph_children) {
|
||||
const auto named_caps = jph.jph_regex->get_named_captures();
|
||||
|
||||
|
@ -26,6 +26,19 @@
|
||||
"transfer-command": "cat > {0:} && chmod ugo+rx ./{0:}"
|
||||
}
|
||||
},
|
||||
"file-format": {
|
||||
"application/vnd.tcpdump.pcap": {
|
||||
"title": "\ud83d\udda5 Pcap",
|
||||
"header": {
|
||||
"expr": {
|
||||
"pcapng": ":header REGEXP '^0a0d0d0a.{8}(?:1a2b3c4d|4d3c2b1a).*'",
|
||||
"pcap": ":header REGEXP '^(?:a1b2c3d4|d4c3b2a1|a1b23c4d|4d3cb2a1).*'"
|
||||
},
|
||||
"size": 24
|
||||
},
|
||||
"converter": "pcap-converter.sh"
|
||||
}
|
||||
},
|
||||
"piper": {
|
||||
"max-size": 10485760,
|
||||
"rotations": 4
|
||||
|
10
src/scripts/pcap-converter.sh
Executable file
10
src/scripts/pcap-converter.sh
Executable file
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check that tshark is installed and return a nice message.
|
||||
if ! command -v tshark; then
|
||||
echo "pcap support requires 'tshark' v3+ to be installed" > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use tshark to convert the pcap file into a JSON-lines log file
|
||||
exec tshark -T ek -P -V -t ad -r $2
|
@ -9,4 +9,5 @@ BUILTIN_LNAVSCRIPTS = \
|
||||
|
||||
BUILTIN_SHSCRIPTS = \
|
||||
$(srcdir)/scripts/dump-pid.sh \
|
||||
$(srcdir)/scripts/pcap-converter.sh \
|
||||
$()
|
||||
|
@ -62,6 +62,7 @@ sqlite_registration_func_t sqlite_registration_funcs[] = {
|
||||
int
|
||||
register_sqlite_funcs(sqlite3* db, sqlite_registration_func_t* reg_funcs)
|
||||
{
|
||||
static bool help_registration_done = false;
|
||||
int lpc;
|
||||
|
||||
require(db != nullptr);
|
||||
@ -94,7 +95,9 @@ register_sqlite_funcs(sqlite3* db, sqlite_registration_func_t* reg_funcs)
|
||||
nullptr,
|
||||
nullptr);
|
||||
|
||||
if (fd.fd_help.ht_context != help_context_t::HC_NONE) {
|
||||
if (!help_registration_done
|
||||
&& fd.fd_help.ht_context != help_context_t::HC_NONE)
|
||||
{
|
||||
help_text& ht = fd.fd_help;
|
||||
|
||||
sqlite_function_help.insert(std::make_pair(ht.ht_name, &ht));
|
||||
@ -115,7 +118,9 @@ register_sqlite_funcs(sqlite3* db, sqlite_registration_func_t* reg_funcs)
|
||||
agg_funcs[i].xStep,
|
||||
agg_funcs[i].xFinalize);
|
||||
|
||||
if (fda.fda_help.ht_context != help_context_t::HC_NONE) {
|
||||
if (!help_registration_done
|
||||
&& fda.fda_help.ht_context != help_context_t::HC_NONE)
|
||||
{
|
||||
help_text& ht = fda.fda_help;
|
||||
|
||||
sqlite_function_help.insert(std::make_pair(ht.ht_name, &ht));
|
||||
@ -746,9 +751,11 @@ register_sqlite_funcs(sqlite3* db, sqlite_registration_func_t* reg_funcs)
|
||||
.with_example({"To count down from five to 1",
|
||||
"SELECT value FROM generate_series(1, 5, -1)"})};
|
||||
|
||||
for (auto& ht : builtin_funcs) {
|
||||
sqlite_function_help.insert(std::make_pair(ht.ht_name, &ht));
|
||||
ht.index_tags();
|
||||
if (!help_registration_done) {
|
||||
for (auto& ht : builtin_funcs) {
|
||||
sqlite_function_help.insert(std::make_pair(ht.ht_name, &ht));
|
||||
ht.index_tags();
|
||||
}
|
||||
}
|
||||
|
||||
static help_text builtin_win_funcs[] = {
|
||||
@ -847,9 +854,11 @@ register_sqlite_funcs(sqlite3* db, sqlite_registration_func_t* reg_funcs)
|
||||
.with_tags({"window"}),
|
||||
};
|
||||
|
||||
for (auto& ht : builtin_win_funcs) {
|
||||
sqlite_function_help.insert(std::make_pair(ht.ht_name, &ht));
|
||||
ht.index_tags();
|
||||
if (!help_registration_done) {
|
||||
for (auto& ht : builtin_win_funcs) {
|
||||
sqlite_function_help.insert(std::make_pair(ht.ht_name, &ht));
|
||||
ht.index_tags();
|
||||
}
|
||||
}
|
||||
|
||||
static help_text idents[] = {
|
||||
@ -1152,16 +1161,20 @@ register_sqlite_funcs(sqlite3* db, sqlite_registration_func_t* reg_funcs)
|
||||
.optional()),
|
||||
};
|
||||
|
||||
for (auto& ht : idents) {
|
||||
sqlite_function_help.insert(make_pair(toupper(ht.ht_name), &ht));
|
||||
for (const auto& param : ht.ht_parameters) {
|
||||
if (!param.ht_flag_name) {
|
||||
continue;
|
||||
if (!help_registration_done) {
|
||||
for (auto& ht : idents) {
|
||||
sqlite_function_help.insert(make_pair(toupper(ht.ht_name), &ht));
|
||||
for (const auto& param : ht.ht_parameters) {
|
||||
if (!param.ht_flag_name) {
|
||||
continue;
|
||||
}
|
||||
sqlite_function_help.insert(
|
||||
make_pair(toupper(param.ht_flag_name), &ht));
|
||||
}
|
||||
sqlite_function_help.insert(
|
||||
make_pair(toupper(param.ht_flag_name), &ht));
|
||||
}
|
||||
}
|
||||
|
||||
help_registration_done = true;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -103,7 +103,10 @@ bind_values(sqlite3_stmt* stmt, Args... args)
|
||||
}
|
||||
|
||||
struct prepared_stmt {
|
||||
prepared_stmt(auto_mem<sqlite3_stmt> stmt) : ps_stmt(std::move(stmt)) {}
|
||||
explicit prepared_stmt(auto_mem<sqlite3_stmt> stmt)
|
||||
: ps_stmt(std::move(stmt))
|
||||
{
|
||||
}
|
||||
|
||||
Result<void, std::string> execute()
|
||||
{
|
||||
@ -195,10 +198,12 @@ prepare_stmt(sqlite3* db, const char* sql, Args... args)
|
||||
sqlite3_errmsg(db)));
|
||||
}
|
||||
|
||||
if (bind_values(retval.in(), args...) != SQLITE_OK) {
|
||||
return Err(
|
||||
fmt::format(FMT_STRING("unable to prepare SQL statement: {}"),
|
||||
sqlite3_errmsg(db)));
|
||||
if (sizeof...(args) > 0) {
|
||||
if (bind_values(retval.in(), args...) != SQLITE_OK) {
|
||||
return Err(
|
||||
fmt::format(FMT_STRING("unable to prepare SQL statement: {}"),
|
||||
sqlite3_errmsg(db)));
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(prepared_stmt{
|
||||
|
@ -549,7 +549,7 @@ sql_encode(sqlite3_value* value, encode_algo algo)
|
||||
|
||||
for (int lpc = 0; lpc < text_len; lpc++) {
|
||||
fmt::format_to(std::back_inserter(buf),
|
||||
FMT_STRING("{:x}"),
|
||||
FMT_STRING("{:02x}"),
|
||||
text[lpc]);
|
||||
}
|
||||
|
||||
|
@ -36,7 +36,7 @@
|
||||
"bold": true
|
||||
},
|
||||
"cursor-line": {
|
||||
"color": "Cyan",
|
||||
"color": "Cyan1",
|
||||
"background-color": "Red",
|
||||
"bold": true,
|
||||
"underline": true
|
||||
|
@ -226,6 +226,19 @@ json_ptr::encode(char* dst, size_t dst_len, const char* src, size_t src_len)
|
||||
return retval;
|
||||
}
|
||||
|
||||
std::string
|
||||
json_ptr::encode_str(const char* src, size_t src_len)
|
||||
{
|
||||
if (src_len == (size_t) -1) {
|
||||
src_len = strlen(src);
|
||||
}
|
||||
|
||||
char retval[src_len + 1];
|
||||
auto rc = encode(retval, sizeof(retval), src, src_len);
|
||||
|
||||
return std::string(retval, rc);
|
||||
}
|
||||
|
||||
size_t
|
||||
json_ptr::decode(char* dst, const char* src, ssize_t src_len)
|
||||
{
|
||||
|
@ -64,7 +64,8 @@ public:
|
||||
void inc_array_index()
|
||||
{
|
||||
if (!this->jpw_array_indexes.empty()
|
||||
&& this->jpw_array_indexes.back() != -1) {
|
||||
&& this->jpw_array_indexes.back() != -1)
|
||||
{
|
||||
this->jpw_array_indexes.back() += 1;
|
||||
}
|
||||
}
|
||||
@ -108,6 +109,12 @@ public:
|
||||
const char* src,
|
||||
size_t src_len = -1);
|
||||
|
||||
static std::string encode_str(const char* src, size_t src_len = -1);
|
||||
static std::string encode_str(const std::string& src)
|
||||
{
|
||||
return encode_str(src.c_str(), src.size());
|
||||
}
|
||||
|
||||
static size_t decode(char* dst, const char* src, ssize_t src_len = -1);
|
||||
|
||||
json_ptr(const char* value) : jp_value(value), jp_pos(value) {}
|
||||
|
@ -213,7 +213,7 @@ json_path_handler_base::gen(yajlpp_gen_context& ygc, yajl_gen handle) const
|
||||
|
||||
if (this->jph_children) {
|
||||
for (const auto& lpath : local_paths) {
|
||||
std::string full_path = lpath;
|
||||
std::string full_path = json_ptr::encode_str(lpath);
|
||||
if (this->jph_path_provider) {
|
||||
full_path += "/";
|
||||
}
|
||||
@ -455,8 +455,8 @@ json_path_handler_base::gen_schema_type(yajlpp_gen_context& ygc) const
|
||||
|
||||
void
|
||||
json_path_handler_base::walk(
|
||||
const std::function<
|
||||
void(const json_path_handler_base&, const std::string&, void*)>& cb,
|
||||
const std::function<void(
|
||||
const json_path_handler_base&, const std::string&, const void*)>& cb,
|
||||
void* root,
|
||||
const std::string& base) const
|
||||
{
|
||||
@ -465,11 +465,11 @@ json_path_handler_base::walk(
|
||||
if (this->jph_path_provider) {
|
||||
this->jph_path_provider(root, local_paths);
|
||||
|
||||
for (auto& lpath : local_paths) {
|
||||
for (const auto& lpath : local_paths) {
|
||||
cb(*this,
|
||||
fmt::format(FMT_STRING("{}{}{}"),
|
||||
base,
|
||||
lpath,
|
||||
json_ptr::encode_str(lpath),
|
||||
this->jph_children ? "/" : ""),
|
||||
nullptr);
|
||||
}
|
||||
@ -477,6 +477,12 @@ json_path_handler_base::walk(
|
||||
local_paths.clear();
|
||||
this->jph_path_provider(root, local_paths);
|
||||
}
|
||||
if (this->jph_field_getter) {
|
||||
const auto* field = this->jph_field_getter(root, nonstd::nullopt);
|
||||
if (field != nullptr) {
|
||||
cb(*this, base, field);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
local_paths.emplace_back(this->jph_property);
|
||||
|
||||
@ -484,6 +490,7 @@ json_path_handler_base::walk(
|
||||
if (this->jph_children) {
|
||||
full_path += "/";
|
||||
}
|
||||
|
||||
cb(*this, full_path, nullptr);
|
||||
}
|
||||
|
||||
@ -493,7 +500,7 @@ json_path_handler_base::walk(
|
||||
static const intern_string_t POSS_SRC
|
||||
= intern_string::lookup("possibilities");
|
||||
|
||||
std::string full_path = base + lpath;
|
||||
std::string full_path = base + json_ptr::encode_str(lpath);
|
||||
if (this->jph_children) {
|
||||
full_path += "/";
|
||||
}
|
||||
@ -509,13 +516,18 @@ json_path_handler_base::walk(
|
||||
static thread_local auto md
|
||||
= lnav::pcre2pp::match_data::unitialized();
|
||||
|
||||
std::string full_path = lpath + "/";
|
||||
const auto short_path = json_ptr::encode_str(lpath) + "/";
|
||||
|
||||
if (!this->jph_regex->capture_from(full_path)
|
||||
if (!this->jph_regex->capture_from(short_path)
|
||||
.into(md)
|
||||
.matches()
|
||||
.ignore_error())
|
||||
{
|
||||
log_error(
|
||||
"path-handler regex (%s) does not match path: "
|
||||
"%s",
|
||||
this->jph_regex->get_pattern().c_str(),
|
||||
full_path.c_str());
|
||||
ensure(false);
|
||||
}
|
||||
child_root = this->jph_obj_provider(
|
||||
@ -527,7 +539,7 @@ json_path_handler_base::walk(
|
||||
}
|
||||
} else {
|
||||
for (auto& lpath : local_paths) {
|
||||
void* field = nullptr;
|
||||
const void* field = nullptr;
|
||||
|
||||
if (this->jph_field_getter) {
|
||||
field = this->jph_field_getter(root, lpath);
|
||||
|
@ -228,11 +228,11 @@ struct json_path_handler_base {
|
||||
yajl_gen_status gen(yajlpp_gen_context& ygc, yajl_gen handle) const;
|
||||
yajl_gen_status gen_schema(yajlpp_gen_context& ygc) const;
|
||||
yajl_gen_status gen_schema_type(yajlpp_gen_context& ygc) const;
|
||||
void walk(
|
||||
const std::function<
|
||||
void(const json_path_handler_base&, const std::string&, void*)>& cb,
|
||||
void* root = nullptr,
|
||||
const std::string& base = "/") const;
|
||||
void walk(const std::function<void(const json_path_handler_base&,
|
||||
const std::string&,
|
||||
const void*)>& cb,
|
||||
void* root = nullptr,
|
||||
const std::string& base = "/") const;
|
||||
|
||||
enum class schema_type_t : std::uint32_t {
|
||||
ANY,
|
||||
@ -255,7 +255,7 @@ struct json_path_handler_base {
|
||||
std::function<void(yajlpp_parse_context& ypc,
|
||||
const json_path_handler_base& jph)>
|
||||
jph_validator;
|
||||
std::function<void*(void* root, nonstd::optional<std::string> name)>
|
||||
std::function<const void*(void* root, nonstd::optional<std::string> name)>
|
||||
jph_field_getter;
|
||||
std::function<void*(const yajlpp_provider_context& pe, void* root)>
|
||||
jph_obj_provider;
|
||||
@ -452,6 +452,16 @@ public:
|
||||
return obj->*MEM;
|
||||
}
|
||||
|
||||
void fill_in_source()
|
||||
{
|
||||
if (this->ypc_locations != nullptr) {
|
||||
(*this->ypc_locations)[this->get_full_path()] = source_location{
|
||||
this->ypc_source,
|
||||
this->get_line_number(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const intern_string_t ypc_source;
|
||||
int ypc_line_number{1};
|
||||
const struct json_path_container* ypc_handlers;
|
||||
|
@ -241,11 +241,13 @@ struct json_path_handler : public json_path_handler_base {
|
||||
|
||||
static int null_field_cb(yajlpp_parse_context* ypc)
|
||||
{
|
||||
ypc->fill_in_source();
|
||||
return ypc->ypc_current_handler->jph_null_cb(ypc);
|
||||
}
|
||||
|
||||
static int bool_field_cb(yajlpp_parse_context* ypc, int val)
|
||||
{
|
||||
ypc->fill_in_source();
|
||||
return ypc->ypc_current_handler->jph_bool_cb(ypc, val);
|
||||
}
|
||||
|
||||
@ -253,16 +255,19 @@ struct json_path_handler : public json_path_handler_base {
|
||||
const unsigned char* str,
|
||||
size_t len)
|
||||
{
|
||||
ypc->fill_in_source();
|
||||
return ypc->ypc_current_handler->jph_str_cb(ypc, str, len);
|
||||
}
|
||||
|
||||
static int int_field_cb(yajlpp_parse_context* ypc, long long val)
|
||||
{
|
||||
ypc->fill_in_source();
|
||||
return ypc->ypc_current_handler->jph_integer_cb(ypc, val);
|
||||
}
|
||||
|
||||
static int dbl_field_cb(yajlpp_parse_context* ypc, double val)
|
||||
{
|
||||
ypc->fill_in_source();
|
||||
return ypc->ypc_current_handler->jph_double_cb(ypc, val);
|
||||
}
|
||||
|
||||
@ -525,6 +530,28 @@ struct json_path_handler : public json_path_handler_base {
|
||||
|
||||
return 1;
|
||||
};
|
||||
this->jph_path_provider =
|
||||
[args...](void* root, std::vector<std::string>& paths_out) {
|
||||
const auto& field = json_path_handler::get_field(root, args...);
|
||||
|
||||
for (const auto& pair : field) {
|
||||
paths_out.emplace_back(pair.first);
|
||||
}
|
||||
};
|
||||
this->jph_field_getter
|
||||
= [args...](void* root,
|
||||
nonstd::optional<std::string> name) -> const void* {
|
||||
const auto& field = json_path_handler::get_field(root, args...);
|
||||
if (!name) {
|
||||
return &field;
|
||||
}
|
||||
|
||||
auto iter = field.find(name.value());
|
||||
if (iter == field.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
return (void*) &iter->second;
|
||||
};
|
||||
this->jph_gen_callback = [args...](yajlpp_gen_context& ygc,
|
||||
const json_path_handler_base& jph,
|
||||
yajl_gen handle) {
|
||||
@ -897,6 +924,10 @@ struct json_path_handler : public json_path_handler_base {
|
||||
|
||||
return gen(field.pp_value);
|
||||
};
|
||||
this->jph_field_getter
|
||||
= [args...](void* root, nonstd::optional<std::string> name) {
|
||||
return (void*) &json_path_handler::get_field(root, args...);
|
||||
};
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
@ -235,6 +235,7 @@ dist_noinst_DATA = \
|
||||
bad-config/formats/invalid-sql/init2.sql \
|
||||
bad-config/formats/no-regexes/format.json \
|
||||
bad-config/formats/no-samples/format.json \
|
||||
bad-config2/configs/invalid-file-format/config.json \
|
||||
bad-config2/formats/invalid-config/config.json \
|
||||
bad-config2/formats/invalid-config/config.bad-schema.json \
|
||||
bad-config2/formats/invalid-config/config.malformed.json \
|
||||
|
19
test/bad-config2/configs/invalid-file-format/config.json
Normal file
19
test/bad-config2/configs/invalid-file-format/config.json
Normal file
@ -0,0 +1,19 @@
|
||||
{
|
||||
"$schema": "https://lnav.org/schemas/config-v1.schema.json",
|
||||
"tuning": {
|
||||
"file-format": {
|
||||
"application/vnd.example.com": {
|
||||
"title": "example",
|
||||
"header": {
|
||||
"expr": {
|
||||
"default": ":header REGEXP 'foobar"
|
||||
},
|
||||
"size": 8
|
||||
}
|
||||
},
|
||||
"application/vnd.example2.com": {
|
||||
"title": "example"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -202,8 +202,6 @@ EXPECTED_FILES = \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d3b69abdfb39e4bfa5828c2f9593e2b2b7ed4d5d.out \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d76d77ad95b9f120825417a6a8220c13df9541fc.err \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d76d77ad95b9f120825417a6a8220c13df9541fc.out \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d7eebacdcf2cb194f25fa4ef97b7b5376b442467.err \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d7eebacdcf2cb194f25fa4ef97b7b5376b442467.out \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d836c84398c831c976df46f46fe3bf5983c44c37.err \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d836c84398c831c976df46f46fe3bf5983c44c37.out \
|
||||
$(srcdir)/%reldir%/test_cmds.sh_d8eeef53a58bdeddbc1028d7c525413e3ca1c8df.err \
|
||||
@ -322,8 +320,6 @@ EXPECTED_FILES = \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_08d731a04c877a34819b35de185e30a74c9fd497.out \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_09bd16e044302f6b121092534708594bdad11b5a.err \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_09bd16e044302f6b121092534708594bdad11b5a.out \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_1c6eee38f66356fcd9a9f0faedaea6dbcc901060.err \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_1c6eee38f66356fcd9a9f0faedaea6dbcc901060.out \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_218ecb88b4753010c4264b3ac351260b4811612f.err \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_218ecb88b4753010c4264b3ac351260b4811612f.out \
|
||||
$(srcdir)/%reldir%/test_logfile.sh_290a3c49e53c2229a7400c107338fa0bb38375e2.err \
|
||||
@ -804,8 +800,6 @@ EXPECTED_FILES = \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_352434d199f7b493668c9f2774472eb69ef0d9f0.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_36fc9005464f1106f969559e640d9fa36d5fadad.err \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_36fc9005464f1106f969559e640d9fa36d5fadad.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_3855d2cc0ab29171cae8e722f130adec25eae36e.err \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_3855d2cc0ab29171cae8e722f130adec25eae36e.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_3de72fe5c1751dd212a1cd45cf2caa7f3b52bced.err \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_3de72fe5c1751dd212a1cd45cf2caa7f3b52bced.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_4b402274da152135c6c99456b693e1ecabca0256.err \
|
||||
@ -830,6 +824,8 @@ EXPECTED_FILES = \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_6607c0dd8baff16930eb3e0daf6354af5b50052b.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.err \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_6ac7ab1f90c064944ff66bef5974f050c8227d4b.err \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_6ac7ab1f90c064944ff66bef5974f050c8227d4b.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_6ff984d8ed3e5099376d19f0dd20d5fd1ed42494.err \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_6ff984d8ed3e5099376d19f0dd20d5fd1ed42494.out \
|
||||
$(srcdir)/%reldir%/test_sql_str_func.sh_71f37db33504b2c08a7a3323c482556f53d88100.err \
|
||||
@ -1010,8 +1006,6 @@ EXPECTED_FILES = \
|
||||
$(srcdir)/%reldir%/test_sql_xml_func.sh_b036c73528a446cba46625767517cdac868aba72.out \
|
||||
$(srcdir)/%reldir%/test_sql_xml_func.sh_fefeb387ae14d4171225ea06cbbff3ec43990cf0.err \
|
||||
$(srcdir)/%reldir%/test_sql_xml_func.sh_fefeb387ae14d4171225ea06cbbff3ec43990cf0.out \
|
||||
$(srcdir)/%reldir%/test_sql_yaml_func.sh_41c6abde708a69e74f5b7fde865d88fa75f91e0a.err \
|
||||
$(srcdir)/%reldir%/test_sql_yaml_func.sh_41c6abde708a69e74f5b7fde865d88fa75f91e0a.out \
|
||||
$(srcdir)/%reldir%/test_sql_yaml_func.sh_dc189d02e8979b7ed245d5d750f68b9965984699.err \
|
||||
$(srcdir)/%reldir%/test_sql_yaml_func.sh_dc189d02e8979b7ed245d5d750f68b9965984699.out \
|
||||
$(srcdir)/%reldir%/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.err \
|
||||
|
@ -36,3 +36,34 @@
|
||||
[1m[31m✘ error[0m: invalid JSON
|
||||
[1m[31mreason[0m: parse error: premature EOF
|
||||
[36m --> [0m[1m{test_dir}/bad-config2/formats/invalid-config/config.truncated.json[0m:3
|
||||
[1m[31m✘ error[0m: invalid value for property “[1m/tuning/file-format/application~1vnd.example.com/header/expr/default[0m”
|
||||
[1m[31mreason[0m: SQL expression is invalid
|
||||
[1m[31m | [0m [1m[31mreason[0m: unrecognized token: "'foobar"
|
||||
[1m[31m | [0m [36m --> [0m[1m/tuning/file-formats/application~1vnd.example.com/header/expr/default[0m
|
||||
[1m[31m | [0m [36m | [0m[37m[40m:header[0m[37m[40m [0m[1m[36m[40mREGEXP[0m[37m[40m [0m[1m[7m[31m[40m'[0m[37m[40mfoobar [0m
|
||||
[36m --> [0m[1m{test_dir}/bad-config2/configs/invalid-file-format/config.json[0m:9
|
||||
[36m =[0m [36mhelp[0m: [1mProperty Synopsis[0m
|
||||
[1m/tuning/file-format/application~1vnd.example.com/header/expr/default[0m
|
||||
[1mDescription[0m
|
||||
SQLite expression
|
||||
[1m[31m✘ error[0m: missing value for property “[1m/tuning/file-format/application~1vnd.example.com/converter[0m”
|
||||
[1m[31mreason[0m: A converter is required for a file format
|
||||
[1m[31m | [0m [1m[31mreason[0m: The converter script transforms the file into a format that can be consumed by lnav
|
||||
[36m =[0m [36mhelp[0m: [1mProperty Synopsis[0m
|
||||
[1m/tuning/file-format/application~1vnd.example.com/converter[0m
|
||||
[1mDescription[0m
|
||||
The script used to convert the file
|
||||
[1m[31m✘ error[0m: missing value for property “[1m/tuning/file-format/application~1vnd.example2.com/header/expr/[0m”
|
||||
[1m[31mreason[0m: At least one header expression is required for a file format
|
||||
[1m[31m | [0m [1m[31mreason[0m: Header expressions are used to detect a format
|
||||
[36m =[0m [36mhelp[0m: [1mProperty Synopsis[0m
|
||||
[1m/tuning/file-format/application~1vnd.example2.com/header/expr/[0m
|
||||
[1mDescription[0m
|
||||
SQLite expression
|
||||
[1m[31m✘ error[0m: missing value for property “[1m/tuning/file-format/application~1vnd.example2.com/converter[0m”
|
||||
[1m[31mreason[0m: A converter is required for a file format
|
||||
[1m[31m | [0m [1m[31mreason[0m: The converter script transforms the file into a format that can be consumed by lnav
|
||||
[36m =[0m [36mhelp[0m: [1mProperty Synopsis[0m
|
||||
[1m/tuning/file-format/application~1vnd.example2.com/converter[0m
|
||||
[1mDescription[0m
|
||||
The script used to convert the file
|
||||
|
@ -0,0 +1,2 @@
|
||||
Row 0:
|
||||
Column encode('hi' || char(10), 'hex'): 68690a
|
@ -109,6 +109,8 @@ run_cap_test ./drive_sql "SELECT encode('foo', null)"
|
||||
|
||||
run_cap_test ./drive_sql "SELECT encode(null, 'base64')"
|
||||
|
||||
run_cap_test ./drive_sql "SELECT encode('hi' || char(10), 'hex')"
|
||||
|
||||
run_cap_test ./drive_sql "SELECT gunzip(decode(encode(gzip('Hello, World!'), 'base64'), 'base64'))"
|
||||
|
||||
#run_cap_test env TEST_COMMENT=invalid_url ./drive_sql <<'EOF'
|
||||
|
@ -11,6 +11,7 @@
|
||||
# include <alloca.h>
|
||||
#endif
|
||||
|
||||
#include <assert.h>
|
||||
#include <ctype.h>
|
||||
#include <fcntl.h>
|
||||
#include <getopt.h>
|
||||
@ -67,7 +68,7 @@ process(struct file_meta* fm, FILE* ofile)
|
||||
}
|
||||
|
||||
unsigned char* buf = malloc(st.st_size);
|
||||
unsigned char* dest = malloc(st.st_size);
|
||||
unsigned char* dest = malloc(st.st_size + 1024);
|
||||
|
||||
int fd = open(fm->fm_name, O_RDONLY);
|
||||
if (fd == -1) {
|
||||
@ -80,8 +81,9 @@ process(struct file_meta* fm, FILE* ofile)
|
||||
fm->fm_size += rc;
|
||||
}
|
||||
|
||||
uLongf destLen = st.st_size;
|
||||
compress(dest, &destLen, buf, st.st_size);
|
||||
uLongf destLen = st.st_size + 1024;
|
||||
int cres = compress(dest, &destLen, buf, st.st_size);
|
||||
assert(cres == Z_OK);
|
||||
fm->fm_compressed_size = destLen;
|
||||
|
||||
int c, col = 1;
|
||||
|
Loading…
Reference in New Issue
Block a user