diff --git a/NEWS b/NEWS index eb7a01d7..c14b64ab 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,7 @@ lnav v0.7.1: filters should be almost instantaneous. * The filter-in, filter-out, and highlight commands now support tab-completion of text in the current view. + * Add a '-i' flag that installs format files in: ~/.lnav/formats/installed lnav v0.7.0: Features: diff --git a/docs/source/formats.rst b/docs/source/formats.rst index ae8e5e8d..55d9f4a9 100644 --- a/docs/source/formats.rst +++ b/docs/source/formats.rst @@ -157,3 +157,31 @@ with the following contents:: ] } } + +Installing Formats +------------------ + +File formats are loaded from subdirectories in :file:`/etc/lnav/formats` and +:file:`~/.lnav/formats/`. You can manually create these subdirectories and +copy the format files into there. Or, you can pass the '-i' option to **lnav** +to automatically install formats from the command-line. For example:: + + $ lnav -i myformat.json + info: installed: /home/example/.lnav/formats/installed/myformat_log.json + +Formats installed using this method will be placed in the :file:`installed` +subdirectory and named based on the first format name found in the file. + +Format files can also be made executable by adding a shebang (#!) line to the +top of the file, like so:: + + #! /usr/bin/env lnav -i + { + "myformat_log" : ... + } + +Executing the format file should then install it automatically:: + + $ chmod ugo+rx myformat.json + $ ./myformat.json + info: installed: /home/example/.lnav/formats/installed/myformat_log.json diff --git a/lnav.1 b/lnav.1 index 66491ab3..175a8084 100644 --- a/lnav.1 +++ b/lnav.1 @@ -49,15 +49,22 @@ Print help and exit \fB\-H\fR Display the internal help text. .TP +\fB\-I\fR path +Add the given configuration directory to the search path. +.TP +\fB\-i\fR +Install the given format files in the $HOME/.lnav/formats/installed directory +and exit. +.TP +\fB\-C\fR +Check the configuration and exit. +.TP \fB\-d\fR file Write debug messages to the given file. .TP \fB\-V\fR Print version information. .TP -\fB\-s\fR -Load the most recent syslog messages file. -.TP \fB\-a\fR Load all of the most recent log file types. .TP diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 50e4ee42..69c6bb33 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -93,6 +93,11 @@ set(diag_STAT_SRCS time_T.hh top_status_source.hh + yajl/api/yajl_common.h + yajl/api/yajl_gen.h + yajl/api/yajl_parse.h + yajl/api/yajl_tree.h + ../../lbuild/src/config.h ) diff --git a/src/lnav.cc b/src/lnav.cc index ae95cac6..33684a96 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -2842,11 +2842,11 @@ static void usage(void) " -h Print this message, then exit.\n" " -H Display the internal help text.\n" " -I path An additional configuration directory.\n" + " -i Install the given format files and exit.\n" " -C Check configuration and then exit.\n" " -d file Write debug messages to the given file.\n" " -V Print version information.\n" "\n" - " -s Load the most recent syslog messages file.\n" " -a Load all of the most recent log file types.\n" " -r Load older rotated log files as well.\n" " -t Prepend timestamps to the lines of data being read in\n" @@ -3930,6 +3930,16 @@ int sql_progress(const struct log_cursor &lc) return 0; } +static void print_errors(vector error_list) +{ + for (std::vector::iterator iter = error_list.begin(); + iter != error_list.end(); + ++iter) { + fprintf(stderr, "%s%s", iter->c_str(), + (*iter)[iter->size() - 1] == '\n' ? "" : "\n"); + } +} + int main(int argc, char *argv[]) { std::vector loader_errors; @@ -3950,7 +3960,7 @@ int main(int argc, char *argv[]) sql_install_logger(); lnav_data.ld_debug_log_name = "/dev/null"; - while ((c = getopt(argc, argv, "hHarsCc:I:f:d:nqtw:VW")) != -1) { + while ((c = getopt(argc, argv, "hHarsCc:I:if:d:nqtw:VW")) != -1) { switch (c) { case 'h': usage(); @@ -4000,6 +4010,10 @@ int main(int argc, char *argv[]) lnav_data.ld_config_paths.push_back(optarg); break; + case 'i': + lnav_data.ld_flags |= LNF_INSTALL; + break; + case 'd': lnav_data.ld_debug_log_name = optarg; break; @@ -4055,14 +4069,55 @@ int main(int argc, char *argv[]) lnav_log_file = fopen(lnav_data.ld_debug_log_name, "a"); + if (lnav_data.ld_flags & LNF_INSTALL) { + string installed_path = dotlnav_path("formats/installed/"); + + if (argc == 0) { + fprintf(stderr, "error: expecting file format paths\n"); + return EXIT_FAILURE; + } + + for (lpc = 0; lpc < argc; lpc++) { + vector format_list = load_format_file(argv[lpc], loader_errors); + + if (!loader_errors.empty()) { + print_errors(loader_errors); + return EXIT_FAILURE; + } + if (format_list.empty()) { + fprintf(stderr, "error: format file is empty: %s\n", argv[lpc]); + return EXIT_FAILURE; + } + + string dst_name = format_list[0] + ".json"; + string dst_path = installed_path + dst_name; + auto_fd in_fd, out_fd; + + if ((in_fd = open(argv[lpc], O_RDONLY)) == -1) { + perror("unable to open file to install"); + } + else if ((out_fd = open(dst_path.c_str(), + O_WRONLY | O_CREAT, 0644)) == -1) { + fprintf(stderr, "error: unable to open destination: %s -- %s\n", + dst_path.c_str(), strerror(errno)); + } + else { + char buffer[2048]; + ssize_t rc; + + while ((rc = read(in_fd, buffer, sizeof(buffer))) > 0) { + write(out_fd, buffer, rc); + } + + fprintf(stderr, "info: installed: %s\n", dst_path.c_str()); + } + } + return EXIT_SUCCESS; + } + load_formats(lnav_data.ld_config_paths, loader_errors); if (!loader_errors.empty()) { - for (std::vector::iterator iter = loader_errors.begin(); - iter != loader_errors.end(); - ++iter) { - fprintf(stderr, "%s%s", iter->c_str(), - (*iter)[iter->size() - 1] == '\n' ? "" : "\n"); - } + print_errors(loader_errors); return EXIT_FAILURE; } diff --git a/src/lnav.hh b/src/lnav.hh index 7ca101df..6b95fb06 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -79,6 +79,7 @@ enum { LNB_QUIET, LNB_ROTATED, LNB_CHECK_CONFIG, + LNB_INSTALL, }; /** Flags set on the lnav command-line. */ @@ -92,6 +93,7 @@ typedef enum { LNF_HEADLESS = (1L << LNB_HEADLESS), LNF_QUIET = (1L << LNB_QUIET), LNF_CHECK_CONFIG = (1L << LNB_CHECK_CONFIG), + LNF_INSTALL = (1L << LNB_INSTALL), LNF__ALL = (LNF_SYSLOG|LNF_HELP) } lnav_flags_t; diff --git a/src/lnav_config.cc b/src/lnav_config.cc index 07ea6d1a..0270f9e2 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -89,7 +89,12 @@ void ensure_dotlnav(void) if (!path.empty()) { mkdir(path.c_str(), 0755); } - + + path = dotlnav_path("formats/installed"); + if (!path.empty()) { + mkdir(path.c_str(), 0755); + } + path = dotlnav_path("crash"); if (!path.empty()) { mkdir(path.c_str(), 0755); diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 2a0555a9..92c68049 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -56,6 +56,7 @@ static map LOG_FORMATS; static external_log_format *ensure_format(yajlpp_parse_context *ypc) { const string &name = ypc->get_path_fragment(0); + vector *formats = (vector *)ypc->ypc_userdata; external_log_format *retval; retval = LOG_FORMATS[name]; @@ -64,6 +65,10 @@ static external_log_format *ensure_format(yajlpp_parse_context *ypc) } retval->elf_source_path.insert(ypc->ypc_source.substr(0, ypc->ypc_source.rfind('/'))); + if (find(formats->begin(), formats->end(), name) == formats->end()) { + formats->push_back(name); + } + return retval; } @@ -415,59 +420,81 @@ static void write_sample_file(void) } } +std::vector load_format_file(const string &filename, std::vector &errors) +{ + std::vector retval; + auto_fd fd; + + log_info("loading formats from file: %s", filename.c_str()); + yajlpp_parse_context ypc(filename, format_handlers); + ypc.ypc_userdata = &retval; + if ((fd = open(filename.c_str(), O_RDONLY)) == -1) { + char errmsg[1024]; + + snprintf(errmsg, sizeof(errmsg), + "error: unable to open format file -- %s", + filename.c_str()); + errors.push_back(errmsg); + } + else { + yajl_handle handle; + char buffer[2048]; + off_t offset = 0; + int rc = -1; + + handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc); + yajl_config(handle, yajl_allow_comments, 1); + while (true) { + rc = read(fd, buffer, sizeof(buffer)); + if (rc == 0) { + break; + } + else if (rc == -1) { + errors.push_back(filename + + ":unable to read file -- " + + string(strerror(errno))); + break; + } + if (offset == 0 && (rc > 2) && + (buffer[0] == '#') && (buffer[1] == '!')) { + // Turn it into a JavaScript comment. + buffer[0] = buffer[1] = '/'; + } + if (yajl_parse(handle, (const unsigned char *)buffer, rc) != yajl_status_ok) { + errors.push_back(filename + + ": invalid json -- " + + string((char *)yajl_get_error(handle, 1, (unsigned char *)buffer, rc))); + break; + } + offset += rc; + } + if (rc == 0) { + if (yajl_complete_parse(handle) != yajl_status_ok) { + errors.push_back(filename + + ": invalid json -- " + + string((char *)yajl_get_error(handle, 0, NULL, 0))); + } + } + yajl_free(handle); + } + + return retval; +} + static void load_from_path(const string &path, std::vector &errors) { string format_path = path + "/formats/*/*.json"; static_root_mem gl; - yajl_handle handle; log_info("loading formats from path: %s", format_path.c_str()); if (glob(format_path.c_str(), 0, NULL, gl.inout()) == 0) { for (int lpc = 0; lpc < (int)gl->gl_pathc; lpc++) { string filename(gl->gl_pathv[lpc]); - auto_fd fd; + vector format_list; - log_info("loading formats from file: %s", filename.c_str()); - yajlpp_parse_context ypc(filename, format_handlers); - if ((fd = open(gl->gl_pathv[lpc], O_RDONLY)) == -1) { - char errmsg[1024]; - - snprintf(errmsg, sizeof(errmsg), - "error: unable to open format file -- %s", - gl->gl_pathv[lpc]); - perror(errmsg); - } - else { - char buffer[2048]; - int rc = -1; - - handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc); - while (true) { - rc = read(fd, buffer, sizeof(buffer)); - if (rc == 0) { - break; - } - else if (rc == -1) { - errors.push_back(filename + - ":unable to read file -- " + - string(strerror(errno))); - break; - } - if (yajl_parse(handle, (const unsigned char *)buffer, rc) != yajl_status_ok) { - errors.push_back(filename + - ": invalid json -- " + - string((char *)yajl_get_error(handle, 1, (unsigned char *)buffer, rc))); - break; - } - } - if (rc == 0) { - if (yajl_complete_parse(handle) != yajl_status_ok) { - errors.push_back(filename + - ": invalid json -- " + - string((char *)yajl_get_error(handle, 0, NULL, 0))); - } - } - yajl_free(handle); + format_list = load_format_file(filename, errors); + if (format_list.empty()) { + log_warning("Empty format file: %s", filename.c_str()); } } } @@ -484,6 +511,8 @@ void load_formats(const std::vector &extra_paths, write_sample_file(); handle = yajl_alloc(&ypc_builtin.ypc_callbacks, NULL, &ypc_builtin); + ypc_builtin.ypc_userdata = &retval; + yajl_config(handle, yajl_allow_comments, 1); if (yajl_parse(handle, (const unsigned char *)default_log_formats_json, strlen(default_log_formats_json)) != yajl_status_ok) { diff --git a/src/log_format_loader.hh b/src/log_format_loader.hh index 3e3550b1..ae1e7d47 100644 --- a/src/log_format_loader.hh +++ b/src/log_format_loader.hh @@ -35,6 +35,9 @@ #include #include +std::vector load_format_file( + const std::string &filename, std::vector &errors); + void load_formats(const std::vector &extra_paths, std::vector &errors);