[json-ext] json_contains() should accept null

Related to #447

NEWS

@@ -7,6 +7,8 @@ lnav v0.10.1:
        results of a SQL query, you would pass "--view=db" to the command.
      * The commands used to access the clipboard are now configured through
        the "tuning" section of the configuration.
+     * Added an "lnav_version()" SQL function that returns the current
+       version string.
      Interface changes:
      * The xclip implementation for accessing the system clipboard now writes
        to the "clipboard" selection instead of the "primary" selection.
@@ -31,6 +33,8 @@ lnav v0.10.1:
        is an issue opening a file.
      * Overwritten files should be reloaded again.
      * The "jget()" SQL function now returns numbers with the correct type.
+     * The "json_contains()" SQL function now returns false if the first
+       argument is NULL instead of NULL.
      * The local copies of remote files are now cleaned up after a couple
        days of the host not being accessed.
      * The initial loading and indexing phase has been optimized.

src/CMakeLists.txt

@@ -9,7 +9,8 @@ check_type_size(off_t SIZEOF_OFF_T)
 check_include_file("pty.h" HAVE_PTY_H)
 check_include_file("util.h" HAVE_UTIL_H)
 
-set(VCS_PACKAGE_STRING "test")
+set(VCS_PACKAGE_STRING "lnav ${CMAKE_PROJECT_VERSION}")
+set(PACKAGE_VERSION "${CMAKE_PROJECT_VERSION}")
 
 configure_file(config.cmake.h.in config.h)
 

src/base/CMakeLists.txt

@@ -62,5 +62,10 @@ add_executable(test_base
         network.tcp.tests.cc
 
         test_base.cc)
+target_include_directories(
+        test_base
+        PUBLIC
+        ../third-party/doctest-root
+)
 target_link_libraries(test_base base pcrepp ZLIB::zlib)
 add_test(NAME test_base COMMAND test_base)

src/base/network.tcp.tests.cc

@@ -31,7 +31,7 @@
 
 #include <iostream>
 
-#include "doctest.hh"
+#include "doctest/doctest.h"
 
 #include "network.tcp.hh"
 

src/command_executor.cc

@@ -332,6 +332,10 @@ Result<string, string> execute_sql(exec_context &ec, const string &sql, string &
 
     gettimeofday(&end_tv, nullptr);
     if (retcode == SQLITE_DONE) {
+        if (lnav_data.ld_log_source.is_line_meta_changed()) {
+            lnav_data.ld_log_source.text_filters_changed();
+            lnav_data.ld_views[LNV_LOG].reload_data();
+        }
         lnav_data.ld_filter_view.reload_data();
         lnav_data.ld_files_view.reload_data();
         lnav_data.ld_views[LNV_DB].reload_data();

src/config.cmake.h.in

@@ -8,6 +8,8 @@
 
 #cmakedefine VCS_PACKAGE_STRING "@VCS_PACKAGE_STRING@"
 
+#cmakedefine PACKAGE_VERSION "@PACKAGE_VERSION@"
+
 #cmakedefine HAVE_PTY_H
 
 #cmakedefine HAVE_UTIL_H

src/internals/sql-ref.rst

@@ -1895,6 +1895,17 @@ lnav_top_file()
 ----
 
 
+.. _lnav_version:
+
+lnav_version()
+^^^^^^^^^^^^^^
+
+  Return the current version of lnav
+
+
+----
+
+
 .. _load_extension:
 
 load_extension(*path*, *\[entry-point\]*)

src/json-extension-functions.cc

@@ -74,6 +74,7 @@ static void null_or_default(sqlite3_context *context, int argc, sqlite3_value **
 
 struct contains_userdata {
     util::variant<const char *, sqlite3_int64, bool> cu_match_value{false};
+    size_t cu_depth{0};
     bool cu_result{false};
 };
 
@@ -81,7 +82,8 @@ static int contains_string(void *ctx, const unsigned char *str, size_t len)
 {
     auto &cu = *((contains_userdata *) ctx);
 
-    if (strncmp((const char *) str, cu.cu_match_value.get<const char *>(), len) == 0) {
+    if (cu.cu_depth <= 1 &&
+        strncmp((const char *) str, cu.cu_match_value.get<const char *>(), len) == 0) {
         cu.cu_result = true;
     }
 
@@ -92,15 +94,29 @@ static int contains_integer(void *ctx, long long value)
 {
     auto &cu = *((contains_userdata *) ctx);
 
-    if (cu.cu_match_value.get<sqlite3_int64>() == value) {
+    if (cu.cu_depth <= 1 && cu.cu_match_value.get<sqlite3_int64>() == value) {
         cu.cu_result = true;
     }
 
     return 1;
 }
 
-static bool json_contains(const char *json_in, sqlite3_value *value)
+static int contains_null(void *ctx)
 {
+    auto &cu = *((contains_userdata *) ctx);
+
+    cu.cu_result = true;
+
+    return 1;
+}
+
+static bool json_contains(vtab_types::nullable<const char> nullable_json_in, sqlite3_value *value)
+{
+    if (nullable_json_in.n_value == nullptr || nullable_json_in.n_value[0] == '\0') {
+        return false;
+    }
+
+    auto json_in = nullable_json_in.n_value;
     auto_mem<yajl_handle_t> handle(yajl_free);
     yajl_callbacks cb;
     contains_userdata cu;
@@ -108,6 +124,35 @@ static bool json_contains(const char *json_in, sqlite3_value *value)
     memset(&cb, 0, sizeof(cb));
     handle = yajl_alloc(&cb, nullptr, &cu);
 
+    cb.yajl_start_array = +[](void *ctx) {
+        auto &cu = *((contains_userdata *) ctx);
+
+        cu.cu_depth += 1;
+
+        return 1;
+    };
+    cb.yajl_end_array = +[](void *ctx) {
+        auto &cu = *((contains_userdata *) ctx);
+
+        cu.cu_depth -= 1;
+
+        return 1;
+    };
+    cb.yajl_start_map = +[](void *ctx) {
+        auto &cu = *((contains_userdata *) ctx);
+
+        cu.cu_depth += 2;
+
+        return 1;
+    };
+    cb.yajl_end_map = +[](void *ctx) {
+        auto &cu = *((contains_userdata *) ctx);
+
+        cu.cu_depth -= 2;
+
+        return 1;
+    };
+
     switch (sqlite3_value_type(value)) {
         case SQLITE3_TEXT:
             cb.yajl_string = contains_string;
@@ -117,6 +162,9 @@ static bool json_contains(const char *json_in, sqlite3_value *value)
             cb.yajl_integer = contains_integer;
             cu.cu_match_value = sqlite3_value_int64(value);
             break;
+        case SQLITE_NULL:
+            cb.yajl_null = contains_null;
+            break;
     }
 
     if (yajl_parse(handle.in(), (const unsigned char *) json_in, strlen(json_in)) != yajl_status_ok ||

src/lnav_commands.cc

@@ -2691,6 +2691,9 @@ static Result<string, string> com_comment(exec_context &ec, string cmdline, vect
         bookmark_metadata &line_meta = bm[lss.at(tc->get_top())];
 
         line_meta.bm_comment = args[1];
+        lss.set_line_meta_changed();
+        lss.text_filters_changed();
+        tc->reload_data();
 
         retval = "info: comment added to line";
     } else {
@@ -2747,6 +2750,10 @@ static Result<string, string> com_clear_comment(exec_context &ec, string cmdline
                 tc->set_user_mark(&textview_curses::BM_META, tc->get_top(), false);
             }
 
+            lss.set_line_meta_changed();
+            lss.text_filters_changed();
+            tc->reload_data();
+
             retval = "info: cleared comment";
         }
         tc->search_new_data();
@@ -2787,6 +2794,9 @@ static Result<string, string> com_tag(exec_context &ec, string cmdline, vector<s
             line_meta.add_tag(tag);
         }
         tc->search_new_data();
+        lss.set_line_meta_changed();
+        lss.text_filters_changed();
+        tc->reload_data();
 
         retval = "info: tag(s) added to line";
     } else {
@@ -2833,6 +2843,9 @@ static Result<string, string> com_untag(exec_context &ec, string cmdline, vector
             }
         }
         tc->search_new_data();
+        lss.set_line_meta_changed();
+        lss.text_filters_changed();
+        tc->reload_data();
 
         retval = "info: tag(s) removed from line";
     } else {

src/log_vtab_impl.cc

@@ -940,6 +940,7 @@ static int vt_update(sqlite3_vtab *tab,
         if (binary_search(bv.begin(), bv.end(), vrowid) && !has_meta) {
             vt->tc->set_user_mark(&textview_curses::BM_META, vrowid, false);
             bm.erase(vt->lss->at(vrowid));
+            vt->lss->set_line_meta_changed();
         }
 
         if (has_meta) {
@@ -968,6 +969,8 @@ static int vt_update(sqlite3_vtab *tab,
             } else {
                 line_meta.bm_tags.clear();
             }
+
+            vt->lss->set_line_meta_changed();
         }
 
         vt->tc->set_user_mark(&textview_curses::BM_USER, vrowid, val);

src/logfile_sub_source.cc

@@ -1068,6 +1068,11 @@ log_accel::direction_t logfile_sub_source::get_line_accel_direction(
 
 void logfile_sub_source::text_filters_changed()
 {
+    if (this->lss_line_meta_changed) {
+        this->invalidate_sql_filter();
+        this->lss_line_meta_changed = false;
+    }
+
     for (auto& ld : *this) {
         auto lf = ld->get_file_ptr();
 
@@ -1102,7 +1107,7 @@ void logfile_sub_source::text_filters_changed()
 
         if (!this->tss_apply_filters ||
             (!(*ld)->ld_filter_state.excluded(filtered_in_mask, filtered_out_mask,
-                                           line_number) &&
+                                              line_number) &&
              this->check_extra_filters(ld, line_iter))) {
             auto eval_res = this->eval_sql_filter(this->lss_marker_stmt.in(),
                                                   ld, line_iter);
@@ -1511,6 +1516,13 @@ bool logfile_sub_source::check_extra_filters(iterator ld, logfile::iterator ll)
     return true;
 }
 
+void logfile_sub_source::invalidate_sql_filter()
+{
+    for (auto& ld : *this) {
+        ld->ld_filter_state.lfo_filter_state.clear_filter_state(0);
+    }
+}
+
 void log_location_history::loc_history_append(vis_line_t top)
 {
     if (top >= vis_line_t(this->llh_log_source.text_line_count())) {

src/logfile_sub_source.hh

@@ -815,6 +815,16 @@ public:
     Result<bool, std::string> eval_sql_filter(
         sqlite3_stmt *stmt, iterator ld, logfile::const_iterator ll);
 
+    void invalidate_sql_filter();
+
+    void set_line_meta_changed() {
+        this->lss_line_meta_changed = true;
+    }
+
+    bool is_line_meta_changed() const {
+        return this->lss_line_meta_changed;
+    }
+
     static const uint64_t MAX_CONTENT_LINES = (1ULL << 40) - 1;
     static const uint64_t MAX_LINES_PER_FILE = 256 * 1024 * 1024;
     static const uint64_t MAX_FILES          = (
@@ -1001,6 +1011,7 @@ private:
     log_location_history lss_location_history;
 
     bool lss_in_value_for_line{false};
+    bool lss_line_meta_changed{false};
 };
 
 #endif

src/state-extension-functions.cc

@@ -88,6 +88,11 @@ static nonstd::optional<std::string> sql_lnav_top_file()
     });
 }
 
+static const char *sql_lnav_version()
+{
+    return PACKAGE_VERSION;
+}
+
 static int64_t sql_error(const char *str)
 {
     throw sqlite_func_error("{}", str);
@@ -115,6 +120,12 @@ int state_extension_functions(struct FuncDef **basic_funcs,
                 .sql_function()
         ),
 
+        sqlite_func_adapter<decltype(&sql_lnav_version), sql_lnav_version>::builder(
+            help_text("lnav_version",
+                      "Return the current version of lnav")
+                .sql_function()
+        ),
+
         sqlite_func_adapter<decltype(&sql_error), sql_error>::builder(
             help_text("raise_error",
                       "Raises an error with the given message when executed")

src/vtab_module.hh

@@ -69,6 +69,21 @@ struct sqlite_func_error : std::exception {
     const std::string e_what;
 };
 
+namespace vtab_types {
+
+template<typename T>
+struct nullable {
+    T *n_value{nullptr};
+};
+
+template<typename>
+struct is_nullable : std::false_type {};
+
+template<typename T>
+struct is_nullable<nullable<T>> : std::true_type {};
+
+}
+
 template<typename T>
 struct from_sqlite {
     using U = typename std::remove_reference<T>::type;
@@ -163,6 +178,13 @@ struct from_sqlite<const std::vector<T> &> {
     }
 };
 
+template<typename T>
+struct from_sqlite<vtab_types::nullable<T>> {
+    inline vtab_types::nullable<T> operator()(int argc, sqlite3_value **val, int argi) {
+        return {from_sqlite<T *>()(argc, val, argi)};
+    }
+};
+
 inline void to_sqlite(sqlite3_context *ctx, const char *str)
 {
     if (str == nullptr) {
@@ -314,6 +336,10 @@ struct sqlite_func_adapter<Return (*)(Args...), f> {
     constexpr static size_t OPT_COUNT = optional_counter<Args...>::value;
     constexpr static size_t VAR_COUNT = variadic_counter<Args...>::value;
     constexpr static size_t REQ_COUNT = sizeof...(Args) - OPT_COUNT - VAR_COUNT;
+    constexpr static bool IS_NULLABLE[] = {vtab_types::is_nullable<Args>::value ... };
+    constexpr static bool IS_SQLITE3_VALUE[] = {
+        std::is_same<Args, sqlite3_value *>::value ...
+    };
 
     template<size_t ... Idx>
     static void func2(sqlite3_context *context,
@@ -362,7 +388,9 @@ struct sqlite_func_adapter<Return (*)(Args...), f> {
         }
 
         for (size_t lpc = 0; lpc < REQ_COUNT; lpc++) {
-            if (sqlite3_value_type(argv[lpc]) == SQLITE_NULL) {
+            if (!IS_NULLABLE[lpc] &&
+                !IS_SQLITE3_VALUE[lpc] &&
+                sqlite3_value_type(argv[lpc]) == SQLITE_NULL) {
                 sqlite3_result_null(context);
                 return;
             }

test/CMakeLists.txt

@@ -53,6 +53,7 @@ target_link_libraries(test_log_accel diag PkgConfig::libpcre)
 add_test(NAME test_log_accel COMMAND test_log_accel)
 
 add_executable(lnav_doctests lnav_doctests.cc)
+target_include_directories(lnav_doctests PUBLIC ../src/third-party/doctest-root)
 target_link_libraries(lnav_doctests diag ${lnav_LIBS})
 add_test(NAME lnav_doctests COMMAND lnav_doctests)
 
@@ -61,6 +62,7 @@ target_link_libraries(test_pcrepp diag PkgConfig::libpcre)
 add_test(NAME test_pcrepp COMMAND test_pcrepp)
 
 add_executable(test_reltime test_reltime.cc)
+target_include_directories(test_reltime PUBLIC ../src/third-party/doctest-root)
 target_link_libraries(test_reltime diag PkgConfig::libpcre)
 add_test(NAME test_reltime COMMAND test_reltime)
 

test/expected_help.txt

@@ -1919,7 +1919,11 @@ Synopsis
 
 
 Synopsis
-  load_extension(path, [entry-point]) -- Loads SQLite extensions out of the 
+  lnav_version() -- Return the current version of lnav
+
+
+Synopsis
+  load_extension(path, [entry-point]) -- Loads SQLite extensions out of the
     given shared library file using the given entry point.
 Parameters
   path          The path to the shared library containing the extension.

test/test_cmds.sh

@@ -121,6 +121,16 @@ check_output "filter-expr for multiline not working" <<EOF
   How are you today?
 EOF
 
+run_test ${lnav_test} -n -d /tmp/lnav.err \
+    -c ":filter-expr not json_contains(:log_tags, '#bad')" \
+    -c ":goto 0" \
+    -c ":tag #bad" \
+    "${test_dir}/logfile_access_log.0"
+
+check_output "filter-expr for multiline not working" <<EOF
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkboot.gz HTTP/1.0" 404 46210 "-" "gPXE/0.9.7"
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
+EOF
 
 run_test ${lnav_test} -n -d /tmp/lnav.err \
     -c ":filter-expr :sc_bytes > 2000" \

test/test_sql_json_func.sh

@@ -28,6 +28,34 @@ Row 0:
   Column json_concat(NULL, json('{"abc": 1}')): [{"abc":1}]
 EOF
 
+run_test ./drive_sql "select json_contains(NULL, 4)"
+
+check_output "json_contains does not work" <<EOF
+Row 0:
+  Column json_contains(NULL, 4): 0
+EOF
+
+run_test ./drive_sql "select json_contains('', 4)"
+
+check_output "json_contains does not work" <<EOF
+Row 0:
+  Column json_contains('', 4): 0
+EOF
+
+run_test ./drive_sql "select json_contains('null', NULL)"
+
+check_output "json_contains does not work" <<EOF
+Row 0:
+  Column json_contains('null', NULL): 1
+EOF
+
+run_test ./drive_sql "select json_contains('[[0]]', 0)"
+
+check_output "json_contains does not work" <<EOF
+Row 0:
+  Column json_contains('[[0]]', 0): 0
+EOF
+
 run_test ./drive_sql "select json_contains('4', 4)"
 
 check_output "json_contains does not work" <<EOF