[sql] add an lnav_views table

NEWS

@@ -16,6 +16,8 @@ lnav v0.8.0:
        make sure the format(s) match all of the lines.
      * Added an "all_logs" SQLite table that contains the message format
        extracted from each log line.
+     * Added an "lnav_views" SQLite table that can be used to query and
+       change the lnav view state.
 
      Fixes:
      * Nested fields in JSON logs are now supported for levels, bodies, etc...

docs/source/data.rst

@@ -1,4 +1,6 @@
 
+.. _data-ext:
+
 Extracting Data
 ===============
 

docs/source/index.rst

@@ -22,6 +22,8 @@ Contents:
    hotkeys
    commands
    sqlext
+   sqltab
+
 
 Indices and tables
 ==================

docs/source/sqltab.rst

@@ -0,0 +1,69 @@
+
+.. _sql-tab:
+
+SQLite Tables Reference
+=======================
+
+In addition to the tables generated for each log format, **lnav** includes
+the following tables:
+
+* environ
+* lnav_views
+* all_logs
+* http_status_codes
+
+These extra tables provide useful information and can let you manipulate
+**lnav**'s internal state.  You can get a dump of the entire database schema
+by executing the '.schema' SQL command, like so::
+
+    ;.schema
+
+environ
+-------
+
+The **environ** table gives you access to the **lnav** process' environment
+variables.  You can SELECT, INSERT, and UPDATE environment variables, like
+so::
+
+    ;SELECT * FROM environ WHERE name = 'SHELL'
+     name   value
+    SHELL /bin/tcsh
+
+    ;UPDATE environ SET value = '/bin/sh' WHERE name = 'SHELL'
+
+Environment variables can be used to store simple values or pass values
+from **lnav**'s SQL environment to **lnav**'s commands.  For example, the
+"open" command will do variable substitution, so you can insert a variable
+named "FILENAME" and then open it in **lnav** by referencing it with
+"$FILENAME"::
+
+    ;INSERT INTO environ VALUES ('FILENAME', '/path/to/file')
+    :open $FILENAME
+
+lnav_views
+----------
+
+The **lnav_views** table allows you to SELECT and UPDATE information related
+to **lnav**'s "views" (e.g. log, text, ...).  The following columns are
+available in this table:
+
+  :name: The name of the view.
+  :top: The line number at the top of the view.  This value can be UPDATEd to
+    move the view to the given line.
+  :left: The left-most column number to display.  This value can be UPDATEd to
+    move the view left or right.
+  :height: The number of lines that are displayed on the screen.
+  :inner_height: The number of lines of content being displayed.
+
+all_logs
+--------
+
+The **all_logs** table lets you query the format derived from the **lnav**
+log message parser that is used to automatically extract data, see
+:ref:`data-ext` for more details.
+
+http_status_codes
+-----------------
+
+The **http_status_codes** table is a handy reference that can be used to turn
+HTTP status codes into human-readable messages.

src/CMakeLists.txt

@@ -50,6 +50,7 @@ set(diag_STAT_SRCS
     strnatcmp.c
     textview_curses.cc
     view_curses.cc
+    views_vtab.cc
     vt52_curses.cc
     log_vtab_impl.cc
     xterm_mouse.cc
@@ -108,6 +109,7 @@ set(diag_STAT_SRCS
     textfile_sub_source.hh
     time_T.hh
     top_status_source.hh
+    views_vtab.hh
 
     yajl/api/yajl_common.h
     yajl/api/yajl_gen.h

src/Makefile.am

@@ -167,6 +167,7 @@ noinst_HEADERS = \
 	timer.hh \
 	top_status_source.hh \
 	view_curses.hh \
+	views_vtab.hh \
 	vt52_curses.hh \
 	log_vtab_impl.hh \
 	log_format_impls.cc \
@@ -245,6 +246,7 @@ libdiag_a_SOURCES = \
 	textview_curses.cc \
 	time_fmts.cc \
 	view_curses.cc \
+	views_vtab.cc \
 	vt52_curses.cc \
 	log_vtab_impl.cc \
 	xterm_mouse.cc \

src/lnav.cc

@@ -115,6 +115,7 @@
 #include "log_data_helper.hh"
 #include "readline_highlighters.hh"
 #include "environ_vtab.hh"
+#include "views_vtab.hh"
 #include "pretty_printer.hh"
 #include "all_logs_vtab.hh"
 
@@ -730,6 +731,7 @@ static void open_schema_view(void)
 
     schema += "\n\n-- Virtual Table Definitions --\n\n";
     schema += ENVIRON_CREATE_STMT;
+    schema += LNAV_VIEWS_CREATE_STMT;
     for (log_vtab_manager::iterator vtab_iter =
             lnav_data.ld_vtab_manager->begin();
          vtab_iter != lnav_data.ld_vtab_manager->end();
@@ -826,7 +828,9 @@ void redo_search(lnav_view_t view_index)
         gp->queue_request(grep_line_t(0));
         gp->start();
     }
-    lnav_data.ld_scroll_broadcaster.invoke(tc);
+    if (tc == lnav_data.ld_view_stack.top()) {
+        lnav_data.ld_scroll_broadcaster.invoke(tc);
+    }
 }
 
 /**
@@ -2394,6 +2398,7 @@ int main(int argc, char *argv[])
     }
 
     register_environ_vtab(lnav_data.ld_db.in());
+    register_views_vtab(lnav_data.ld_db.in());
 
     lnav_data.ld_vtab_manager =
         new log_vtab_manager(lnav_data.ld_db,

src/top_status_source.hh

@@ -112,7 +112,7 @@ public:
             listview_value_for_row(*lc, lc->get_top(), al);
             string_attrs_t &sa = al.get_attrs();
             line_attr = find_string_attr(sa, &logline::L_FILE);
-            if (line_attr != sa.end()) {
+            if (line_attr != sa.cend()) {
                 logfile *lf = (logfile *)line_attr->sa_value.sav_ptr;
 
                 if (lf->get_format()) {

src/views_vtab.cc

@@ -0,0 +1,296 @@
+/**
+ * Copyright (c) 2015, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "lnav.hh"
+#include "auto_mem.hh"
+#include "lnav_log.hh"
+#include "sql_util.hh"
+#include "views_vtab.hh"
+#include "view_curses.hh"
+
+using namespace std;
+
+const char *LNAV_VIEWS_CREATE_STMT = "\
+-- Access lnav's views through this table.\n\
+CREATE TABLE lnav_views (\n\
+    name text PRIMARY KEY,\n\
+    top integer,\n\
+    left integer,\n\
+    height integer,\n\
+    inner_height integer\
+);\n\
+";
+
+struct vtab {
+    sqlite3_vtab        base;
+    sqlite3 *           db;
+};
+
+struct vtab_cursor {
+    sqlite3_vtab_cursor        base;
+    lnav_view_t vc_cursor;
+};
+
+static int vt_destructor(sqlite3_vtab *p_svt);
+
+static int vt_create(sqlite3 *db,
+                     void *pAux,
+                     int argc, const char *const *argv,
+                     sqlite3_vtab **pp_vt,
+                     char **pzErr)
+{
+    vtab *p_vt;
+
+    /* Allocate the sqlite3_vtab/vtab structure itself */
+    p_vt = (vtab *)sqlite3_malloc(sizeof(*p_vt));
+
+    if (p_vt == NULL) {
+        return SQLITE_NOMEM;
+    }
+
+    memset(&p_vt->base, 0, sizeof(sqlite3_vtab));
+    p_vt->db = db;
+
+    *pp_vt = &p_vt->base;
+
+    int rc = sqlite3_declare_vtab(db, LNAV_VIEWS_CREATE_STMT);
+
+    return rc;
+}
+
+
+static int vt_destructor(sqlite3_vtab *p_svt)
+{
+    vtab *p_vt = (vtab *)p_svt;
+
+    /* Free the SQLite structure */
+    sqlite3_free(p_vt);
+
+    return SQLITE_OK;
+}
+
+static int vt_connect(sqlite3 *db, void *p_aux,
+                      int argc, const char *const *argv,
+                      sqlite3_vtab **pp_vt, char **pzErr)
+{
+    return vt_create(db, p_aux, argc, argv, pp_vt, pzErr);
+}
+
+static int vt_disconnect(sqlite3_vtab *pVtab)
+{
+    return vt_destructor(pVtab);
+}
+
+static int vt_destroy(sqlite3_vtab *p_vt)
+{
+    return vt_destructor(p_vt);
+}
+
+static int vt_next(sqlite3_vtab_cursor *cur);
+
+static int vt_open(sqlite3_vtab *p_svt, sqlite3_vtab_cursor **pp_cursor)
+{
+    vtab *p_vt = (vtab *)p_svt;
+
+    p_vt->base.zErrMsg = NULL;
+
+    vtab_cursor *p_cur = (vtab_cursor *)new vtab_cursor();
+
+    if (p_cur == NULL) {
+        return SQLITE_NOMEM;
+    } else {
+        *pp_cursor = (sqlite3_vtab_cursor *)p_cur;
+
+        p_cur->base.pVtab = p_svt;
+        p_cur->vc_cursor = (lnav_view_t) -1;
+
+        vt_next((sqlite3_vtab_cursor *)p_cur);
+    }
+
+    return SQLITE_OK;
+}
+
+static int vt_close(sqlite3_vtab_cursor *cur)
+{
+    vtab_cursor *p_cur = (vtab_cursor *)cur;
+
+    /* Free cursor struct. */
+    delete p_cur;
+
+    return SQLITE_OK;
+}
+
+static int vt_eof(sqlite3_vtab_cursor *cur)
+{
+    vtab_cursor *vc = (vtab_cursor *)cur;
+
+    return vc->vc_cursor == LNV__MAX;
+}
+
+static int vt_next(sqlite3_vtab_cursor *cur)
+{
+    vtab_cursor *vc   = (vtab_cursor *)cur;
+
+    if (vc->vc_cursor < LNV__MAX) {
+        vc->vc_cursor = (lnav_view_t) (vc->vc_cursor + 1);
+    }
+
+    return SQLITE_OK;
+}
+
+static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
+{
+    vtab_cursor *vc = (vtab_cursor *)cur;
+    textview_curses &tc = lnav_data.ld_views[vc->vc_cursor];
+    unsigned long width;
+    vis_line_t height;
+
+    tc.get_dimensions(height, width);
+    switch (col) {
+        case 0:
+            sqlite3_result_text(ctx,
+                                lnav_view_strings[vc->vc_cursor], -1,
+                                SQLITE_STATIC);
+            break;
+        case 1:
+            sqlite3_result_int(ctx, (int) tc.get_top());
+            break;
+        case 2:
+            sqlite3_result_int(ctx, tc.get_left());
+            break;
+        case 3:
+            sqlite3_result_int(ctx, height);
+            break;
+        case 4:
+            sqlite3_result_int(ctx, tc.get_inner_height());
+            break;
+    }
+
+    return SQLITE_OK;
+}
+
+static int vt_rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *p_rowid)
+{
+    vtab_cursor *p_cur = (vtab_cursor *)cur;
+
+    *p_rowid = p_cur->vc_cursor;
+
+    return SQLITE_OK;
+}
+
+static int vt_best_index(sqlite3_vtab *tab, sqlite3_index_info *p_info)
+{
+    return SQLITE_OK;
+}
+
+static int vt_filter(sqlite3_vtab_cursor *p_vtc,
+                     int idxNum, const char *idxStr,
+                     int argc, sqlite3_value **argv)
+{
+    return SQLITE_OK;
+}
+
+static int vt_update(sqlite3_vtab *tab,
+                     int argc,
+                     sqlite3_value **argv,
+                     sqlite_int64 *rowid)
+{
+    if (argc <= 1) {
+        tab->zErrMsg = sqlite3_mprintf(
+                "Rows cannot be deleted from the lnav_views table");
+        return SQLITE_ERROR;
+    }
+
+    if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+        tab->zErrMsg = sqlite3_mprintf(
+                "Rows cannot be inserted into the lnav_views table");
+        return SQLITE_ERROR;
+    }
+
+    int64_t index = sqlite3_value_int64(argv[0]);
+
+    if (index != sqlite3_value_int64(argv[1])) {
+        tab->zErrMsg = sqlite3_mprintf(
+                "The rowids in the lnav_views table cannot be changed");
+        return SQLITE_ERROR;
+    }
+
+    textview_curses &tc = lnav_data.ld_views[index];
+    int64_t top = sqlite3_value_int64(argv[3]);
+    int64_t left = sqlite3_value_int64(argv[4]);
+
+    tc.set_top(vis_line_t(top));
+    tc.set_left(left);
+
+    return SQLITE_OK;
+}
+
+static sqlite3_module vtab_module = {
+    0,              /* iVersion */
+    vt_create,      /* xCreate       - create a vtable */
+    vt_connect,     /* xConnect      - associate a vtable with a connection */
+    vt_best_index,  /* xBestIndex    - best index */
+    vt_disconnect,  /* xDisconnect   - disassociate a vtable with a connection */
+    vt_destroy,     /* xDestroy      - destroy a vtable */
+    vt_open,        /* xOpen         - open a cursor */
+    vt_close,       /* xClose        - close a cursor */
+    vt_filter,      /* xFilter       - configure scan constraints */
+    vt_next,        /* xNext         - advance a cursor */
+    vt_eof,         /* xEof          - inidicate end of result set*/
+    vt_column,      /* xColumn       - read data */
+    vt_rowid,       /* xRowid        - read data */
+    vt_update,      /* xUpdate       - write data */
+    NULL,           /* xBegin        - begin transaction */
+    NULL,           /* xSync         - sync transaction */
+    NULL,           /* xCommit       - commit transaction */
+    NULL,           /* xRollback     - rollback transaction */
+    NULL,           /* xFindFunction - function overloading */
+};
+
+int register_views_vtab(sqlite3 *db)
+{
+    auto_mem<char, sqlite3_free> errmsg;
+    int rc;
+
+    rc = sqlite3_create_module(db, "views_vtab_impl", &vtab_module, NULL);
+    assert(rc == SQLITE_OK);
+    if ((rc = sqlite3_exec(db,
+             "CREATE VIRTUAL TABLE lnav_views USING views_vtab_impl()",
+             NULL, NULL, errmsg.out())) != SQLITE_OK) {
+        fprintf(stderr, "wtf %s\n", errmsg.in());
+    }
+    return rc;
+}

src/views_vtab.hh

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2015, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __views_vtab_hh
+#define __views_vtab_hh
+
+#include <sqlite3.h>
+
+int register_views_vtab(sqlite3 *db);
+
+extern const char *LNAV_VIEWS_CREATE_STMT;
+
+#endif

test/test_sql.sh

@@ -240,8 +240,51 @@ check_output "delete environ table does not work" <<EOF
 EOF
 
 
+run_test ${lnav_test} -n \
+    -c ';DELETE FROM lnav_views' \
+    -c ';SELECT count(*) FROM lnav_views' \
+    -c ':write-csv-to -' \
+    ${test_dir}/logfile_access_log.0
+
+check_output "delete from lnav_views table works?" <<EOF
+count(*)
+9
+EOF
+
+
+run_test ${lnav_test} -n \
+    -c ";INSERT INTO lnav_views (name) VALUES ('foo')" \
+    -c ';SELECT count(*) FROM lnav_views' \
+    -c ':write-csv-to -' \
+    ${test_dir}/logfile_access_log.0
+
+check_output "insert into lnav_views table works?" <<EOF
+count(*)
+9
+EOF
+
+
+run_test ${lnav_test} -n \
+    -c ";UPDATE lnav_views SET top = 1 WHERE name = 'log'" \
+    ${test_dir}/logfile_access_log.0
+
+check_output "updating lnav_views.top does not work?" <<EOF
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkboot.gz HTTP/1.0" 404 46210 "-" "gPXE/0.9.7"
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
+EOF
+
+
+run_test ${lnav_test} -n \
+    -c ";UPDATE lnav_views SET top = inner_height - 1 WHERE name = 'log'" \
+    ${test_dir}/logfile_access_log.0
+
+check_output "updating lnav_views.top using inner_height does not work?" <<EOF
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
+EOF
+
+
 schema_dump() {
-    ${lnav_test} -n -c ';.schema' ${test_dir}/logfile_access_log.0 | head -n8
+    ${lnav_test} -n -c ';.schema' ${test_dir}/logfile_access_log.0 | head -n9
 }
 
 run_test schema_dump
@@ -249,6 +292,7 @@ run_test schema_dump
 check_output "schema view is not working" <<EOF
 ATTACH DATABASE '' AS 'main';
 CREATE VIRTUAL TABLE environ USING environ_vtab_impl();
+CREATE VIRTUAL TABLE lnav_views USING views_vtab_impl();
 CREATE TABLE http_status_codes (
     status integer PRIMARY KEY,
     message text,