[sql] add a timeslice() function

NEWS

@@ -34,6 +34,8 @@ lnav v0.8.0:
      * Added a "log_msg_instance" column to the logline and log_search
        tables to make it easier to join tables that are matching log
        messages that are ordered.
+     * Added a "timeslice()" function to SQLite so that it is easier to
+       group log messages by time buckets.
      * The 'goto' command now supports relative time values like
        'a minute ago', 'an hour later', and many more.
 

docs/source/sqlext.rst

@@ -139,6 +139,17 @@ Network information functions:
 * gethostbyaddr - Convert an IPv4/IPv6 address into a host name.  If the
   reverse lookup fails, the input value will be returned.
 
+Time
+----
+
+Time functions:
+
+* timeslice(t, s) - Given a time stamp (t) and a time slice (s), return a
+  timestamp for the bucket of time that the timestamp falls in.  For example,
+  with the timestamp "2015-03-01 11:02:00' and slice '5min' the returned value
+  will be '2015-03-01 11:00:00'.  This function can be useful when trying to
+  group together log messages into buckets.
+
 Internal State
 --------------
 

src/Makefile.am

@@ -251,6 +251,7 @@ libdiag_a_SOURCES = \
 	strnatcmp.c \
 	sysclip.cc \
 	textview_curses.cc \
+	time-extension-functions.cc \
 	time_fmts.cc \
 	view_curses.cc \
 	views_vtab.cc \

src/readline_highlighters.cc

@@ -398,7 +398,7 @@ void readline_sqlite_highlighter(attr_line_t &al, int x)
     static int error_attrs = (
         A_BOLD|A_REVERSE|view_colors::ansi_color_pair(COLOR_RED, COLOR_BLACK));
 
-    static string keyword_re_str = sql_keyword_re() + "|\\.schema";
+    static string keyword_re_str = sql_keyword_re() + "|\\.schema|\\.msgformats";
     static pcrepp keyword_pcre(keyword_re_str.c_str(), PCRE_CASELESS);
     static pcrepp string_literal_pcre("'[^']*('(?:'[^']*')*|$)");
     static pcrepp ident_pcre("(\\$?\\b[a-z_]\\w*)|\"([^\"]+)\"|\\[([^\\]]+)]", PCRE_CASELESS);

src/relative_time.hh

@@ -172,6 +172,26 @@ public:
         }
     };
 
+    int64_t to_microseconds() {
+        int64_t retval;
+
+        retval = this->rt_field[RTF_YEARS] * 12;
+        retval = (retval + this->rt_field[RTF_MONTHS]) * 30;
+        retval = (retval + this->rt_field[RTF_DAYS]) * 24;
+        retval = (retval + this->rt_field[RTF_HOURS]) * 60;
+        retval = (retval + this->rt_field[RTF_MINUTES]) * 60;
+        retval = (retval + this->rt_field[RTF_SECONDS]) * 1000 * 1000;
+
+        return retval;
+    };
+
+    void to_timeval(struct timeval &tv_out) {
+        int64_t us = this->to_microseconds();
+
+        tv_out.tv_sec = us / (1000 * 1000);
+        tv_out.tv_usec = us % (1000 * 1000);
+    };
+
     std::string to_string() {
         char dst[128];
 

src/sqlite-extension-func.c

@@ -43,6 +43,7 @@ sqlite_registration_func_t sqlite_registration_funcs[] = {
     network_extension_functions,
     fs_extension_functions,
     json_extension_functions,
+    time_extension_functions,
 
     NULL
 };

src/sqlite-extension-func.h

@@ -78,6 +78,9 @@ int fs_extension_functions(const struct FuncDef **basic_funcs,
 int json_extension_functions(const struct FuncDef **basic_funcs,
                              const struct FuncDefAgg **agg_funcs);
 
+int time_extension_functions(const struct FuncDef **basic_funcs,
+                             const struct FuncDefAgg **agg_funcs);
+
 extern sqlite_registration_func_t sqlite_registration_funcs[];
 
 int register_sqlite_funcs(sqlite3 *db, sqlite_registration_func_t *reg_funcs);

src/time-extension-functions.cc

@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) 2015, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @file time-extension-functions.cc
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <stdint.h>
+
+#include <string>
+#include <stddef.h>
+
+#include "lnav_util.hh"
+#include "sql_util.hh"
+#include "relative_time.hh"
+
+#include "sqlite3.h"
+
+#include "sqlite-extension-func.h"
+
+static void timeslice(sqlite3_context *context,
+                      int argc, sqlite3_value **argv)
+{
+    relative_time::parse_error pe;
+    const char *slice_in, *time_in;
+    date_time_scanner dts;
+    relative_time rt;
+    time_t now;
+
+    time(&now);
+    dts.set_base_time(now);
+
+    if (sqlite3_value_type(argv[0]) == SQLITE_NULL ||
+            sqlite3_value_type(argv[1]) == SQLITE_NULL) {
+        sqlite3_result_null(context);
+        return;
+    }
+
+    time_in = (const char *)sqlite3_value_text(argv[0]);
+    slice_in = (const char *)sqlite3_value_text(argv[1]);
+
+    if (!rt.parse(slice_in, strlen(slice_in), pe)) {
+        sqlite3_result_error(context, "unable to parse time slice value", -1);
+        return;
+    }
+
+    if (rt.empty()) {
+        sqlite3_result_error(context, "no time slice value given", -1);
+        return;
+    }
+
+    if (rt.is_absolute()) {
+        sqlite3_result_error(context, "absolute time slices are not valid", -1);
+        return;
+    }
+
+    struct exttm tm;
+    struct timeval tv;
+
+    if (dts.scan(time_in, strlen(time_in), NULL, &tm, tv) == NULL) {
+        sqlite3_result_error(context, "unable to parse time value", -1);
+        return;
+    }
+
+    int64_t us = tv.tv_sec * 1000 * 1000 + tv.tv_usec, remainder;
+
+    remainder = us % rt.to_microseconds();
+    us -= remainder;
+
+    tv.tv_sec = us / (1000 * 1000);
+    tv.tv_usec = us % (1000 * 1000);
+
+    char ts[64];
+    ssize_t tslen = sql_strftime(ts, sizeof(ts), tv);
+    sqlite3_result_text(context, ts, tslen, SQLITE_TRANSIENT);
+}
+
+int time_extension_functions(const struct FuncDef **basic_funcs,
+                             const struct FuncDefAgg **agg_funcs)
+{
+    static const struct FuncDef time_funcs[] = {
+        { "timeslice",  2, 0, SQLITE_UTF8, 0, timeslice },
+
+        /*
+         * TODO: add other functions like readlink, normpath, ... 
+         */
+
+        { NULL }
+    };
+
+    *basic_funcs = time_funcs;
+    *agg_funcs   = NULL;
+
+    return SQLITE_OK;
+}

test/Makefile.am

@@ -192,6 +192,7 @@ dist_noinst_SCRIPTS = \
 	test_sql_coll_func.sh \
 	test_sql_json_func.sh \
 	test_sql_str_func.sh \
+	test_sql_time_func.sh \
 	test_sql_fs_func.sh \
 	test_view_colors.sh \
 	test_vt52_curses.sh \
@@ -305,6 +306,7 @@ TESTS = \
 	test_sql_json_func.sh \
 	test_sql_fs_func.sh \
 	test_sql_str_func.sh \
+	test_sql_time_func.sh \
 	test_view_colors.sh \
 	test_vt52_curses.sh \
 	test_top_status \

test/test_sql_time_func.sh

@@ -0,0 +1,54 @@
+#! /bin/bash
+
+run_test ./drive_sql "select timeslice()"
+
+check_error_output "timeslice()" <<EOF
+error: sqlite3_exec failed -- wrong number of arguments to function timeslice()
+EOF
+
+run_test ./drive_sql "select timeslice(1)"
+
+check_error_output "timeslice(1)" <<EOF
+error: sqlite3_exec failed -- wrong number of arguments to function timeslice()
+EOF
+
+run_test ./drive_sql "select timeslice('', '')"
+
+check_error_output "timeslice empty" <<EOF
+error: sqlite3_exec failed -- no time slice value given
+EOF
+
+run_test ./drive_sql "select timeslice('2015-08-07 12:01:00', '8 am')"
+
+check_error_output "timeslice abs" <<EOF
+error: sqlite3_exec failed -- absolute time slices are not valid
+EOF
+
+run_test ./drive_sql "select timeslice(null, null)"
+
+check_output "timeslice(null, null)" <<EOF
+Row 0:
+  Column timeslice(null, null): (null)
+EOF
+
+run_test ./drive_sql "select timeslice('2015-08-07 12:01:00', '5m')"
+
+check_output "timeslice 5m" <<EOF
+Row 0:
+  Column timeslice('2015-08-07 12:01:00', '5m'): 2015-08-07 12:00:00.000
+EOF
+
+run_test ./drive_sql "select timeslice('2015-08-07 12:01:00', '1d')"
+
+check_output "timeslice 1d" <<EOF
+Row 0:
+  Column timeslice('2015-08-07 12:01:00', '1d'): 2015-08-07 00:00:00.000
+EOF
+
+run_test ./drive_sql "select timeslice('2015-08-07 12:01:00', '1 month')"
+
+# XXX This is wrong...
+check_output "timeslice 1 month" <<EOF
+Row 0:
+  Column timeslice('2015-08-07 12:01:00', '1 month'): 2015-08-03 00:00:00.000
+EOF