mirror of
https://github.com/tstack/lnav.git
synced 2024-08-16 08:20:29 +03:00
[perf] improve initial indexing times
This commit is contained in:
parent
2e10ca09d0
commit
2589345e5c
19
configure.ac
19
configure.ac
@ -56,23 +56,6 @@ AS_VAR_IF([enable_profiling], [yes],
|
||||
|
||||
AC_ARG_VAR(SFTP_TEST_URL)
|
||||
|
||||
AC_ARG_ENABLE([profiling],
|
||||
AS_HELP_STRING([--enable-profiling],
|
||||
[Compile with gprof(1) profiling support]))
|
||||
|
||||
AC_MSG_CHECKING(gprof(4) profiling support)
|
||||
|
||||
AS_VAR_IF([enable_profiling], [yes],
|
||||
[CFLAGS="$CFLAGS -pg -gstabs"
|
||||
CXXFLAGS="$CXXFLAGS -pg -gstabs"
|
||||
LDFLAGS="$LDFLAGS -pg"],
|
||||
[enable_profiling=no]dnl
|
||||
)
|
||||
|
||||
AC_MSG_RESULT($enable_profiling)
|
||||
|
||||
AC_SUBST(CFLAGS_PG)
|
||||
|
||||
AC_PROG_INSTALL
|
||||
AC_PROG_RANLIB
|
||||
AM_PROG_AR
|
||||
@ -134,7 +117,7 @@ AS_CASE(["$host_os"],
|
||||
)
|
||||
)
|
||||
|
||||
AC_CHECK_HEADERS(execinfo.h pty.h util.h zlib.h bzlib.h libutil.h sys/ttydefaults.h)
|
||||
AC_CHECK_HEADERS(execinfo.h pty.h util.h zlib.h bzlib.h libutil.h sys/ttydefaults.h x86intrin.h)
|
||||
|
||||
LNAV_WITH_JEMALLOC
|
||||
|
||||
|
@ -19,6 +19,7 @@ set(diag_STAT_SRCS
|
||||
hist_source.cc
|
||||
hotkeys.cc
|
||||
intern_string.cc
|
||||
is_utf8.cc
|
||||
json-extension-functions.cc
|
||||
json_op.cc
|
||||
json_ptr.cc
|
||||
@ -116,6 +117,7 @@ set(diag_STAT_SRCS
|
||||
hotkeys.hh
|
||||
init-sql.hh
|
||||
intern_string.hh
|
||||
is_utf8.hh
|
||||
k_merge_tree.h
|
||||
log_data_helper.hh
|
||||
log_data_table.hh
|
||||
@ -136,6 +138,7 @@ set(diag_STAT_SRCS
|
||||
relative_time.hh
|
||||
sequence_sink.hh
|
||||
shlex.hh
|
||||
simdutf8check.h
|
||||
spectro_source.hh
|
||||
strong_int.hh
|
||||
sysclip.hh
|
||||
|
@ -96,7 +96,7 @@ time_fmts.cc: ptimec
|
||||
|
||||
if HAVE_RE2C
|
||||
%.cc: %.re
|
||||
$(RE2C_V)$(RE2C_CMD) -8 -o $@ $<
|
||||
$(RE2C_V)$(RE2C_CMD) --tags -8 -o $@ $<
|
||||
$(REC2_V)test $@ -ef $(srcdir)/$*.cc || cp $@ $(srcdir)/$*.cc
|
||||
endif
|
||||
|
||||
@ -173,6 +173,7 @@ noinst_HEADERS = \
|
||||
init.sql \
|
||||
init-sql.hh \
|
||||
intern_string.hh \
|
||||
is_utf8.hh \
|
||||
json_op.hh \
|
||||
json_ptr.hh \
|
||||
k_merge_tree.h \
|
||||
@ -189,6 +190,7 @@ noinst_HEADERS = \
|
||||
log_format.hh \
|
||||
log_format_loader.hh \
|
||||
log_level.hh \
|
||||
log_level_re.re \
|
||||
log_search_table.hh \
|
||||
logfile.hh \
|
||||
logfile_sub_source.hh \
|
||||
@ -216,6 +218,7 @@ noinst_HEADERS = \
|
||||
session_data.hh \
|
||||
shared_buffer.hh \
|
||||
shlex.hh \
|
||||
simdutf8check.h \
|
||||
spectro_source.hh \
|
||||
sql_util.hh \
|
||||
sqlite-extension-func.hh \
|
||||
@ -284,6 +287,7 @@ libdiag_a_SOURCES = \
|
||||
hist_source.cc \
|
||||
hotkeys.cc \
|
||||
intern_string.cc \
|
||||
is_utf8.cc \
|
||||
json-extension-functions.cc \
|
||||
json_op.cc \
|
||||
json_ptr.cc \
|
||||
@ -297,6 +301,7 @@ libdiag_a_SOURCES = \
|
||||
log_format.cc \
|
||||
log_format_loader.cc \
|
||||
log_level.cc \
|
||||
log_level_re.cc \
|
||||
logfile.cc \
|
||||
logfile_sub_source.cc \
|
||||
network-extension-functions.cc \
|
||||
@ -389,11 +394,12 @@ ptimec_LDADD =
|
||||
|
||||
DISTCLEANFILES = \
|
||||
data_scanner_re.cc \
|
||||
default-config-json.c \
|
||||
default-log-formats-json.c \
|
||||
dump-pid-sh.c \
|
||||
help.c \
|
||||
init-sql.c \
|
||||
default-log-formats-json.c \
|
||||
default-config-json.c \
|
||||
log_level_re.cc \
|
||||
time_fmts.cc \
|
||||
xterm-palette.c
|
||||
|
||||
|
@ -433,7 +433,7 @@ private:
|
||||
hist_value b_values[HT__MAX];
|
||||
};
|
||||
|
||||
static const unsigned int BLOCK_SIZE = 100;
|
||||
static const int64_t BLOCK_SIZE = 100;
|
||||
|
||||
struct bucket_block {
|
||||
bucket_block() : bb_used(0) {
|
||||
@ -445,7 +445,7 @@ private:
|
||||
};
|
||||
|
||||
bucket_t &find_bucket(int64_t index) {
|
||||
struct bucket_block &bb = this->hs_blocks[index / this->BLOCK_SIZE];
|
||||
struct bucket_block &bb = this->hs_blocks[index / BLOCK_SIZE];
|
||||
unsigned int intra_block_index = index % BLOCK_SIZE;
|
||||
bb.bb_used = std::max(intra_block_index, bb.bb_used);
|
||||
this->hs_line_count = std::max(this->hs_line_count, index + 1);
|
||||
|
298
src/is_utf8.cc
Normal file
298
src/is_utf8.cc
Normal file
@ -0,0 +1,298 @@
|
||||
/*
|
||||
* is_utf8 is distributed under the following terms:
|
||||
*
|
||||
* Copyright (c) 2013 Palard Julien. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include "is_utf8.hh"
|
||||
|
||||
/*
|
||||
Check if the given unsigned char * is a valid utf-8 sequence.
|
||||
|
||||
Return value :
|
||||
If the string is valid utf-8, 0 is returned.
|
||||
Else the position, starting from 1, is returned.
|
||||
|
||||
Source:
|
||||
http://www.unicode.org/versions/Unicode7.0.0/UnicodeStandard-7.0.pdf
|
||||
page 124, 3.9 "Unicode Encoding Forms", "UTF-8"
|
||||
|
||||
|
||||
Table 3-7. Well-Formed UTF-8 Byte Sequences
|
||||
-----------------------------------------------------------------------------
|
||||
| Code Points | First Byte | Second Byte | Third Byte | Fourth Byte |
|
||||
| U+0000..U+007F | 00..7F | | | |
|
||||
| U+0080..U+07FF | C2..DF | 80..BF | | |
|
||||
| U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
|
||||
| U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | |
|
||||
| U+D000..U+D7FF | ED | 80..9F | 80..BF | |
|
||||
| U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | |
|
||||
| U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
|
||||
| U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
|
||||
| U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
|
||||
-----------------------------------------------------------------------------
|
||||
|
||||
Returns the first erroneous byte position, and give in
|
||||
`faulty_bytes` the number of actually existing bytes taking part in this error.
|
||||
*/
|
||||
ssize_t is_utf8(unsigned char *str, size_t len, const char **message, int *faulty_bytes)
|
||||
{
|
||||
size_t i = 0;
|
||||
|
||||
*message = nullptr;
|
||||
*faulty_bytes = 0;
|
||||
while (i < len)
|
||||
{
|
||||
if (str[i] == '\n') {
|
||||
*message = nullptr;
|
||||
return i;
|
||||
}
|
||||
|
||||
if (str[i] <= 0x7F) /* 00..7F */
|
||||
{
|
||||
i += 1;
|
||||
}
|
||||
else if (str[i] >= 0xC2 && str[i] <= 0xDF) /* C2..DF 80..BF */
|
||||
{
|
||||
if (i + 1 < len) /* Expect a 2nd byte */
|
||||
{
|
||||
if (str[i + 1] < 0x80 || str[i + 1] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte between C2 and DF, expecting a 2nd byte between 80 and BF";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte between C2 and DF, expecting a 2nd byte.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 2;
|
||||
}
|
||||
else if (str[i] == 0xE0) /* E0 A0..BF 80..BF */
|
||||
{
|
||||
if (i + 2 < len) /* Expect a 2nd and 3rd byte */
|
||||
{
|
||||
if (str[i + 1] < 0xA0 || str[i + 1] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of E0, expecting a 2nd byte between A0 and BF.";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 2] < 0x80 || str[i + 2] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of E0, expecting a 3nd byte between 80 and BF.";
|
||||
*faulty_bytes = 3;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte of E0, expecting two following bytes.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 3;
|
||||
}
|
||||
else if (str[i] >= 0xE1 && str[i] <= 0xEC) /* E1..EC 80..BF 80..BF */
|
||||
{
|
||||
if (i + 2 < len) /* Expect a 2nd and 3rd byte */
|
||||
{
|
||||
if (str[i + 1] < 0x80 || str[i + 1] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte between E1 and EC, expecting the 2nd byte between 80 and BF.";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 2] < 0x80 || str[i + 2] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte between E1 and EC, expecting the 3rd byte between 80 and BF.";
|
||||
*faulty_bytes = 3;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte between E1 and EC, expecting two following bytes.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 3;
|
||||
}
|
||||
else if (str[i] == 0xED) /* ED 80..9F 80..BF */
|
||||
{
|
||||
if (i + 2 < len) /* Expect a 2nd and 3rd byte */
|
||||
{
|
||||
if (str[i + 1] < 0x80 || str[i + 1] > 0x9F)
|
||||
{
|
||||
*message = "After a first byte of ED, expecting 2nd byte between 80 and 9F.";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 2] < 0x80 || str[i + 2] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of ED, expecting 3rd byte between 80 and BF.";
|
||||
*faulty_bytes = 3;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte of ED, expecting two following bytes.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 3;
|
||||
}
|
||||
else if (str[i] >= 0xEE && str[i] <= 0xEF) /* EE..EF 80..BF 80..BF */
|
||||
{
|
||||
if (i + 2 < len) /* Expect a 2nd and 3rd byte */
|
||||
{
|
||||
if (str[i + 1] < 0x80 || str[i + 1] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte between EE and EF, expecting 2nd byte between 80 and BF.";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 2] < 0x80 || str[i + 2] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte between EE and EF, expecting 3rd byte between 80 and BF.";
|
||||
*faulty_bytes = 3;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte between EE and EF, two following bytes.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 3;
|
||||
}
|
||||
else if (str[i] == 0xF0) /* F0 90..BF 80..BF 80..BF */
|
||||
{
|
||||
if (i + 3 < len) /* Expect a 2nd, 3rd 3th byte */
|
||||
{
|
||||
if (str[i + 1] < 0x90 || str[i + 1] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F0, expecting 2nd byte between 90 and BF.";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 2] < 0x80 || str[i + 2] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F0, expecting 3rd byte between 80 and BF.";
|
||||
*faulty_bytes = 3;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 3] < 0x80 || str[i + 3] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F0, expecting 4th byte between 80 and BF.";
|
||||
*faulty_bytes = 4;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte of F0, expecting three following bytes.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 4;
|
||||
}
|
||||
else if (str[i] >= 0xF1 && str[i] <= 0xF3) /* F1..F3 80..BF 80..BF 80..BF */
|
||||
{
|
||||
if (i + 3 < len) /* Expect a 2nd, 3rd 3th byte */
|
||||
{
|
||||
if (str[i + 1] < 0x80 || str[i + 1] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F1, F2, or F3, expecting a 2nd byte between 80 and BF.";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 2] < 0x80 || str[i + 2] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F1, F2, or F3, expecting a 3rd byte between 80 and BF.";
|
||||
*faulty_bytes = 3;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 3] < 0x80 || str[i + 3] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F1, F2, or F3, expecting a 4th byte between 80 and BF.";
|
||||
*faulty_bytes = 4;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte of F1, F2, or F3, expecting three following bytes.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 4;
|
||||
}
|
||||
else if (str[i] == 0xF4) /* F4 80..8F 80..BF 80..BF */
|
||||
{
|
||||
if (i + 3 < len) /* Expect a 2nd, 3rd 3th byte */
|
||||
{
|
||||
if (str[i + 1] < 0x80 || str[i + 1] > 0x8F)
|
||||
{
|
||||
*message = "After a first byte of F4, expecting 2nd byte between 80 and 8F.";
|
||||
*faulty_bytes = 2;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 2] < 0x80 || str[i + 2] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F4, expecting 3rd byte between 80 and BF.";
|
||||
*faulty_bytes = 3;
|
||||
return i;
|
||||
}
|
||||
if (str[i + 3] < 0x80 || str[i + 3] > 0xBF)
|
||||
{
|
||||
*message = "After a first byte of F4, expecting 4th byte between 80 and BF.";
|
||||
*faulty_bytes = 4;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "After a first byte of F4, expecting three following bytes.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
i += 4;
|
||||
}
|
||||
else
|
||||
{
|
||||
*message = "Expecting bytes in the following ranges: 00..7F C2..F4.";
|
||||
*faulty_bytes = 1;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
36
src/is_utf8.hh
Normal file
36
src/is_utf8.hh
Normal file
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* is_utf8 is distributed under the following terms:
|
||||
*
|
||||
* Copyright (c) 2013 Palard Julien. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef _IS_UTF8_H
|
||||
#define _IS_UTF8_H
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
ssize_t is_utf8(unsigned char *str, size_t len, const char **message, int *faulty_bytes);
|
||||
|
||||
#endif /* _IS_UTF8_H */
|
@ -43,6 +43,11 @@
|
||||
|
||||
#include <set>
|
||||
|
||||
#ifdef HAVE_X86INTRIN_H
|
||||
#include "simdutf8check.h"
|
||||
#endif
|
||||
|
||||
#include "is_utf8.hh"
|
||||
#include "lnav_util.hh"
|
||||
#include "line_buffer.hh"
|
||||
|
||||
@ -497,6 +502,7 @@ bool line_buffer::read_line(off_t &offset, line_value &lv, bool include_delim)
|
||||
|
||||
lv.lv_len = 0;
|
||||
lv.lv_partial = false;
|
||||
lv.lv_valid_utf = true;
|
||||
while (!retval) {
|
||||
char *line_start, *lf;
|
||||
|
||||
@ -505,7 +511,30 @@ bool line_buffer::read_line(off_t &offset, line_value &lv, bool include_delim)
|
||||
/* Find the data in the cache and */
|
||||
line_start = this->get_range(offset, lv.lv_len);
|
||||
/* ... look for the end-of-line or end-of-file. */
|
||||
if (((lf = (char *)memchr(line_start, '\n', lv.lv_len)) != NULL) ||
|
||||
ssize_t utf8_end = -1;
|
||||
|
||||
#ifdef HAVE_X86INTRIN_H
|
||||
if (!validate_utf8_fast(line_start, lv.lv_len, &utf8_end)) {
|
||||
lv.lv_valid_utf = false;
|
||||
}
|
||||
#else
|
||||
{
|
||||
const char *msg;
|
||||
int faulty_bytes;
|
||||
|
||||
utf8_end = is_utf8(line_start, lv.lv_len, &msg, &faulty_bytes);
|
||||
if (msg != nullptr) {
|
||||
lv.lv_valid_utf = false;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (utf8_end >= 0) {
|
||||
lf = line_start + utf8_end;
|
||||
} else {
|
||||
lf = nullptr;
|
||||
}
|
||||
|
||||
if (lf != nullptr ||
|
||||
(lv.lv_len >= MAX_LINE_BUFFER_SIZE) ||
|
||||
(request_size == MAX_LINE_BUFFER_SIZE) ||
|
||||
((request_size > lv.lv_len) && lv.lv_len > 0)) {
|
||||
@ -604,6 +633,22 @@ bool line_buffer::read_line(off_t &offset_inout, shared_buffer_ref &sbr, line_va
|
||||
sbr.disown();
|
||||
if ((retval = this->read_line(offset_inout, *lv))) {
|
||||
sbr.share(this->lb_share_manager, lv->lv_start, lv->lv_len);
|
||||
if (!lv->lv_valid_utf) {
|
||||
auto *bits = (unsigned char *) sbr.get_writable_data();
|
||||
const char *msg;
|
||||
int faulty_bytes;
|
||||
|
||||
while (true) {
|
||||
ssize_t utf8_end = is_utf8(bits, sbr.length(), &msg, &faulty_bytes);
|
||||
|
||||
if (msg == nullptr) {
|
||||
break;
|
||||
}
|
||||
for (int lpc = 0; lpc < faulty_bytes; lpc++) {
|
||||
bits[utf8_end + lpc] = '?';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return retval;
|
||||
|
@ -48,6 +48,7 @@ struct line_value {
|
||||
char *lv_start;
|
||||
size_t lv_len;
|
||||
bool lv_partial;
|
||||
bool lv_valid_utf;
|
||||
|
||||
void terminate() {
|
||||
this->lv_start[this->lv_len] = '\0';
|
||||
|
24
src/lnav.cc
24
src/lnav.cc
@ -365,15 +365,15 @@ bool setup_logline_table(exec_context &ec)
|
||||
iter.second->get_foreign_keys(db_key_names);
|
||||
}
|
||||
|
||||
db_key_names.push_back("device");
|
||||
db_key_names.push_back("inode");
|
||||
db_key_names.push_back("rowid");
|
||||
db_key_names.push_back("st_dev");
|
||||
db_key_names.push_back("st_ino");
|
||||
db_key_names.push_back("st_mode");
|
||||
db_key_names.push_back("st_rdev");
|
||||
db_key_names.push_back("st_uid");
|
||||
db_key_names.push_back("st_gid");
|
||||
db_key_names.emplace_back("device");
|
||||
db_key_names.emplace_back("inode");
|
||||
db_key_names.emplace_back("rowid");
|
||||
db_key_names.emplace_back("st_dev");
|
||||
db_key_names.emplace_back("st_ino");
|
||||
db_key_names.emplace_back("st_mode");
|
||||
db_key_names.emplace_back("st_rdev");
|
||||
db_key_names.emplace_back("st_uid");
|
||||
db_key_names.emplace_back("st_gid");
|
||||
|
||||
stable_sort(db_key_names.begin(), db_key_names.end());
|
||||
|
||||
@ -490,7 +490,7 @@ class textfile_callback {
|
||||
public:
|
||||
textfile_callback() : force(false), front_file(NULL), front_top(-1) { };
|
||||
|
||||
void closed_file(shared_ptr<logfile> lf) {
|
||||
void closed_file(const shared_ptr<logfile> &lf) {
|
||||
log_info("closed text file: %s", lf->get_filename().c_str());
|
||||
if (!lf->is_valid_filename()) {
|
||||
lnav_data.ld_file_names.erase(lf->get_filename());
|
||||
@ -503,7 +503,7 @@ public:
|
||||
regenerate_unique_file_names();
|
||||
};
|
||||
|
||||
void promote_file(shared_ptr<logfile> lf) {
|
||||
void promote_file(const shared_ptr<logfile> &lf) {
|
||||
if (lnav_data.ld_log_source.insert_file(lf)) {
|
||||
force = true;
|
||||
|
||||
@ -523,7 +523,7 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
void scanned_file(shared_ptr<logfile> lf) {
|
||||
void scanned_file(const shared_ptr<logfile> &lf) {
|
||||
if (!lnav_data.ld_files_to_front.empty() &&
|
||||
lnav_data.ld_files_to_front.front().first ==
|
||||
lf->get_filename()) {
|
||||
|
@ -179,11 +179,11 @@ const char *log_format::log_scanf(const char *line,
|
||||
va_start(args, tv_out);
|
||||
|
||||
pi.reset(line, 0, len);
|
||||
if (!fmt[curr_fmt].pcre.match(pc, pi)) {
|
||||
if (!fmt[curr_fmt].pcre.match(pc, pi, PCRE_NO_UTF8_CHECK)) {
|
||||
retval = NULL;
|
||||
}
|
||||
else {
|
||||
pcre_context::capture_t *ts = pc["timestamp"];
|
||||
pcre_context::capture_t *ts = pc[fmt[curr_fmt].pf_timestamp_index];
|
||||
|
||||
for (auto &iter : pc) {
|
||||
pcre_context::capture_t *cap = va_arg(
|
||||
|
@ -790,13 +790,14 @@ protected:
|
||||
|
||||
struct pcre_format {
|
||||
pcre_format(const char *regex) : name(regex), pcre(regex) {
|
||||
|
||||
this->pf_timestamp_index = this->pcre.name_index("timestamp");
|
||||
};
|
||||
|
||||
pcre_format() : name(NULL), pcre("") { };
|
||||
|
||||
const char *name;
|
||||
pcrepp pcre;
|
||||
int pf_timestamp_index{-1};
|
||||
};
|
||||
|
||||
static bool next_format(pcre_format *fmt, int &index, int &locked_index);
|
||||
@ -1140,7 +1141,7 @@ public:
|
||||
log_level_t convert_level(const pcre_input &pi, pcre_context::capture_t *level_cap) const {
|
||||
log_level_t retval = LEVEL_INFO;
|
||||
|
||||
if (level_cap != NULL && level_cap->is_valid()) {
|
||||
if (level_cap != nullptr && level_cap->is_valid()) {
|
||||
pcre_context_static<128> pc_level;
|
||||
pcre_input pi_level(pi.get_substr_start(level_cap),
|
||||
0,
|
||||
@ -1149,11 +1150,9 @@ public:
|
||||
if (this->elf_level_patterns.empty()) {
|
||||
retval = string2level(pi_level.get_string(), level_cap->length());
|
||||
} else {
|
||||
for (auto iter = this->elf_level_patterns.begin();
|
||||
iter != this->elf_level_patterns.end();
|
||||
++iter) {
|
||||
if (iter->second.lp_pcre->match(pc_level, pi_level)) {
|
||||
retval = iter->first;
|
||||
for (const auto &elf_level_pattern : this->elf_level_patterns) {
|
||||
if (elf_level_pattern.second.lp_pcre->match(pc_level, pi_level)) {
|
||||
retval = elf_level_pattern.first;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
@ -165,7 +165,7 @@ class generic_log_format : public log_format {
|
||||
this->check_for_new_year(dst, log_time, log_tv);
|
||||
}
|
||||
|
||||
dst.push_back(logline(offset, log_tv, level_val));
|
||||
dst.emplace_back(offset, log_tv, level_val);
|
||||
return SCAN_MATCH;
|
||||
}
|
||||
|
||||
|
@ -50,36 +50,6 @@ const char *level_names[LEVEL__MAX + 1] = {
|
||||
NULL
|
||||
};
|
||||
|
||||
static pcrepp LEVEL_RE(
|
||||
"(?i)(TRACE|DEBUG\\d*|INFO|NOTICE|STATS|WARN(?:ING)?|ERR(?:OR)?|CRITICAL|SEVERE|FATAL)");
|
||||
|
||||
log_level_t string2level(const char *levelstr, ssize_t len, bool exact)
|
||||
{
|
||||
log_level_t retval = LEVEL_UNKNOWN;
|
||||
|
||||
if (len == (ssize_t)-1) {
|
||||
len = strlen(levelstr);
|
||||
}
|
||||
|
||||
if (((len == 1) || ((len > 1) && (levelstr[1] == ' '))) &&
|
||||
(retval = abbrev2level(levelstr, 1)) != LEVEL_UNKNOWN) {
|
||||
return retval;
|
||||
}
|
||||
|
||||
pcre_input pi(levelstr, 0, len);
|
||||
pcre_context_static<10> pc;
|
||||
|
||||
if (LEVEL_RE.match(pc, pi)) {
|
||||
auto iter = pc.begin();
|
||||
if (!exact || pc[0]->c_begin == 0) {
|
||||
retval = abbrev2level(pi.get_substr_start(iter),
|
||||
pi.get_substr_len(iter));
|
||||
}
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
log_level_t abbrev2level(const char *levelstr, ssize_t len)
|
||||
{
|
||||
if (len == 0 || levelstr[0] == '\0') {
|
||||
|
590
src/log_level_re.cc
Normal file
590
src/log_level_re.cc
Normal file
@ -0,0 +1,590 @@
|
||||
/* Generated by re2c 1.1.1 on Tue Oct 16 06:58:50 2018 */
|
||||
#line 1 "../../lnav2/src/log_level_re.re"
|
||||
/**
|
||||
* Copyright (c) 2018, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include "log_level.hh"
|
||||
|
||||
log_level_t string2level(const char *levelstr, ssize_t len, bool exact)
|
||||
{
|
||||
log_level_t retval = LEVEL_UNKNOWN;
|
||||
|
||||
if (len == (ssize_t)-1) {
|
||||
len = strlen(levelstr);
|
||||
}
|
||||
|
||||
if (((len == 1) || ((len > 1) && (levelstr[1] == ' '))) &&
|
||||
(retval = abbrev2level(levelstr, 1)) != LEVEL_UNKNOWN) {
|
||||
return retval;
|
||||
}
|
||||
|
||||
# define YYCTYPE unsigned char
|
||||
# define RET(tok) { \
|
||||
return tok; \
|
||||
}
|
||||
|
||||
const YYCTYPE *YYCURSOR = (const unsigned char *) levelstr;
|
||||
const YYCTYPE *YYLIMIT = (const unsigned char *) levelstr + len;
|
||||
const YYCTYPE *YYMARKER = YYCURSOR;
|
||||
const YYCTYPE *debug_level = nullptr;
|
||||
|
||||
# define YYPEEK() (YYCURSOR < YYLIMIT ? *YYCURSOR : 0)
|
||||
# define YYSKIP() ++YYCURSOR
|
||||
# define YYBACKUP() YYMARKER = YYCURSOR
|
||||
# define YYRESTORE() YYCURSOR = YYMARKER
|
||||
# define YYSTAGP(x) x = YYCURSOR - 1
|
||||
|
||||
const unsigned char *yyt1;
|
||||
loop:
|
||||
|
||||
#line 73 "log_level_re.cc"
|
||||
{
|
||||
YYCTYPE yych;
|
||||
unsigned int yyaccept = 0;
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 0x00: goto yy2;
|
||||
case 'C':
|
||||
case 'c': goto yy6;
|
||||
case 'D':
|
||||
case 'd': goto yy7;
|
||||
case 'E':
|
||||
case 'e': goto yy8;
|
||||
case 'F':
|
||||
case 'f': goto yy9;
|
||||
case 'I':
|
||||
case 'i': goto yy10;
|
||||
case 'N':
|
||||
case 'n': goto yy11;
|
||||
case 'S':
|
||||
case 's': goto yy12;
|
||||
case 'T':
|
||||
case 't': goto yy13;
|
||||
case 'W':
|
||||
case 'w': goto yy14;
|
||||
default: goto yy4;
|
||||
}
|
||||
yy2:
|
||||
YYSKIP ();
|
||||
#line 75 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_UNKNOWN); }
|
||||
#line 104 "log_level_re.cc"
|
||||
yy4:
|
||||
YYSKIP ();
|
||||
yy5:
|
||||
#line 102 "../../lnav2/src/log_level_re.re"
|
||||
{ goto loop; }
|
||||
#line 110 "log_level_re.cc"
|
||||
yy6:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'R':
|
||||
case 'r': goto yy15;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy7:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'E':
|
||||
case 'e': goto yy17;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy8:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'R':
|
||||
case 'r': goto yy18;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy9:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'A':
|
||||
case 'a': goto yy19;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy10:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'N':
|
||||
case 'n': goto yy20;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy11:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'O':
|
||||
case 'o': goto yy21;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy12:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'E':
|
||||
case 'e': goto yy22;
|
||||
case 'T':
|
||||
case 't': goto yy23;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy13:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'R':
|
||||
case 'r': goto yy24;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy14:
|
||||
yyaccept = 0;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'A':
|
||||
case 'a': goto yy25;
|
||||
default: goto yy5;
|
||||
}
|
||||
yy15:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'I':
|
||||
case 'i': goto yy26;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy16:
|
||||
YYRESTORE ();
|
||||
switch (yyaccept) {
|
||||
case 0: goto yy5;
|
||||
case 1: goto yy29;
|
||||
default: goto yy48;
|
||||
}
|
||||
yy17:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'B':
|
||||
case 'b': goto yy27;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy18:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'R':
|
||||
case 'r': goto yy28;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy19:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'T':
|
||||
case 't': goto yy30;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy20:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'F':
|
||||
case 'f': goto yy31;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy21:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'T':
|
||||
case 't': goto yy32;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy22:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'V':
|
||||
case 'v': goto yy33;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy23:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'A':
|
||||
case 'a': goto yy34;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy24:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'A':
|
||||
case 'a': goto yy35;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy25:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'R':
|
||||
case 'r': goto yy36;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy26:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'T':
|
||||
case 't': goto yy37;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy27:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'U':
|
||||
case 'u': goto yy38;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy28:
|
||||
yyaccept = 1;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'O':
|
||||
case 'o': goto yy39;
|
||||
default: goto yy29;
|
||||
}
|
||||
yy29:
|
||||
#line 98 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_ERROR); }
|
||||
#line 319 "log_level_re.cc"
|
||||
yy30:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'A':
|
||||
case 'a': goto yy40;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy31:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'O':
|
||||
case 'o': goto yy41;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy32:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'I':
|
||||
case 'i': goto yy43;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy33:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'E':
|
||||
case 'e': goto yy44;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy34:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'T':
|
||||
case 't': goto yy45;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy35:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'C':
|
||||
case 'c': goto yy46;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy36:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'N':
|
||||
case 'n': goto yy47;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy37:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'I':
|
||||
case 'i': goto yy49;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy38:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'G':
|
||||
case 'g': goto yy50;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy39:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'R':
|
||||
case 'r': goto yy52;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy40:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'L':
|
||||
case 'l': goto yy53;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy41:
|
||||
YYSKIP ();
|
||||
#line 94 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_INFO); }
|
||||
#line 412 "log_level_re.cc"
|
||||
yy43:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'C':
|
||||
case 'c': goto yy55;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy44:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'R':
|
||||
case 'r': goto yy56;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy45:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'S':
|
||||
case 's': goto yy57;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy46:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'E':
|
||||
case 'e': goto yy59;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy47:
|
||||
yyaccept = 2;
|
||||
YYSKIP ();
|
||||
YYBACKUP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'I':
|
||||
case 'i': goto yy61;
|
||||
default: goto yy48;
|
||||
}
|
||||
yy48:
|
||||
#line 97 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_WARNING); }
|
||||
#line 458 "log_level_re.cc"
|
||||
yy49:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'C':
|
||||
case 'c': goto yy62;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy50:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5': goto yy63;
|
||||
default:
|
||||
YYSTAGP (yyt1);
|
||||
goto yy51;
|
||||
}
|
||||
yy51:
|
||||
debug_level = yyt1;
|
||||
#line 77 "../../lnav2/src/log_level_re.re"
|
||||
{
|
||||
if (debug_level == nullptr) {
|
||||
RET(LEVEL_DEBUG);
|
||||
}
|
||||
switch (*debug_level) {
|
||||
case '2':
|
||||
RET(LEVEL_DEBUG2);
|
||||
case '3':
|
||||
RET(LEVEL_DEBUG3);
|
||||
case '4':
|
||||
RET(LEVEL_DEBUG4);
|
||||
case '5':
|
||||
RET(LEVEL_DEBUG5);
|
||||
default:
|
||||
RET(LEVEL_DEBUG);
|
||||
}
|
||||
}
|
||||
#line 499 "log_level_re.cc"
|
||||
yy52:
|
||||
YYSKIP ();
|
||||
goto yy29;
|
||||
yy53:
|
||||
YYSKIP ();
|
||||
#line 101 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_FATAL); }
|
||||
#line 507 "log_level_re.cc"
|
||||
yy55:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'E':
|
||||
case 'e': goto yy64;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy56:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'E':
|
||||
case 'e': goto yy66;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy57:
|
||||
YYSKIP ();
|
||||
#line 96 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_STATS); }
|
||||
#line 528 "log_level_re.cc"
|
||||
yy59:
|
||||
YYSKIP ();
|
||||
#line 76 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_TRACE); }
|
||||
#line 533 "log_level_re.cc"
|
||||
yy61:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'N':
|
||||
case 'n': goto yy68;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy62:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'A':
|
||||
case 'a': goto yy69;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy63:
|
||||
YYSKIP ();
|
||||
YYSTAGP (yyt1);
|
||||
goto yy51;
|
||||
yy64:
|
||||
YYSKIP ();
|
||||
#line 95 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_INFO); }
|
||||
#line 558 "log_level_re.cc"
|
||||
yy66:
|
||||
YYSKIP ();
|
||||
#line 100 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_CRITICAL); }
|
||||
#line 563 "log_level_re.cc"
|
||||
yy68:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'G':
|
||||
case 'g': goto yy70;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy69:
|
||||
YYSKIP ();
|
||||
yych = YYPEEK ();
|
||||
switch (yych) {
|
||||
case 'L':
|
||||
case 'l': goto yy71;
|
||||
default: goto yy16;
|
||||
}
|
||||
yy70:
|
||||
YYSKIP ();
|
||||
goto yy48;
|
||||
yy71:
|
||||
YYSKIP ();
|
||||
#line 99 "../../lnav2/src/log_level_re.re"
|
||||
{ RET(LEVEL_CRITICAL); }
|
||||
#line 587 "log_level_re.cc"
|
||||
}
|
||||
#line 104 "../../lnav2/src/log_level_re.re"
|
||||
|
||||
}
|
105
src/log_level_re.re
Normal file
105
src/log_level_re.re
Normal file
@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Copyright (c) 2018, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include "log_level.hh"
|
||||
|
||||
log_level_t string2level(const char *levelstr, ssize_t len, bool exact)
|
||||
{
|
||||
log_level_t retval = LEVEL_UNKNOWN;
|
||||
|
||||
if (len == (ssize_t)-1) {
|
||||
len = strlen(levelstr);
|
||||
}
|
||||
|
||||
if (((len == 1) || ((len > 1) && (levelstr[1] == ' '))) &&
|
||||
(retval = abbrev2level(levelstr, 1)) != LEVEL_UNKNOWN) {
|
||||
return retval;
|
||||
}
|
||||
|
||||
# define YYCTYPE unsigned char
|
||||
# define RET(tok) { \
|
||||
return tok; \
|
||||
}
|
||||
|
||||
const YYCTYPE *YYCURSOR = (const unsigned char *) levelstr;
|
||||
const YYCTYPE *YYLIMIT = (const unsigned char *) levelstr + len;
|
||||
const YYCTYPE *YYMARKER = YYCURSOR;
|
||||
const YYCTYPE *debug_level = nullptr;
|
||||
|
||||
# define YYPEEK() (YYCURSOR < YYLIMIT ? *YYCURSOR : 0)
|
||||
# define YYSKIP() ++YYCURSOR
|
||||
# define YYBACKUP() YYMARKER = YYCURSOR
|
||||
# define YYRESTORE() YYCURSOR = YYMARKER
|
||||
# define YYSTAGP(x) x = YYCURSOR - 1
|
||||
|
||||
/*!stags:re2c format = 'const unsigned char *@@;'; */
|
||||
loop:
|
||||
/*!re2c
|
||||
re2c:yyfill:enable = 0;
|
||||
re2c:flags:input = custom;
|
||||
|
||||
EOF = "\x00";
|
||||
|
||||
EOF { RET(LEVEL_UNKNOWN); }
|
||||
'trace' { RET(LEVEL_TRACE); }
|
||||
'debug' [2-5]? @debug_level {
|
||||
if (debug_level == nullptr) {
|
||||
RET(LEVEL_DEBUG);
|
||||
}
|
||||
switch (*debug_level) {
|
||||
case '2':
|
||||
RET(LEVEL_DEBUG2);
|
||||
case '3':
|
||||
RET(LEVEL_DEBUG3);
|
||||
case '4':
|
||||
RET(LEVEL_DEBUG4);
|
||||
case '5':
|
||||
RET(LEVEL_DEBUG5);
|
||||
default:
|
||||
RET(LEVEL_DEBUG);
|
||||
}
|
||||
}
|
||||
'info' { RET(LEVEL_INFO); }
|
||||
'notice' { RET(LEVEL_INFO); }
|
||||
'stats' { RET(LEVEL_STATS); }
|
||||
'warn'|'warning' { RET(LEVEL_WARNING); }
|
||||
'err'|'error' { RET(LEVEL_ERROR); }
|
||||
'critical' { RET(LEVEL_CRITICAL); }
|
||||
'severe' { RET(LEVEL_CRITICAL); }
|
||||
'fatal' { RET(LEVEL_FATAL); }
|
||||
* { goto loop; }
|
||||
|
||||
*/
|
||||
}
|
@ -56,7 +56,7 @@ static const size_t INDEX_RESERVE_INCREMENT = 1024;
|
||||
logfile::logfile(const string &filename, logfile_open_options &loo)
|
||||
: lf_filename(filename)
|
||||
{
|
||||
require(filename.size() > 0);
|
||||
require(!filename.empty());
|
||||
|
||||
memset(&this->lf_stat, 0, sizeof(this->lf_stat));
|
||||
if (loo.loo_fd == -1) {
|
||||
@ -107,7 +107,7 @@ logfile::~logfile()
|
||||
{
|
||||
}
|
||||
|
||||
bool logfile::exists(void) const
|
||||
bool logfile::exists() const
|
||||
{
|
||||
struct stat st;
|
||||
|
||||
@ -210,14 +210,7 @@ bool logfile::process_prefix(off_t offset, shared_buffer_ref &sbr)
|
||||
|
||||
if (latest < second_to_last) {
|
||||
if (this->lf_format->lf_time_ordered) {
|
||||
log_debug(
|
||||
"%s:%d: out-of-time-order line detected %d.%03d < %d.%03d",
|
||||
this->lf_filename.c_str(),
|
||||
prescan_size,
|
||||
latest.get_time(),
|
||||
latest.get_millis(),
|
||||
second_to_last.get_time(),
|
||||
second_to_last.get_millis());
|
||||
this->lf_out_of_time_order_count += 1;
|
||||
for (size_t lpc = prescan_size;
|
||||
lpc < this->lf_index.size(); lpc++) {
|
||||
logline &line_to_update = this->lf_index[lpc];
|
||||
@ -416,6 +409,13 @@ logfile::rebuild_result_t logfile::rebuild_index()
|
||||
this->lf_index_time = st.st_mtime;
|
||||
}
|
||||
|
||||
if (this->lf_out_of_time_order_count) {
|
||||
log_info("Detected %d out-of-time-order lines in file: %s",
|
||||
this->lf_out_of_time_order_count,
|
||||
this->lf_filename.c_str());
|
||||
this->lf_out_of_time_order_count = 0;
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
|
@ -1,3 +1,5 @@
|
||||
#include <utility>
|
||||
|
||||
/**
|
||||
* Copyright (c) 2007-2012, Timothy Stack
|
||||
*
|
||||
@ -109,8 +111,8 @@ public:
|
||||
|
||||
class error {
|
||||
public:
|
||||
error(const std::string &filename, int err)
|
||||
: e_filename(filename),
|
||||
error(std::string filename, int err)
|
||||
: e_filename(std::move(filename)),
|
||||
e_err(err) { };
|
||||
|
||||
std::string e_filename;
|
||||
@ -205,26 +207,24 @@ public:
|
||||
else {
|
||||
timeradd(&old_time, &tv, &this->lf_time_offset);
|
||||
}
|
||||
for (iterator iter = this->begin();
|
||||
iter != this->end();
|
||||
++iter) {
|
||||
for (auto &iter : *this) {
|
||||
struct timeval curr, diff, new_time;
|
||||
|
||||
curr = iter->get_timeval();
|
||||
curr = iter.get_timeval();
|
||||
timersub(&curr, &old_time, &diff);
|
||||
timeradd(&diff, &this->lf_time_offset, &new_time);
|
||||
iter->set_time(new_time);
|
||||
iter.set_time(new_time);
|
||||
}
|
||||
this->lf_sort_needed = true;
|
||||
};
|
||||
|
||||
void clear_time_offset(void) {
|
||||
void clear_time_offset() {
|
||||
struct timeval tv = { 0, 0 };
|
||||
|
||||
this->adjust_content_time(-1, tv);
|
||||
};
|
||||
|
||||
bool is_time_adjusted(void) const {
|
||||
bool is_time_adjusted() const {
|
||||
return (this->lf_time_offset.tv_sec != 0 ||
|
||||
this->lf_time_offset.tv_usec != 0);
|
||||
}
|
||||
@ -392,9 +392,9 @@ public:
|
||||
};
|
||||
|
||||
/** Check the invariants for this object. */
|
||||
bool invariant(void)
|
||||
bool invariant()
|
||||
{
|
||||
require(this->lf_filename.size() > 0);
|
||||
require(!this->lf_filename.empty());
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -435,6 +435,7 @@ protected:
|
||||
logfile_observer *lf_logfile_observer{nullptr};
|
||||
size_t lf_longest_line{0};
|
||||
text_format_t lf_text_format{TF_UNKNOWN};
|
||||
uint32_t lf_out_of_time_order_count{0};
|
||||
};
|
||||
|
||||
class logline_observer {
|
||||
|
@ -541,7 +541,7 @@ bool logfile_sub_source::rebuild_index(bool force)
|
||||
{
|
||||
iterator iter;
|
||||
size_t total_lines = 0;
|
||||
bool retval, full_sort = false;
|
||||
bool retval, full_sort = false, new_order = false;
|
||||
int file_count = 0;
|
||||
|
||||
force = force || this->lss_force_rebuild;
|
||||
@ -584,6 +584,7 @@ bool logfile_sub_source::rebuild_index(bool force)
|
||||
case logfile::RR_NEW_ORDER:
|
||||
retval = true;
|
||||
force = true;
|
||||
new_order = true;
|
||||
break;
|
||||
}
|
||||
file_count += 1;
|
||||
@ -644,7 +645,10 @@ bool logfile_sub_source::rebuild_index(bool force)
|
||||
}
|
||||
}
|
||||
|
||||
sort(this->lss_index.begin(), this->lss_index.end(), line_cmper);
|
||||
if (new_order || (this->lss_files.size() > 1)) {
|
||||
sort(this->lss_index.begin(), this->lss_index.end(),
|
||||
line_cmper);
|
||||
}
|
||||
} else {
|
||||
kmerge_tree_c<logline, logfile_data, logfile::iterator> merge(
|
||||
file_count);
|
||||
|
@ -85,7 +85,7 @@ void pcrepp::find_captures(const char *pattern)
|
||||
in_class = true;
|
||||
break;
|
||||
case '(':
|
||||
cap_in_progress.push_back(pcre_context::capture(lpc, lpc));
|
||||
cap_in_progress.emplace_back(lpc, lpc);
|
||||
break;
|
||||
case ')': {
|
||||
if (!cap_in_progress.empty()) {
|
||||
|
@ -373,8 +373,12 @@ public:
|
||||
const char *errptr;
|
||||
int eoff;
|
||||
|
||||
if (!(options & PCRE_NEVER_UTF)) {
|
||||
options |= PCRE_UTF8;
|
||||
}
|
||||
|
||||
if ((this->p_code = pcre_compile(pattern,
|
||||
options | PCRE_UTF8,
|
||||
options,
|
||||
&errptr,
|
||||
&eoff,
|
||||
NULL)) == NULL) {
|
||||
@ -421,16 +425,15 @@ public:
|
||||
};
|
||||
|
||||
pcre_named_capture::iterator named_begin() const {
|
||||
return pcre_named_capture::iterator(this->p_named_entries,
|
||||
this->p_name_len);
|
||||
return {this->p_named_entries, static_cast<size_t>(this->p_name_len)};
|
||||
};
|
||||
|
||||
pcre_named_capture::iterator named_end() const {
|
||||
char *ptr = (char *)this->p_named_entries;
|
||||
|
||||
ptr += this->p_named_count * this->p_name_len;
|
||||
return pcre_named_capture::iterator((pcre_named_capture *)ptr,
|
||||
this->p_name_len);
|
||||
return {(pcre_named_capture *)ptr,
|
||||
static_cast<size_t>(this->p_name_len)};
|
||||
};
|
||||
|
||||
const std::vector<pcre_context::capture> &captures() const {
|
||||
@ -565,6 +568,7 @@ public:
|
||||
return length;
|
||||
};
|
||||
|
||||
// #undef PCRE_STUDY_JIT_COMPILE
|
||||
#ifdef PCRE_STUDY_JIT_COMPILE
|
||||
static pcre_jit_stack *jit_stack(void);
|
||||
|
||||
|
237
src/simdutf8check.h
Normal file
237
src/simdutf8check.h
Normal file
@ -0,0 +1,237 @@
|
||||
/**
|
||||
* https://github.com/lemire/fastvalidate-utf-8
|
||||
*/
|
||||
|
||||
#ifndef SIMDUTF8CHECK_H
|
||||
#define SIMDUTF8CHECK_H
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <x86intrin.h>
|
||||
|
||||
#include "lnav_log.hh"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* legal utf-8 byte sequence
|
||||
* http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
|
||||
*
|
||||
* Code Points 1st 2s 3s 4s
|
||||
* U+0000..U+007F 00..7F
|
||||
* U+0080..U+07FF C2..DF 80..BF
|
||||
* U+0800..U+0FFF E0 A0..BF 80..BF
|
||||
* U+1000..U+CFFF E1..EC 80..BF 80..BF
|
||||
* U+D000..U+D7FF ED 80..9F 80..BF
|
||||
* U+E000..U+FFFF EE..EF 80..BF 80..BF
|
||||
* U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
|
||||
* U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
|
||||
* U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
|
||||
*
|
||||
*/
|
||||
|
||||
// all byte values must be no larger than 0xF4
|
||||
static inline void checkSmallerThan0xF4(__m128i current_bytes,
|
||||
__m128i *has_error)
|
||||
{
|
||||
// unsigned, saturates to 0 below max
|
||||
*has_error = _mm_or_si128(*has_error,
|
||||
_mm_subs_epu8(current_bytes,
|
||||
_mm_set1_epi8(0xF4)));
|
||||
}
|
||||
|
||||
static inline __m128i continuationLengths(__m128i high_nibbles)
|
||||
{
|
||||
return _mm_shuffle_epi8(
|
||||
_mm_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII)
|
||||
0, 0, 0, 0, // 10xx (continuation)
|
||||
2, 2, // 110x
|
||||
3, // 1110
|
||||
4), // 1111, next should be 0 (not checked here)
|
||||
high_nibbles);
|
||||
}
|
||||
|
||||
static inline __m128i carryContinuations(__m128i initial_lengths,
|
||||
__m128i previous_carries)
|
||||
{
|
||||
|
||||
__m128i right1 = _mm_subs_epu8(
|
||||
_mm_alignr_epi8(initial_lengths, previous_carries, 16 - 1),
|
||||
_mm_set1_epi8(1));
|
||||
__m128i sum = _mm_add_epi8(initial_lengths, right1);
|
||||
|
||||
__m128i right2 = _mm_subs_epu8(
|
||||
_mm_alignr_epi8(sum, previous_carries, 16 - 2),
|
||||
_mm_set1_epi8(2));
|
||||
return _mm_add_epi8(sum, right2);
|
||||
}
|
||||
|
||||
static inline void checkContinuations(__m128i initial_lengths,
|
||||
__m128i carries,
|
||||
__m128i *has_error)
|
||||
{
|
||||
|
||||
// overlap || underlap
|
||||
// carry > length && length > 0 || !(carry > length) && !(length > 0)
|
||||
// (carries > length) == (lengths > 0)
|
||||
__m128i overunder = _mm_cmpeq_epi8(
|
||||
_mm_cmpgt_epi8(carries, initial_lengths),
|
||||
_mm_cmpgt_epi8(initial_lengths, _mm_setzero_si128()));
|
||||
|
||||
*has_error = _mm_or_si128(*has_error, overunder);
|
||||
}
|
||||
|
||||
// when 0xED is found, next byte must be no larger than 0x9F
|
||||
// when 0xF4 is found, next byte must be no larger than 0x8F
|
||||
// next byte must be continuation, ie sign bit is set, so signed < is ok
|
||||
static inline void checkFirstContinuationMax(__m128i current_bytes,
|
||||
__m128i off1_current_bytes,
|
||||
__m128i *has_error)
|
||||
{
|
||||
__m128i maskED = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xED));
|
||||
__m128i maskF4 = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xF4));
|
||||
|
||||
__m128i badfollowED = _mm_and_si128(
|
||||
_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x9F)),
|
||||
maskED);
|
||||
__m128i badfollowF4 = _mm_and_si128(
|
||||
_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x8F)),
|
||||
maskF4);
|
||||
|
||||
*has_error = _mm_or_si128(*has_error,
|
||||
_mm_or_si128(badfollowED, badfollowF4));
|
||||
}
|
||||
|
||||
// map off1_hibits => error condition
|
||||
// hibits off1 cur
|
||||
// C => < C2 && true
|
||||
// E => < E1 && < A0
|
||||
// F => < F1 && < 90
|
||||
// else false && false
|
||||
static inline void checkOverlong(__m128i current_bytes,
|
||||
__m128i off1_current_bytes,
|
||||
__m128i hibits,
|
||||
__m128i previous_hibits,
|
||||
__m128i *has_error)
|
||||
{
|
||||
__m128i off1_hibits = _mm_alignr_epi8(hibits, previous_hibits, 16 - 1);
|
||||
__m128i initial_mins = _mm_shuffle_epi8(
|
||||
_mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128,
|
||||
-128, -128, -128, -128, // 10xx => false
|
||||
0xC2, -128, // 110x
|
||||
0xE1, // 1110
|
||||
0xF1),
|
||||
off1_hibits);
|
||||
|
||||
__m128i initial_under = _mm_cmpgt_epi8(initial_mins, off1_current_bytes);
|
||||
|
||||
__m128i second_mins = _mm_shuffle_epi8(
|
||||
_mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128,
|
||||
-128, -128, -128, -128, // 10xx => false
|
||||
127, 127, // 110x => true
|
||||
0xA0, // 1110
|
||||
0x90),
|
||||
off1_hibits);
|
||||
__m128i second_under = _mm_cmpgt_epi8(second_mins, current_bytes);
|
||||
*has_error = _mm_or_si128(*has_error,
|
||||
_mm_and_si128(initial_under, second_under));
|
||||
}
|
||||
|
||||
struct processed_utf_bytes {
|
||||
__m128i rawbytes;
|
||||
__m128i high_nibbles;
|
||||
__m128i carried_continuations;
|
||||
};
|
||||
|
||||
static inline void count_nibbles(__m128i bytes,
|
||||
struct processed_utf_bytes *answer)
|
||||
{
|
||||
answer->rawbytes = bytes;
|
||||
answer->high_nibbles = _mm_and_si128(_mm_srli_epi16(bytes, 4),
|
||||
_mm_set1_epi8(0x0F));
|
||||
}
|
||||
|
||||
// check whether the current bytes are valid UTF-8
|
||||
// at the end of the function, previous gets updated
|
||||
static struct processed_utf_bytes
|
||||
checkUTF8Bytes(__m128i current_bytes, struct processed_utf_bytes *previous,
|
||||
__m128i *has_error)
|
||||
{
|
||||
struct processed_utf_bytes pb;
|
||||
count_nibbles(current_bytes, &pb);
|
||||
|
||||
checkSmallerThan0xF4(current_bytes, has_error);
|
||||
|
||||
__m128i initial_lengths = continuationLengths(pb.high_nibbles);
|
||||
|
||||
pb.carried_continuations = carryContinuations(
|
||||
initial_lengths,
|
||||
previous->carried_continuations);
|
||||
|
||||
checkContinuations(initial_lengths, pb.carried_continuations, has_error);
|
||||
|
||||
__m128i off1_current_bytes =
|
||||
_mm_alignr_epi8(pb.rawbytes, previous->rawbytes, 16 - 1);
|
||||
checkFirstContinuationMax(current_bytes, off1_current_bytes,
|
||||
has_error);
|
||||
|
||||
checkOverlong(current_bytes, off1_current_bytes,
|
||||
pb.high_nibbles, previous->high_nibbles, has_error);
|
||||
return pb;
|
||||
}
|
||||
|
||||
static bool validate_utf8_fast(const char *src, size_t len, ssize_t *len_out)
|
||||
{
|
||||
size_t i = 0, orig_len = len;
|
||||
__m128i has_error = _mm_setzero_si128();
|
||||
__m128i lfchars = _mm_set1_epi8('\n');
|
||||
__m128i lfresult = _mm_setzero_si128();
|
||||
struct processed_utf_bytes previous = {.rawbytes = _mm_setzero_si128(),
|
||||
.high_nibbles = _mm_setzero_si128(),
|
||||
.carried_continuations = _mm_setzero_si128()};
|
||||
if (len >= 16) {
|
||||
for (; i <= len - 16; i += 16) {
|
||||
__m128i current_bytes = _mm_loadu_si128(
|
||||
(const __m128i *) (src + i));
|
||||
previous = checkUTF8Bytes(current_bytes, &previous, &has_error);
|
||||
lfresult = _mm_cmpeq_epi8(current_bytes, lfchars);
|
||||
if (_mm_movemask_epi8(lfresult)) {
|
||||
for (; src[i] != '\n'; i++) {
|
||||
}
|
||||
len = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//last part
|
||||
if (i < len) {
|
||||
char buffer[16];
|
||||
memset(buffer, 0, 16);
|
||||
memcpy(buffer, src + i, len - i);
|
||||
__m128i current_bytes = _mm_loadu_si128((const __m128i *) (buffer));
|
||||
previous = checkUTF8Bytes(current_bytes, &previous, &has_error);
|
||||
for (; i < len && src[i] != '\n'; i++) {
|
||||
}
|
||||
} else {
|
||||
has_error = _mm_or_si128(_mm_cmpgt_epi8(previous.carried_continuations,
|
||||
_mm_setr_epi8(9, 9, 9, 9, 9, 9,
|
||||
9, 9, 9, 9, 9, 9,
|
||||
9, 9, 9, 1)),
|
||||
has_error);
|
||||
}
|
||||
|
||||
if (i < orig_len && src[i] == '\n') {
|
||||
*len_out = i;
|
||||
}
|
||||
|
||||
return _mm_testz_si128(has_error, has_error);
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
@ -335,6 +335,7 @@ dist_noinst_DATA = \
|
||||
textfile_json_one_line.0 \
|
||||
textfile_quoted_json.0 \
|
||||
toplevel.lnav \
|
||||
UTF-8-test.txt \
|
||||
view_colors_output.0 \
|
||||
vt52_curses_input.0 \
|
||||
vt52_curses_input.1 \
|
||||
|
BIN
test/UTF-8-test.txt
Normal file
BIN
test/UTF-8-test.txt
Normal file
Binary file not shown.
@ -52,136 +52,139 @@ using namespace std;
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
int c, rnd_iters = 5, retval = EXIT_SUCCESS;
|
||||
vector<pair<int, int> > index;
|
||||
auto_fd fd = STDIN_FILENO;
|
||||
int c, rnd_iters = 5, retval = EXIT_SUCCESS;
|
||||
vector<pair<int, int> > index;
|
||||
auto_fd fd = STDIN_FILENO;
|
||||
int offseti = 0;
|
||||
off_t offset = 0;
|
||||
struct stat st;
|
||||
int count = 1000;
|
||||
struct stat st;
|
||||
|
||||
while ((c = getopt(argc, argv, "o:i:n:")) != -1) {
|
||||
switch (c) {
|
||||
case 'o':
|
||||
if (sscanf(optarg, "%d", &offseti) != 1) {
|
||||
fprintf(stderr,
|
||||
"error: offset is not an integer -- %s\n",
|
||||
optarg);
|
||||
retval = EXIT_FAILURE;
|
||||
} else {
|
||||
offset = offseti;
|
||||
}
|
||||
break;
|
||||
case 'n':
|
||||
if (sscanf(optarg, "%d", &rnd_iters) != 1) {
|
||||
fprintf(stderr,
|
||||
"error: offset is not an integer -- %s\n",
|
||||
optarg);
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
break;
|
||||
case 'i':
|
||||
{
|
||||
FILE *file;
|
||||
while ((c = getopt(argc, argv, "o:i:n:c:")) != -1) {
|
||||
switch (c) {
|
||||
case 'o':
|
||||
if (sscanf(optarg, "%d", &offseti) != 1) {
|
||||
fprintf(stderr,
|
||||
"error: offset is not an integer -- %s\n",
|
||||
optarg);
|
||||
retval = EXIT_FAILURE;
|
||||
} else {
|
||||
offset = offseti;
|
||||
}
|
||||
break;
|
||||
case 'n':
|
||||
if (sscanf(optarg, "%d", &rnd_iters) != 1) {
|
||||
fprintf(stderr,
|
||||
"error: offset is not an integer -- %s\n",
|
||||
optarg);
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
break;
|
||||
case 'c':
|
||||
if (sscanf(optarg, "%d", &count) != 1) {
|
||||
fprintf(stderr,
|
||||
"error: count is not an integer -- %s\n",
|
||||
optarg);
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
break;
|
||||
case 'i': {
|
||||
FILE *file;
|
||||
|
||||
if ((file = fopen(optarg, "r")) == NULL) {
|
||||
perror("open");
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
else {
|
||||
int line_number = 1, line_offset;
|
||||
if ((file = fopen(optarg, "r")) == NULL) {
|
||||
perror("open");
|
||||
retval = EXIT_FAILURE;
|
||||
} else {
|
||||
int line_number = 1, line_offset;
|
||||
|
||||
while (fscanf(file, "%d", &line_offset) == 1) {
|
||||
index.push_back(
|
||||
make_pair(line_number, line_offset));
|
||||
line_number += 1;
|
||||
}
|
||||
fclose(file);
|
||||
file = NULL;
|
||||
while (fscanf(file, "%d", &line_offset) == 1) {
|
||||
index.push_back(
|
||||
make_pair(line_number, line_offset));
|
||||
line_number += 1;
|
||||
}
|
||||
fclose(file);
|
||||
file = NULL;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
retval = EXIT_FAILURE;
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
retval = EXIT_FAILURE;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
argc -= optind;
|
||||
argv += optind;
|
||||
argc -= optind;
|
||||
argv += optind;
|
||||
|
||||
if (retval != EXIT_SUCCESS) {
|
||||
}
|
||||
else if ((argc == 0) && (index.size() > 0)) {
|
||||
fprintf(stderr, "error: cannot randomize stdin\n");
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
else if ((argc > 0) && (fd = open(argv[0], O_RDONLY)) == -1) {
|
||||
perror("open");
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
else if ((argc > 0) && (fstat(fd, &st) == -1)) {
|
||||
perror("fstat");
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
else {
|
||||
try {
|
||||
off_t last_offset = offset;
|
||||
line_buffer lb;
|
||||
line_value lv;
|
||||
char *maddr;
|
||||
|
||||
lb.set_fd(fd);
|
||||
if (index.size() == 0) {
|
||||
while (lb.read_line(offset, lv)) {
|
||||
lv.terminate();
|
||||
printf("%s", lv.lv_start);
|
||||
if ((off_t)(last_offset + lv.lv_len) < offset)
|
||||
printf("\n");
|
||||
last_offset = offset;
|
||||
}
|
||||
}
|
||||
else if ((maddr = (char *)mmap(NULL,
|
||||
st.st_size,
|
||||
PROT_READ,
|
||||
MAP_FILE | MAP_PRIVATE,
|
||||
lb.get_fd(),
|
||||
0)) == MAP_FAILED) {
|
||||
perror("mmap");
|
||||
if (retval != EXIT_SUCCESS) {
|
||||
} else if ((argc == 0) && (index.size() > 0)) {
|
||||
fprintf(stderr, "error: cannot randomize stdin\n");
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
else {
|
||||
off_t seq_offset = 0;
|
||||
} else if ((argc > 0) && (fd = open(argv[0], O_RDONLY)) == -1) {
|
||||
perror("open");
|
||||
retval = EXIT_FAILURE;
|
||||
} else if ((argc > 0) && (fstat(fd, &st) == -1)) {
|
||||
perror("fstat");
|
||||
retval = EXIT_FAILURE;
|
||||
} else {
|
||||
try {
|
||||
off_t last_offset = offset;
|
||||
line_buffer lb;
|
||||
line_value lv;
|
||||
char *maddr;
|
||||
|
||||
while (lb.read_line(seq_offset, lv)) { }
|
||||
do {
|
||||
bool ret;
|
||||
size_t lpc;
|
||||
lb.set_fd(fd);
|
||||
if (index.size() == 0) {
|
||||
shared_buffer_ref sbr;
|
||||
|
||||
random_shuffle(index.begin(), index.end());
|
||||
for (lpc = 0; lpc < index.size(); lpc++) {
|
||||
while (count && lb.read_line(offset, sbr, &lv)) {
|
||||
printf("%.*s", (int) sbr.length(), sbr.get_data());
|
||||
if ((off_t) (last_offset + lv.lv_len) < offset)
|
||||
printf("\n");
|
||||
last_offset = offset;
|
||||
count -= 1;
|
||||
}
|
||||
} else if ((maddr = (char *) mmap(NULL,
|
||||
st.st_size,
|
||||
PROT_READ,
|
||||
MAP_FILE | MAP_PRIVATE,
|
||||
lb.get_fd(),
|
||||
0)) == MAP_FAILED) {
|
||||
perror("mmap");
|
||||
retval = EXIT_FAILURE;
|
||||
} else {
|
||||
off_t seq_offset = 0;
|
||||
|
||||
offset = index[lpc].second;
|
||||
ret = lb.read_line(offset, lv);
|
||||
while (lb.read_line(seq_offset, lv)) {}
|
||||
do {
|
||||
bool ret;
|
||||
size_t lpc;
|
||||
|
||||
assert(ret);
|
||||
assert(offset >= 0);
|
||||
assert(offset <= st.st_size);
|
||||
assert(memcmp(lv.lv_start,
|
||||
&maddr[index[lpc].second],
|
||||
lv.lv_len) == 0);
|
||||
}
|
||||
random_shuffle(index.begin(), index.end());
|
||||
for (lpc = 0; lpc < index.size(); lpc++) {
|
||||
|
||||
rnd_iters -= 1;
|
||||
} while (rnd_iters);
|
||||
offset = index[lpc].second;
|
||||
ret = lb.read_line(offset, lv);
|
||||
|
||||
printf("All done\n");
|
||||
}
|
||||
assert(ret);
|
||||
assert(offset >= 0);
|
||||
assert(offset <= st.st_size);
|
||||
assert(memcmp(lv.lv_start,
|
||||
&maddr[index[lpc].second],
|
||||
lv.lv_len) == 0);
|
||||
}
|
||||
|
||||
rnd_iters -= 1;
|
||||
} while (rnd_iters);
|
||||
|
||||
printf("All done\n");
|
||||
}
|
||||
}
|
||||
catch (line_buffer::error &e) {
|
||||
fprintf(stderr, "error: %s\n", strerror(e.e_err));
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
catch (line_buffer::error &e) {
|
||||
fprintf(stderr, "error: %s\n", strerror(e.e_err));
|
||||
retval = EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
return retval;
|
||||
return retval;
|
||||
}
|
||||
|
@ -161,7 +161,9 @@ int main(int argc, char *argv[]) {
|
||||
break;
|
||||
case MODE_LEVELS:
|
||||
for (logfile::iterator iter = lf.begin(); iter != lf.end(); ++iter) {
|
||||
printf("0x%02x\n", iter->get_level_and_flags());
|
||||
log_level_t level = iter->get_level_and_flags();
|
||||
printf("%s 0x%x\n", level_names[level & ~LEVEL__FLAGS],
|
||||
level & LEVEL__FLAGS);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
@ -42,6 +42,12 @@ check_output "Seeking in the line buffer doesn't work?" <<EOF
|
||||
5
|
||||
EOF
|
||||
|
||||
run_test ./drive_line_buffer -o 4424 -c 1 ${srcdir}/UTF-8-test.txt
|
||||
|
||||
check_output "Invalid UTF is not scrubbed?" <<EOF
|
||||
2.1.5 5 bytes (U-00200000): "?????" |
|
||||
EOF
|
||||
|
||||
cat "${top_srcdir}/src/"*.hh "${top_srcdir}/src/"*.cc > lb-2.dat
|
||||
grep -b '$' lb-2.dat | cut -f 1 -d : > lb.index
|
||||
line_count=`wc -l lb-2.dat`
|
||||
|
@ -222,41 +222,41 @@ EOF
|
||||
run_test ./drive_logfile -v -f syslog_log ${srcdir}/logfile_syslog.0
|
||||
|
||||
check_output "Syslog level interpreted incorrectly?" <<EOF
|
||||
0x0a
|
||||
0x07
|
||||
0x0a
|
||||
0x07
|
||||
error 0x0
|
||||
info 0x0
|
||||
error 0x0
|
||||
info 0x0
|
||||
EOF
|
||||
|
||||
run_test ./drive_logfile -v -f tcsh_history ${srcdir}/logfile_tcsh_history.0
|
||||
|
||||
check_output "TCSH level interpreted incorrectly?" <<EOF
|
||||
0x07
|
||||
0x87
|
||||
0x07
|
||||
0x87
|
||||
info 0x0
|
||||
info 0x80
|
||||
info 0x0
|
||||
info 0x80
|
||||
EOF
|
||||
|
||||
run_test ./drive_logfile -v -f access_log ${srcdir}/logfile_access_log.0
|
||||
|
||||
check_output "access_log level interpreted incorrectly?" <<EOF
|
||||
0x07
|
||||
0x0a
|
||||
0x07
|
||||
info 0x0
|
||||
error 0x0
|
||||
info 0x0
|
||||
EOF
|
||||
|
||||
run_test ./drive_logfile -v -f strace_log ${srcdir}/logfile_strace_log.0
|
||||
|
||||
check_output "strace_log level interpreted incorrectly?" <<EOF
|
||||
0x07
|
||||
0x07
|
||||
0x07
|
||||
0x0a
|
||||
0x07
|
||||
0x0a
|
||||
0x07
|
||||
0x07
|
||||
0x07
|
||||
info 0x0
|
||||
info 0x0
|
||||
info 0x0
|
||||
error 0x0
|
||||
info 0x0
|
||||
error 0x0
|
||||
info 0x0
|
||||
info 0x0
|
||||
info 0x0
|
||||
EOF
|
||||
|
||||
run_test ./drive_logfile -t -f generic_log ${srcdir}/logfile_generic.0
|
||||
@ -269,22 +269,22 @@ EOF
|
||||
run_test ./drive_logfile -v -f generic_log ${srcdir}/logfile_generic.0
|
||||
|
||||
check_output "generic_log level interpreted incorrectly?" <<EOF
|
||||
0x06
|
||||
0x09
|
||||
debug 0x0
|
||||
warning 0x0
|
||||
EOF
|
||||
|
||||
run_test ./drive_logfile -v -f generic_log ${srcdir}/logfile_generic.1
|
||||
|
||||
check_output "generic_log (1) level interpreted incorrectly?" <<EOF
|
||||
0x07
|
||||
0x0a
|
||||
info 0x0
|
||||
error 0x0
|
||||
EOF
|
||||
|
||||
run_test ./drive_logfile -v -f generic_log ${srcdir}/logfile_generic.2
|
||||
|
||||
check_output "generic_log (2) level interpreted incorrectly?" <<EOF
|
||||
0x0a
|
||||
0x0a
|
||||
error 0x0
|
||||
error 0x0
|
||||
EOF
|
||||
|
||||
touch -t 200711030923 ${srcdir}/logfile_glog.0
|
||||
@ -303,13 +303,13 @@ EOF
|
||||
run_test ./drive_logfile -v -f glog_log ${srcdir}/logfile_glog.0
|
||||
|
||||
check_output "glog_log level interpreted incorrectly?" <<EOF
|
||||
0x0a
|
||||
0x07
|
||||
0x07
|
||||
0x09
|
||||
0x07
|
||||
0x07
|
||||
0x0a
|
||||
error 0x0
|
||||
info 0x0
|
||||
info 0x0
|
||||
warning 0x0
|
||||
info 0x0
|
||||
info 0x0
|
||||
error 0x0
|
||||
EOF
|
||||
|
||||
cp ${srcdir}/logfile_syslog.0 truncfile.0
|
||||
|
Loading…
Reference in New Issue
Block a user