From 87c04109146a0956261f7caa4af28167c406cc6d Mon Sep 17 00:00:00 2001 From: Andrey Meshkov Date: Fri, 21 Feb 2020 12:57:12 +0300 Subject: [PATCH] -(dnsforward): start using lastProbeLineIdx in the Seek --- querylog/qlog_file.go | 23 ++++++++++++++--------- querylog/qlog_file_test.go | 31 ++++++++++++++++++++++++++----- querylog/qlog_reader.go | 2 +- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/querylog/qlog_file.go b/querylog/qlog_file.go index 24c5a4e2..54d1896c 100644 --- a/querylog/qlog_file.go +++ b/querylog/qlog_file.go @@ -59,10 +59,12 @@ func NewQLogFile(path string) (*QLogFile, error) { // it shifts seek position to 3/4 of the file. Otherwise, to 1/4 of the file. // 5. It performs the search again, every time the search scope is narrowed twice. // -// It returns the position of the the line with the timestamp we were looking for +// Returns: +// * It returns the position of the the line with the timestamp we were looking for // so that when we call "ReadNext" this line was returned. -// If we could not find it, it returns 0 and ErrSeekNotFound -func (q *QLogFile) Seek(timestamp int64) (int64, error) { +// * Depth of the search (how many times we compared timestamps). +// * If we could not find it, it returns ErrSeekNotFound +func (q *QLogFile) Seek(timestamp int64) (int64, int, error) { q.lock.Lock() defer q.lock.Unlock() @@ -72,7 +74,7 @@ func (q *QLogFile) Seek(timestamp int64) (int64, error) { // First of all, check the file size fileInfo, err := q.file.Stat() if err != nil { - return 0, err + return 0, 0, err } // Define the search scope @@ -91,14 +93,14 @@ func (q *QLogFile) Seek(timestamp int64) (int64, error) { // Get the line at the specified position line, lineIdx, err = q.readProbeLine(probe) if err != nil { - return 0, err + return 0, depth, err } // Get the timestamp from the query log record ts := readQLogTimestamp(line) if ts == 0 { - return 0, ErrSeekNotFound + return 0, depth, ErrSeekNotFound } if ts == timestamp { @@ -109,9 +111,12 @@ func (q *QLogFile) Seek(timestamp int64) (int64, error) { if lastProbeLineIdx == lineIdx { // If we're testing the same line twice then most likely // the scope is too narrow and we won't find anything anymore - return 0, ErrSeekNotFound + return 0, depth, ErrSeekNotFound } + // Save the last found idx + lastProbeLineIdx = lineIdx + // Narrow the scope and repeat the search if ts > timestamp { // If the timestamp we're looking for is OLDER than what we found @@ -128,12 +133,12 @@ func (q *QLogFile) Seek(timestamp int64) (int64, error) { depth++ if depth >= 100 { log.Error("Seek depth is too high, aborting. File %s, ts %v", q.file.Name(), timestamp) - return 0, ErrSeekNotFound + return 0, depth, ErrSeekNotFound } } q.position = lineIdx + int64(len(line)) - return q.position, nil + return q.position, depth, nil } // SeekStart changes the current position to the end of the file diff --git a/querylog/qlog_file_test.go b/querylog/qlog_file_test.go index c5871475..a0fa07f1 100644 --- a/querylog/qlog_file_test.go +++ b/querylog/qlog_file_test.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "io" "io/ioutil" + "math" "net" "os" "strings" @@ -95,13 +96,23 @@ func TestQLogFileSeekLargeFile(t *testing.T) { testSeekLineQLogFile(t, q, count) // CASE 5: Seek non-existent (too low) - _, err = q.Seek(123) + _, _, err = q.Seek(123) assert.NotNil(t, err) // CASE 6: Seek non-existent (too high) ts, _ := time.Parse(time.RFC3339, "2100-01-02T15:04:05Z07:00") - _, err = q.Seek(ts.UnixNano()) + _, _, err = q.Seek(ts.UnixNano()) assert.NotNil(t, err) + + // CASE 7: "Almost" found + line, err := getQLogFileLine(q, count/2) + assert.Nil(t, err) + // ALMOST the record we need + timestamp := readQLogTimestamp(line) - 1 + assert.NotEqual(t, uint64(0), timestamp) + _, depth, err := q.Seek(timestamp) + assert.NotNil(t, err) + assert.True(t, depth <= int(math.Log2(float64(count))+3)) } func TestQLogFileSeekSmallFile(t *testing.T) { @@ -131,13 +142,23 @@ func TestQLogFileSeekSmallFile(t *testing.T) { testSeekLineQLogFile(t, q, count) // CASE 5: Seek non-existent (too low) - _, err = q.Seek(123) + _, _, err = q.Seek(123) assert.NotNil(t, err) // CASE 6: Seek non-existent (too high) ts, _ := time.Parse(time.RFC3339, "2100-01-02T15:04:05Z07:00") - _, err = q.Seek(ts.UnixNano()) + _, _, err = q.Seek(ts.UnixNano()) assert.NotNil(t, err) + + // CASE 7: "Almost" found + line, err := getQLogFileLine(q, count/2) + assert.Nil(t, err) + // ALMOST the record we need + timestamp := readQLogTimestamp(line) - 1 + assert.NotEqual(t, uint64(0), timestamp) + _, depth, err := q.Seek(timestamp) + assert.NotNil(t, err) + assert.True(t, depth <= int(math.Log2(float64(count))+3)) } func testSeekLineQLogFile(t *testing.T, q *QLogFile, lineNumber int) { @@ -147,7 +168,7 @@ func testSeekLineQLogFile(t *testing.T, q *QLogFile, lineNumber int) { assert.NotEqual(t, uint64(0), ts) // try seeking to that line now - pos, err := q.Seek(ts) + pos, _, err := q.Seek(ts) assert.Nil(t, err) assert.NotEqual(t, int64(0), pos) diff --git a/querylog/qlog_reader.go b/querylog/qlog_reader.go index 5f4f4dfa..ee2f617d 100644 --- a/querylog/qlog_reader.go +++ b/querylog/qlog_reader.go @@ -51,7 +51,7 @@ func NewQLogReader(files []string) (*QLogReader, error) { func (r *QLogReader) Seek(timestamp int64) error { for i := len(r.qFiles) - 1; i >= 0; i-- { q := r.qFiles[i] - _, err := q.Seek(timestamp) + _, _, err := q.Seek(timestamp) if err == nil { // Our search is finished, we found the element we were looking for // Update currentFile only, position is already set properly in the QLogFile