mirror of
https://github.com/AdguardTeam/AdGuardHome.git
synced 2024-08-16 22:20:30 +03:00
Pull request: 2504 querylog interval
Merge in DNS/adguard-home from 2504-querylog-ivl to master
Updates #2504.
Squashed commit of the following:
commit 5d15a6f735cd195fc81c8af909b56fbc7db1fe21
Merge: 8cd5c30d 97073d0d
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 18:45:10 2021 +0300
Merge branch 'master' into 2504-querylog-ivl
commit 8cd5c30de6f72d4b12162dbc9e3d90132795fe94
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 18:35:50 2021 +0300
client: fix fmt
commit e95d462c31d886bacec0735acc567fec7c962149
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 17:58:06 2021 +0300
home: imp code
commit 48737b249c52a997a4f34dac45fbaf699477b007
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 17:23:18 2021 +0300
home: imp duration
commit 44f5dc3d3ada5120d74caa24cace9a253b8f15d3
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 16:55:31 2021 +0300
home: imp code, docs
commit bb2826521b7e5d248ce2ab686528219c312b8ba2
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 16:11:40 2021 +0300
all: imp code, docs
commit d688aed1f340807a8bac8807c263956b0fc16f5b
Author: Eugene Burkov <e.burkov@adguard.com>
Date: Thu Jul 1 13:49:42 2021 +0300
all: change querylog interval setting format
This commit is contained in:
parent
97073d0d9e
commit
e113b276e7
@ -15,6 +15,7 @@ and this project adheres to
|
||||
|
||||
### Added
|
||||
|
||||
- New possible value of `6h` for `querylog_interval` setting ([#2504]).
|
||||
- Blocking access using client IDs ([#2624], [#3162]).
|
||||
- `source` directives support in `/etc/network/interfaces` on Linux ([#3257]).
|
||||
- RFC 9000 support in DNS-over-QUIC.
|
||||
@ -40,6 +41,7 @@ and this project adheres to
|
||||
|
||||
### Changed
|
||||
|
||||
- `querylog_interval` setting is now formatted in hours.
|
||||
- Query log search now supports internationalized domains ([#3012]).
|
||||
- Internationalized domains are now shown decoded in the query log with the
|
||||
original encoded version shown in request details ([#3013]).
|
||||
@ -82,6 +84,7 @@ released by then.
|
||||
[#2439]: https://github.com/AdguardTeam/AdGuardHome/issues/2439
|
||||
[#2441]: https://github.com/AdguardTeam/AdGuardHome/issues/2441
|
||||
[#2443]: https://github.com/AdguardTeam/AdGuardHome/issues/2443
|
||||
[#2504]: https://github.com/AdguardTeam/AdGuardHome/issues/2504
|
||||
[#2624]: https://github.com/AdguardTeam/AdGuardHome/issues/2624
|
||||
[#2763]: https://github.com/AdguardTeam/AdGuardHome/issues/2763
|
||||
[#3012]: https://github.com/AdguardTeam/AdGuardHome/issues/3012
|
||||
|
@ -484,6 +484,7 @@
|
||||
"encryption_key_source_content": "Paste the private key contents",
|
||||
"stats_params": "Statistics configuration",
|
||||
"config_successfully_saved": "Configuration successfully saved",
|
||||
"interval_6_hour": "6 hours",
|
||||
"interval_24_hour": "24 hours",
|
||||
"interval_days": "{{count}} day",
|
||||
"interval_days_plural": "{{count}} days",
|
||||
|
@ -4,26 +4,33 @@ import { Field, reduxForm } from 'redux-form';
|
||||
import { Trans, withTranslation } from 'react-i18next';
|
||||
import flow from 'lodash/flow';
|
||||
|
||||
import { CheckboxField, renderRadioField, toNumber } from '../../../helpers/form';
|
||||
import { CheckboxField, renderRadioField, toFloatNumber } from '../../../helpers/form';
|
||||
import { FORM_NAME, QUERY_LOG_INTERVALS_DAYS } from '../../../helpers/constants';
|
||||
import '../FormButton.css';
|
||||
|
||||
const getIntervalFields = (processing, t, toNumber) => QUERY_LOG_INTERVALS_DAYS.map((interval) => {
|
||||
const title = interval === 1 ? t('interval_24_hour') : t('interval_days', { count: interval });
|
||||
const getIntervalTitle = (interval, t) => {
|
||||
switch (interval) {
|
||||
case 0.25:
|
||||
return t('interval_6_hour');
|
||||
case 1:
|
||||
return t('interval_24_hour');
|
||||
default:
|
||||
return t('interval_days', { count: interval });
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Field
|
||||
key={interval}
|
||||
name="interval"
|
||||
type="radio"
|
||||
component={renderRadioField}
|
||||
value={interval}
|
||||
placeholder={title}
|
||||
normalize={toNumber}
|
||||
disabled={processing}
|
||||
/>
|
||||
);
|
||||
});
|
||||
const getIntervalFields = (processing, t, toNumber) => QUERY_LOG_INTERVALS_DAYS.map((interval) => (
|
||||
<Field
|
||||
key={interval}
|
||||
name="interval"
|
||||
type="radio"
|
||||
component={renderRadioField}
|
||||
value={interval}
|
||||
placeholder={getIntervalTitle(interval, t)}
|
||||
normalize={toNumber}
|
||||
disabled={processing}
|
||||
/>
|
||||
));
|
||||
|
||||
const Form = (props) => {
|
||||
const {
|
||||
@ -56,7 +63,7 @@ const Form = (props) => {
|
||||
</label>
|
||||
<div className="form__group form__group--settings">
|
||||
<div className="custom-controls-stacked">
|
||||
{getIntervalFields(processing, t, toNumber)}
|
||||
{getIntervalFields(processing, t, toFloatNumber)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="mt-5">
|
||||
|
@ -357,7 +357,7 @@ export const NOT_FILTERED = 'NotFiltered';
|
||||
|
||||
export const STATS_INTERVALS_DAYS = [0, 1, 7, 30, 90];
|
||||
|
||||
export const QUERY_LOG_INTERVALS_DAYS = [1, 7, 30, 90];
|
||||
export const QUERY_LOG_INTERVALS_DAYS = [0.25, 1, 7, 30, 90];
|
||||
|
||||
export const FILTERS_INTERVALS_HOURS = [0, 1, 12, 24, 72, 168];
|
||||
|
||||
|
@ -276,6 +276,12 @@ export const ip4ToInt = (ip) => {
|
||||
*/
|
||||
export const toNumber = (value) => value && parseInt(value, 10);
|
||||
|
||||
/**
|
||||
* @param value {string}
|
||||
* @returns {*|number}
|
||||
*/
|
||||
export const toFloatNumber = (value) => value && parseFloat(value, 10);
|
||||
|
||||
/**
|
||||
* @param value {string}
|
||||
* @returns {boolean}
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
|
||||
@ -102,11 +103,12 @@ type dnsConfig struct {
|
||||
// time interval for statistics (in days)
|
||||
StatsInterval uint32 `yaml:"statistics_interval"`
|
||||
|
||||
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
|
||||
QueryLogFileEnabled bool `yaml:"querylog_file_enabled"` // if true, query log will be written to a file
|
||||
QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
|
||||
QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
|
||||
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats
|
||||
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
|
||||
QueryLogFileEnabled bool `yaml:"querylog_file_enabled"` // if true, query log will be written to a file
|
||||
// QueryLogInterval is the interval for query log's files rotation.
|
||||
QueryLogInterval Duration `yaml:"querylog_interval"`
|
||||
QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
|
||||
AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats
|
||||
|
||||
dnsforward.FilteringConfig `yaml:",inline"`
|
||||
|
||||
@ -185,7 +187,7 @@ var config = configuration{
|
||||
},
|
||||
FilteringEnabled: true, // whether or not use filter lists
|
||||
FiltersUpdateIntervalHours: 24,
|
||||
UpstreamTimeout: Duration{dnsforward.DefaultTimeout},
|
||||
UpstreamTimeout: Duration{Duration: dnsforward.DefaultTimeout},
|
||||
LocalDomainName: "lan",
|
||||
ResolveClients: true,
|
||||
UsePrivateRDNS: true,
|
||||
@ -212,7 +214,7 @@ func initConfig() {
|
||||
|
||||
config.DNS.QueryLogEnabled = true
|
||||
config.DNS.QueryLogFileEnabled = true
|
||||
config.DNS.QueryLogInterval = 90
|
||||
config.DNS.QueryLogInterval = Duration{Duration: 90 * 24 * time.Hour}
|
||||
config.DNS.QueryLogMemSize = 1000
|
||||
|
||||
config.DNS.CacheSize = 4 * 1024 * 1024
|
||||
@ -281,7 +283,7 @@ func parseConfig() error {
|
||||
}
|
||||
|
||||
if config.DNS.UpstreamTimeout.Duration == 0 {
|
||||
config.DNS.UpstreamTimeout = Duration{dnsforward.DefaultTimeout}
|
||||
config.DNS.UpstreamTimeout = Duration{Duration: dnsforward.DefaultTimeout}
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -328,7 +330,7 @@ func (c *configuration) write() error {
|
||||
Context.queryLog.WriteDiskConfig(&dc)
|
||||
config.DNS.QueryLogEnabled = dc.Enabled
|
||||
config.DNS.QueryLogFileEnabled = dc.FileEnabled
|
||||
config.DNS.QueryLogInterval = dc.RotationIvl
|
||||
config.DNS.QueryLogInterval = Duration{Duration: dc.RotationIvl}
|
||||
config.DNS.QueryLogMemSize = dc.MemSize
|
||||
config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
|
||||
}
|
||||
|
@ -48,7 +48,7 @@ func initDNSServer() error {
|
||||
HTTPRegister: httpRegister,
|
||||
FindClient: Context.clients.findMultiple,
|
||||
BaseDir: baseDir,
|
||||
RotationIvl: config.DNS.QueryLogInterval,
|
||||
RotationIvl: config.DNS.QueryLogInterval.Duration,
|
||||
MemSize: config.DNS.QueryLogMemSize,
|
||||
Enabled: config.DNS.QueryLogEnabled,
|
||||
FileEnabled: config.DNS.QueryLogFileEnabled,
|
||||
|
@ -12,6 +12,35 @@ type Duration struct {
|
||||
time.Duration
|
||||
}
|
||||
|
||||
// String implements the fmt.Stringer interface for Duration. It wraps
|
||||
// time.Duration.String method and additionally cuts off non-leading zero values
|
||||
// of minutes and seconds. Some values which are differ between the
|
||||
// implementations:
|
||||
//
|
||||
// Duration: "1m", time.Duration: "1m0s"
|
||||
// Duration: "1h", time.Duration: "1h0m0s"
|
||||
// Duration: "1h1m", time.Duration: "1h1m0s"
|
||||
//
|
||||
func (d Duration) String() (str string) {
|
||||
str = d.Duration.String()
|
||||
secs := d.Seconds()
|
||||
var secsInt int
|
||||
if secsInt = int(secs); float64(secsInt) != secs || secsInt%60 != 0 {
|
||||
return str
|
||||
}
|
||||
|
||||
const (
|
||||
tailMin = len(`0s`)
|
||||
tailMinSec = len(`0m0s`)
|
||||
)
|
||||
|
||||
if (secsInt%3600)/60 != 0 {
|
||||
return str[:len(str)-tailMin]
|
||||
}
|
||||
|
||||
return str[:len(str)-tailMinSec]
|
||||
}
|
||||
|
||||
// MarshalText implements the encoding.TextMarshaler interface for Duration.
|
||||
func (d Duration) MarshalText() (text []byte, err error) {
|
||||
return []byte(d.String()), nil
|
||||
@ -19,6 +48,8 @@ func (d Duration) MarshalText() (text []byte, err error) {
|
||||
|
||||
// UnmarshalText implements the encoding.TextUnmarshaler interface for
|
||||
// *Duration.
|
||||
//
|
||||
// TODO(e.burkov): Make it able to parse larger units like days.
|
||||
func (d *Duration) UnmarshalText(b []byte) (err error) {
|
||||
defer func() { err = errors.Annotate(err, "unmarshalling duration: %w") }()
|
||||
|
||||
|
@ -12,6 +12,50 @@ import (
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
func TestDuration_String(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
val time.Duration
|
||||
}{{
|
||||
name: "1s",
|
||||
val: time.Second,
|
||||
}, {
|
||||
name: "1m",
|
||||
val: time.Minute,
|
||||
}, {
|
||||
name: "1h",
|
||||
val: time.Hour,
|
||||
}, {
|
||||
name: "1m1s",
|
||||
val: time.Minute + time.Second,
|
||||
}, {
|
||||
name: "1h1m",
|
||||
val: time.Hour + time.Minute,
|
||||
}, {
|
||||
name: "1h0m1s",
|
||||
val: time.Hour + time.Second,
|
||||
}, {
|
||||
name: "1ms",
|
||||
val: time.Millisecond,
|
||||
}, {
|
||||
name: "1h0m0.001s",
|
||||
val: time.Hour + time.Millisecond,
|
||||
}, {
|
||||
name: "1.001s",
|
||||
val: time.Second + time.Millisecond,
|
||||
}, {
|
||||
name: "1m1.001s",
|
||||
val: time.Minute + time.Second + time.Millisecond,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
d := Duration{Duration: tc.val}
|
||||
assert.Equal(t, tc.name, d.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// durationEncodingTester is a helper struct to simplify testing different
|
||||
// Duration marshalling and unmarshalling cases.
|
||||
type durationEncodingTester struct {
|
||||
|
@ -9,6 +9,7 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
@ -19,7 +20,7 @@ import (
|
||||
)
|
||||
|
||||
// currentSchemaVersion is the current schema version.
|
||||
const currentSchemaVersion = 11
|
||||
const currentSchemaVersion = 12
|
||||
|
||||
// These aliases are provided for convenience.
|
||||
type (
|
||||
@ -82,6 +83,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
|
||||
upgradeSchema8to9,
|
||||
upgradeSchema9to10,
|
||||
upgradeSchema10to11,
|
||||
upgradeSchema11to12,
|
||||
}
|
||||
|
||||
n := 0
|
||||
@ -647,6 +649,46 @@ func upgradeSchema10to11(diskConf yobj) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// upgradeSchema11to12 performs the following changes:
|
||||
//
|
||||
// # BEFORE:
|
||||
// 'querylog_interval': 90
|
||||
//
|
||||
// # AFTER:
|
||||
// 'querylog_interval': '2160h'
|
||||
//
|
||||
func upgradeSchema11to12(diskConf yobj) (err error) {
|
||||
log.Printf("Upgrade yaml: 11 to 12")
|
||||
diskConf["schema_version"] = 12
|
||||
|
||||
dnsVal, ok := diskConf["dns"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
var dns yobj
|
||||
dns, ok = dnsVal.(yobj)
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
|
||||
}
|
||||
|
||||
const field = "querylog_interval"
|
||||
|
||||
// Set the initial value from home.initConfig function.
|
||||
qlogIvl := 90
|
||||
qlogIvlVal, ok := dns[field]
|
||||
if ok {
|
||||
qlogIvl, ok = qlogIvlVal.(int)
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected type of %s: %T", field, qlogIvlVal)
|
||||
}
|
||||
}
|
||||
|
||||
dns[field] = Duration{Duration: time.Duration(qlogIvl) * 24 * time.Hour}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Replace with log.Output when we port it to our logging
|
||||
// package.
|
||||
func funcName() string {
|
||||
|
@ -2,6 +2,7 @@ package home
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@ -415,3 +416,98 @@ func TestUpgradeSchema10to11(t *testing.T) {
|
||||
check(t, conf)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUpgradeSchema11to12(t *testing.T) {
|
||||
testCases := []struct {
|
||||
ivl any
|
||||
want any
|
||||
wantErr string
|
||||
name string
|
||||
}{{
|
||||
ivl: 1,
|
||||
want: Duration{Duration: 24 * time.Hour},
|
||||
wantErr: "",
|
||||
name: "success",
|
||||
}, {
|
||||
ivl: 0.25,
|
||||
want: 0,
|
||||
wantErr: "unexpected type of querylog_interval: float64",
|
||||
name: "fail",
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
conf := yobj{
|
||||
"dns": yobj{
|
||||
"querylog_interval": tc.ivl,
|
||||
},
|
||||
"schema_version": 11,
|
||||
}
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := upgradeSchema11to12(conf)
|
||||
|
||||
if tc.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tc.wantErr, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, conf["schema_version"], 12)
|
||||
|
||||
dnsVal, ok := conf["dns"]
|
||||
require.True(t, ok)
|
||||
|
||||
var newDNSConf yobj
|
||||
newDNSConf, ok = dnsVal.(yobj)
|
||||
require.True(t, ok)
|
||||
|
||||
var newIvl Duration
|
||||
newIvl, ok = newDNSConf["querylog_interval"].(Duration)
|
||||
require.True(t, ok)
|
||||
|
||||
assert.Equal(t, tc.want, newIvl)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("no_dns", func(t *testing.T) {
|
||||
err := upgradeSchema11to12(yobj{})
|
||||
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("bad_dns", func(t *testing.T) {
|
||||
err := upgradeSchema11to12(yobj{
|
||||
"dns": 0,
|
||||
})
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "unexpected type of dns: int", err.Error())
|
||||
})
|
||||
|
||||
t.Run("no_field", func(t *testing.T) {
|
||||
conf := yobj{
|
||||
"dns": yobj{},
|
||||
}
|
||||
|
||||
err := upgradeSchema11to12(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
dns, ok := conf["dns"]
|
||||
require.True(t, ok)
|
||||
|
||||
var dnsVal yobj
|
||||
dnsVal, ok = dns.(yobj)
|
||||
require.True(t, ok)
|
||||
|
||||
var ivl interface{}
|
||||
ivl, ok = dnsVal["querylog_interval"]
|
||||
require.True(t, ok)
|
||||
|
||||
var ivlVal Duration
|
||||
ivlVal, ok = ivl.(Duration)
|
||||
require.True(t, ok)
|
||||
|
||||
assert.Equal(t, 90*24*time.Hour, ivlVal.Duration)
|
||||
})
|
||||
}
|
||||
|
@ -16,9 +16,11 @@ import (
|
||||
)
|
||||
|
||||
type qlogConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Interval uint32 `json:"interval"`
|
||||
AnonymizeClientIP bool `json:"anonymize_client_ip"`
|
||||
Enabled bool `json:"enabled"`
|
||||
// Use float64 here to support fractional numbers and not mess the API
|
||||
// users by changing the units.
|
||||
Interval float64 `json:"interval"`
|
||||
AnonymizeClientIP bool `json:"anonymize_client_ip"`
|
||||
}
|
||||
|
||||
// Register web handlers
|
||||
@ -71,7 +73,7 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
|
||||
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
|
||||
resp := qlogConfig{}
|
||||
resp.Enabled = l.conf.Enabled
|
||||
resp.Interval = l.conf.RotationIvl
|
||||
resp.Interval = l.conf.RotationIvl.Hours() / 24
|
||||
resp.AnonymizeClientIP = l.conf.AnonymizeClientIP
|
||||
|
||||
jsonVal, err := json.Marshal(resp)
|
||||
@ -95,7 +97,8 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
if req.Exists("interval") && !checkInterval(d.Interval) {
|
||||
ivl := time.Duration(24*d.Interval) * time.Hour
|
||||
if req.Exists("interval") && !checkInterval(ivl) {
|
||||
httpError(r, w, http.StatusBadRequest, "Unsupported interval")
|
||||
return
|
||||
}
|
||||
@ -107,7 +110,7 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
|
||||
conf.Enabled = d.Enabled
|
||||
}
|
||||
if req.Exists("interval") {
|
||||
conf.RotationIvl = d.Interval
|
||||
conf.RotationIvl = ivl
|
||||
}
|
||||
if req.Exists("anonymize_client_ip") {
|
||||
conf.AnonymizeClientIP = d.AnonymizeClientIP
|
||||
|
@ -100,8 +100,17 @@ func (l *queryLog) Close() {
|
||||
_ = l.flushLogBuffer(true)
|
||||
}
|
||||
|
||||
func checkInterval(days uint32) bool {
|
||||
return days == 1 || days == 7 || days == 30 || days == 90
|
||||
func checkInterval(ivl time.Duration) (ok bool) {
|
||||
// The constants for possible values of query log's rotation interval.
|
||||
const (
|
||||
quarterDay = 6 * time.Hour
|
||||
day = 24 * time.Hour
|
||||
week = day * 7
|
||||
month = day * 30
|
||||
threeMonths = day * 90
|
||||
)
|
||||
|
||||
return ivl == quarterDay || ivl == day || ivl == week || ivl == month || ivl == threeMonths
|
||||
}
|
||||
|
||||
func (l *queryLog) WriteDiskConfig(c *Config) {
|
||||
|
@ -26,7 +26,7 @@ func TestQueryLog(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
FileEnabled: true,
|
||||
RotationIvl: 1,
|
||||
RotationIvl: 24 * time.Hour,
|
||||
MemSize: 100,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
@ -128,7 +128,7 @@ func TestQueryLog(t *testing.T) {
|
||||
func TestQueryLogOffsetLimit(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
RotationIvl: 1,
|
||||
RotationIvl: 24 * time.Hour,
|
||||
MemSize: 100,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
@ -153,33 +153,34 @@ func TestQueryLogOffsetLimit(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
want string
|
||||
wantLen int
|
||||
offset int
|
||||
limit int
|
||||
wantLen int
|
||||
want string
|
||||
}{{
|
||||
name: "page_1",
|
||||
want: firstPageDomain,
|
||||
wantLen: 10,
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
wantLen: 10,
|
||||
want: firstPageDomain,
|
||||
}, {
|
||||
name: "page_2",
|
||||
want: secondPageDomain,
|
||||
wantLen: 10,
|
||||
offset: 10,
|
||||
limit: 10,
|
||||
wantLen: 10,
|
||||
want: secondPageDomain,
|
||||
}, {
|
||||
name: "page_2.5",
|
||||
want: secondPageDomain,
|
||||
wantLen: 5,
|
||||
offset: 15,
|
||||
limit: 10,
|
||||
wantLen: 5,
|
||||
want: secondPageDomain,
|
||||
}, {
|
||||
name: "page_3",
|
||||
want: "",
|
||||
wantLen: 0,
|
||||
offset: 20,
|
||||
limit: 10,
|
||||
wantLen: 0,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
@ -202,7 +203,7 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
FileEnabled: true,
|
||||
RotationIvl: 1,
|
||||
RotationIvl: 24 * time.Hour,
|
||||
MemSize: 100,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
@ -230,7 +231,7 @@ func TestQueryLogFileDisabled(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
FileEnabled: false,
|
||||
RotationIvl: 1,
|
||||
RotationIvl: 24 * time.Hour,
|
||||
MemSize: 2,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
|
@ -41,10 +41,17 @@ type Config struct {
|
||||
// BaseDir is the base directory for log files.
|
||||
BaseDir string
|
||||
|
||||
// RotationIvl is the interval for log rotation, in days. After that
|
||||
// period, the old log file will be renamed, NOT deleted, so the actual
|
||||
// log retention time is twice the interval.
|
||||
RotationIvl uint32
|
||||
// RotationIvl is the interval for log rotation. After that period, the
|
||||
// old log file will be renamed, NOT deleted, so the actual log
|
||||
// retention time is twice the interval. The value must be one of:
|
||||
//
|
||||
// 6 * time.Hour
|
||||
// 24 * time.Hour
|
||||
// 7 * 24 * time.Hour
|
||||
// 30 * 24 * time.Hour
|
||||
// 90 * 24 * time.Hour
|
||||
//
|
||||
RotationIvl time.Duration
|
||||
|
||||
// MemSize is the number of entries kept in a memory buffer before they
|
||||
// are flushed to disk.
|
||||
@ -118,7 +125,7 @@ func newQueryLog(conf Config) (l *queryLog) {
|
||||
"querylog: warning: unsupported rotation interval %d, setting to 1 day",
|
||||
conf.RotationIvl,
|
||||
)
|
||||
l.conf.RotationIvl = 1
|
||||
l.conf.RotationIvl = 24 * time.Hour
|
||||
}
|
||||
|
||||
return l
|
||||
|
@ -104,43 +104,52 @@ func (l *queryLog) rotate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *queryLog) readFileFirstTimeValue() int64 {
|
||||
f, err := os.Open(l.logFile)
|
||||
func (l *queryLog) readFileFirstTimeValue() (first time.Time, err error) {
|
||||
var f *os.File
|
||||
f, err = os.Open(l.logFile)
|
||||
if err != nil {
|
||||
return -1
|
||||
return time.Time{}, err
|
||||
}
|
||||
defer func() {
|
||||
derr := f.Close()
|
||||
if derr != nil {
|
||||
log.Error("querylog: closing file: %s", derr)
|
||||
}
|
||||
}()
|
||||
|
||||
buf := make([]byte, 500)
|
||||
r, err := f.Read(buf)
|
||||
defer func() { err = errors.WithDeferred(err, f.Close()) }()
|
||||
|
||||
buf := make([]byte, 512)
|
||||
var r int
|
||||
r, err = f.Read(buf)
|
||||
if err != nil {
|
||||
return -1
|
||||
return time.Time{}, err
|
||||
}
|
||||
buf = buf[:r]
|
||||
|
||||
val := readJSONValue(string(buf), `"T":"`)
|
||||
val := readJSONValue(string(buf[:r]), `"T":"`)
|
||||
t, err := time.Parse(time.RFC3339Nano, val)
|
||||
if err != nil {
|
||||
return -1
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
log.Debug("querylog: the oldest log entry: %s", val)
|
||||
return t.Unix()
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (l *queryLog) periodicRotate() {
|
||||
intervalSeconds := uint64(l.conf.RotationIvl) * 24 * 60 * 60
|
||||
defer log.OnPanic("querylog: rotating")
|
||||
|
||||
var err error
|
||||
for {
|
||||
oldest := l.readFileFirstTimeValue()
|
||||
if uint64(oldest)+intervalSeconds <= uint64(time.Now().Unix()) {
|
||||
_ = l.rotate()
|
||||
var oldest time.Time
|
||||
oldest, err = l.readFileFirstTimeValue()
|
||||
if err != nil {
|
||||
log.Debug("%s", err)
|
||||
}
|
||||
|
||||
if oldest.Add(l.conf.RotationIvl).After(time.Now()) {
|
||||
err = l.rotate()
|
||||
if err != nil {
|
||||
log.Debug("%s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// What?
|
||||
time.Sleep(24 * time.Hour)
|
||||
}
|
||||
}
|
||||
|
@ -37,7 +37,7 @@ func TestQueryLog_Search_findClient(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
FindClient: findClient,
|
||||
BaseDir: t.TempDir(),
|
||||
RotationIvl: 1,
|
||||
RotationIvl: 24 * time.Hour,
|
||||
MemSize: 100,
|
||||
Enabled: true,
|
||||
FileEnabled: true,
|
||||
|
@ -4,7 +4,17 @@
|
||||
|
||||
## v0.107: API changes
|
||||
|
||||
### Client IDs in Access Settings
|
||||
### New possible value of `"interval"` field in `QueryLogConfig`
|
||||
|
||||
* The value of `"interval"` field in `POST /control/querylog_config` and `GET
|
||||
/control/querylog_info` methods could now take the value of `0.25`. It's
|
||||
equal to 6 hours.
|
||||
|
||||
* All the possible values of `"interval"` field are enumerated.
|
||||
|
||||
* The type of `"interval"` field is now `number` instead of `integer`.
|
||||
|
||||
### Client IDs in Access Settings
|
||||
|
||||
* The `POST /control/access/set` HTTP API now accepts client IDs in
|
||||
`"allowed_clients"` and `"disallowed_clients"` fields.
|
||||
|
@ -2008,8 +2008,15 @@
|
||||
'type': 'boolean'
|
||||
'description': 'Is query log enabled'
|
||||
'interval':
|
||||
'type': 'integer'
|
||||
'description': 'Time period to keep data (1 | 7 | 30 | 90)'
|
||||
'description': >
|
||||
Time period for query log rotation.
|
||||
'type': 'number'
|
||||
'enum':
|
||||
- 0.25
|
||||
- 1
|
||||
- 7
|
||||
- 30
|
||||
- 90
|
||||
'anonymize_client_ip':
|
||||
'type': 'boolean'
|
||||
'description': "Anonymize clients' IP addresses"
|
||||
|
Loading…
Reference in New Issue
Block a user