Serve audit logs to client

This commit is contained in:
Chocobozzz 2019-12-11 14:14:01 +01:00
parent 92e0f42e8c
commit 566c125d6e
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
18 changed files with 287 additions and 87 deletions

View File

@ -15,7 +15,7 @@
<ng-template pTemplate="body" let-expanded="expanded" let-videoAbuse>
<tr>
<td>
<td class="expand-cell">
<span class="expander" [pRowToggler]="videoAbuse">
<i [ngClass]="expanded ? 'glyphicon glyphicon-menu-down' : 'glyphicon glyphicon-menu-right'"></i>
</span>

View File

@ -15,7 +15,7 @@
<ng-template pTemplate="body" let-videoBlacklist let-expanded="expanded">
<tr>
<td>
<td class="expand-cell">
<span *ngIf="videoBlacklist.reason" class="expander" [pRowToggler]="videoBlacklist">
<i [ngClass]="expanded ? 'glyphicon glyphicon-menu-down' : 'glyphicon glyphicon-menu-right'"></i>
</span>

View File

@ -38,7 +38,7 @@
<ng-template pTemplate="body" let-expanded="expanded" let-job>
<tr>
<td>
<td class="expand-cell">
<span class="expander" [pRowToggler]="job">
<i [ngClass]="expanded ? 'glyphicon glyphicon-menu-down' : 'glyphicon glyphicon-menu-right'"></i>
</span>

View File

@ -8,6 +8,10 @@ export class LogRow {
message: string
meta: string
by: string
domain: string
action: string
constructor (row: any) {
this.date = new Date(row.timestamp)
this.localeDate = this.date.toLocaleString()
@ -17,5 +21,20 @@ export class LogRow {
const metaObj = omit(row, 'timestamp', 'level', 'message', 'label')
if (Object.keys(metaObj).length !== 0) this.meta = JSON.stringify(metaObj, undefined, 2)
if (row.level === 'audit') {
try {
const message = JSON.parse(row.message)
this.by = message.user
this.domain = message.domain
this.action = message.action
this.meta = JSON.stringify(message, null, 2)
this.message = ''
} catch (err) {
console.error('Cannot parse audit message.', err)
}
}
}
}

View File

@ -1,11 +1,17 @@
<div class="header">
<div class="peertube-select-container">
<select [(ngModel)]="logType" (ngModelChange)="refresh()">
<option *ngFor="let logTypeChoice of logTypeChoices" [value]="logTypeChoice.id">{{ logTypeChoice.label }}</option>
</select>
</div>
<div class="peertube-select-container">
<select [(ngModel)]="startDate" (ngModelChange)="refresh()">
<option *ngFor="let timeChoice of timeChoices" [value]="timeChoice.id">{{ timeChoice.label }}</option>
</select>
</div>
<div class="peertube-select-container">
<div class="peertube-select-container" *ngIf="!isAuditLog()">
<select [(ngModel)]="level" (ngModelChange)="refresh()">
<option *ngFor="let levelChoice of levelChoices" [value]="levelChoice.id">{{ levelChoice.label }}</option>
</select>
@ -23,9 +29,12 @@
<span class="log-date">[{{ log.localeDate }}]</span>
<strong class="log-by" *ngIf="log.by" i18n>By {{ log.by }} -></strong>
<strong class="log-domain-action" *ngIf="log.domain">{{ log.domain }} -> {{ log.action }}</strong>
{{ log.message }}
{{ log.meta }}
<pre>{{ log.meta }}</pre>
</div>
</div>
</div>

View File

@ -23,6 +23,10 @@
margin-right: 5px;
}
.log-by {
margin: 0 5px;
}
.warn {
color: $orange-color;
}
@ -30,6 +34,12 @@
.error {
color: $red;
}
pre {
margin-bottom: 5px;
white-space: pre-wrap;
word-wrap: break-word;
}
}
.header {

View File

@ -17,9 +17,11 @@ export class LogsComponent implements OnInit {
logs: LogRow[] = []
timeChoices: { id: string, label: string }[] = []
levelChoices: { id: LogLevel, label: string }[] = []
logTypeChoices: { id: 'audit' | 'standard', label: string }[] = []
startDate: string
level: LogLevel
logType: 'audit' | 'standard'
constructor (
private logsService: LogsService,
@ -30,6 +32,7 @@ export class LogsComponent implements OnInit {
ngOnInit (): void {
this.buildTimeChoices()
this.buildLevelChoices()
this.buildLogTypeChoices()
this.load()
}
@ -42,7 +45,7 @@ export class LogsComponent implements OnInit {
load () {
this.loading = true
this.logsService.getLogs(this.level, this.startDate)
this.logsService.getLogs({ isAuditLog: this.isAuditLog(), level: this.level, startDate: this.startDate })
.subscribe(
logs => {
this.logs = logs
@ -58,6 +61,10 @@ export class LogsComponent implements OnInit {
)
}
isAuditLog () {
return this.logType === 'audit'
}
buildTimeChoices () {
const lastHour = new Date()
lastHour.setHours(lastHour.getHours() - 1)
@ -108,4 +115,19 @@ export class LogsComponent implements OnInit {
this.level = 'warn'
}
buildLogTypeChoices () {
this.logTypeChoices = [
{
id: 'standard',
label: this.i18n('Standard logs')
},
{
id: 'audit',
label: this.i18n('Audit logs')
}
]
this.logType = 'audit'
}
}

View File

@ -10,6 +10,7 @@ import { LogLevel } from '@shared/models/server/log-level.type'
@Injectable()
export class LogsService {
private static BASE_LOG_URL = environment.apiUrl + '/api/v1/server/logs'
private static BASE_AUDIT_LOG_URL = environment.apiUrl + '/api/v1/server/audit-logs'
constructor (
private authHttp: HttpClient,
@ -17,14 +18,25 @@ export class LogsService {
private restExtractor: RestExtractor
) {}
getLogs (level: LogLevel, startDate: string, endDate?: string): Observable<any[]> {
getLogs (options: {
isAuditLog: boolean,
startDate: string,
level?: LogLevel,
endDate?: string
}): Observable<any[]> {
const { isAuditLog, startDate } = options
let params = new HttpParams()
params = params.append('startDate', startDate)
params = params.append('level', level)
if (endDate) params.append('endDate', endDate)
if (!isAuditLog) params = params.append('level', options.level)
if (options.endDate) params.append('endDate', options.endDate)
return this.authHttp.get<any[]>(LogsService.BASE_LOG_URL, { params })
const path = isAuditLog
? LogsService.BASE_AUDIT_LOG_URL
: LogsService.BASE_LOG_URL
return this.authHttp.get<any[]>(path, { params })
.pipe(
map(rows => rows.map(r => new LogRow(r))),
catchError(err => this.restExtractor.handleError(err))

View File

@ -49,7 +49,7 @@
<ng-template pTemplate="body" let-expanded="expanded" let-user>
<tr [pSelectableRow]="user" [ngClass]="{ banned: user.blocked }">
<td>
<td class="expand-cell">
<p-tableCheckbox [value]="user"></p-tableCheckbox>
</td>

View File

@ -37,7 +37,7 @@ p-table {
td {
padding-left: 15px !important;
&:not(.action-cell) {
&:not(.action-cell):not(.expand-cell) {
overflow: hidden !important;
text-overflow: ellipsis !important;
white-space: nowrap !important;

View File

@ -3,11 +3,12 @@ import { UserRight } from '../../../../shared/models/users'
import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
import { mtimeSortFilesDesc } from '../../../../shared/core-utils/logs/logs'
import { readdir, readFile } from 'fs-extra'
import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants'
import { AUDIT_LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS, LOG_FILENAME } from '../../../initializers/constants'
import { join } from 'path'
import { getLogsValidator } from '../../../middlewares/validators/logs'
import { getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs'
import { LogLevel } from '../../../../shared/models/server/log-level.type'
import { CONFIG } from '../../../initializers/config'
import { logger } from '@server/helpers/logger'
const logsRouter = express.Router()
@ -18,6 +19,13 @@ logsRouter.get('/logs',
asyncMiddleware(getLogs)
)
logsRouter.get('/audit-logs',
authenticate,
ensureUserHasRight(UserRight.MANAGE_LOGS),
getAuditLogsValidator,
asyncMiddleware(getAuditLogs)
)
// ---------------------------------------------------------------------------
export {
@ -26,18 +34,50 @@ export {
// ---------------------------------------------------------------------------
const auditLogNameFilter = generateLogNameFilter(AUDIT_LOG_FILENAME)
async function getAuditLogs (req: express.Request, res: express.Response) {
const output = await generateOutput({
startDateQuery: req.query.startDate,
endDateQuery: req.query.endDate,
level: 'audit',
nameFilter: auditLogNameFilter
})
return res.json(output).end()
}
const logNameFilter = generateLogNameFilter(LOG_FILENAME)
async function getLogs (req: express.Request, res: express.Response) {
const output = await generateOutput({
startDateQuery: req.query.startDate,
endDateQuery: req.query.endDate,
level: req.query.level || 'info',
nameFilter: logNameFilter
})
return res.json(output).end()
}
async function generateOutput (options: {
startDateQuery: string,
endDateQuery?: string,
level: LogLevel,
nameFilter: RegExp
}) {
const { startDateQuery, level, nameFilter } = options
const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR)
let currentSize = 0
const startDate = new Date(req.query.startDate)
const endDate = req.query.endDate ? new Date(req.query.endDate) : new Date()
const level: LogLevel = req.query.level || 'info'
const startDate = new Date(startDateQuery)
const endDate = options.endDateQuery ? new Date(options.endDateQuery) : new Date()
let output: string[] = []
for (const meta of sortedLogFiles) {
if (nameFilter.exec(meta.file) === null) continue
const path = join(CONFIG.STORAGE.LOG_DIR, meta.file)
const result = await getOutputFromFile(path, startDate, endDate, level, currentSize)
@ -49,7 +89,7 @@ async function getLogs (req: express.Request, res: express.Response) {
if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS || (result.logTime && result.logTime < startDate.getTime())) break
}
return res.json(output).end()
return output
}
async function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) {
@ -58,6 +98,7 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date,
let logTime: number
const logsLevel: { [ id in LogLevel ]: number } = {
audit: -1,
debug: 0,
info: 1,
warn: 2,
@ -93,3 +134,7 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date,
return { currentSize, output: output.reverse(), logTime }
}
function generateLogNameFilter (baseName: string) {
return new RegExp('^' + baseName.replace(/\.log$/, '') + '\d*.log$')
}

View File

@ -9,6 +9,7 @@ import { User, VideoAbuse, VideoChannel, VideoDetails, VideoImport } from '../..
import { VideoComment } from '../../shared/models/videos/video-comment.model'
import { CustomConfig } from '../../shared/models/server/custom-config.model'
import { CONFIG } from '../initializers/config'
import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
function getAuditIdFromRes (res: express.Response) {
return res.locals.oauth.token.User.username
@ -29,7 +30,7 @@ const auditLogger = winston.createLogger({
levels: { audit: 0 },
transports: [
new winston.transports.File({
filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube-audit.log'),
filename: path.join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME),
level: 'audit',
maxsize: 5242880,
maxFiles: 5,

View File

@ -5,6 +5,7 @@ import * as winston from 'winston'
import { FileTransportOptions } from 'winston/lib/winston/transports'
import { CONFIG } from '../initializers/config'
import { omit } from 'lodash'
import { LOG_FILENAME } from '@server/initializers/constants'
const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@ -58,7 +59,7 @@ const labelFormatter = winston.format.label({
})
const fileLoggerOptions: FileTransportOptions = {
filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'),
filename: path.join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME),
handleExceptions: true,
format: winston.format.combine(
winston.format.timestamp(),

View File

@ -603,6 +603,8 @@ const FEEDS = {
}
const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000
const LOG_FILENAME = 'peertube.log'
const AUDIT_LOG_FILENAME = 'peertube-audit.log'
// ---------------------------------------------------------------------------
@ -684,6 +686,7 @@ export {
BCRYPT_SALT_SIZE,
TRACKER_RATE_LIMITS,
FILES_CACHE,
LOG_FILENAME,
CONSTRAINTS_FIELDS,
EMBED_SIZE,
REDUNDANCY,
@ -693,6 +696,7 @@ export {
OAUTH_LIFETIME,
CUSTOM_HTML_TAG_COMMENTS,
BROADCAST_CONCURRENCY,
AUDIT_LOG_FILENAME,
PAGINATION,
ACTOR_FOLLOW_SCORE,
PREVIEWS_SIZE,

View File

@ -24,8 +24,25 @@ const getLogsValidator = [
}
]
const getAuditLogsValidator = [
query('startDate')
.custom(isDateValid).withMessage('Should have a valid start date'),
query('endDate')
.optional()
.custom(isDateValid).withMessage('Should have a valid end date'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking getAuditLogsValidator parameters.', { parameters: req.query })
if (areValidationErrors(req, res)) return
return next()
}
]
// ---------------------------------------------------------------------------
export {
getLogsValidator
getLogsValidator,
getAuditLogsValidator
}

View File

@ -2,17 +2,10 @@
import * as chai from 'chai'
import 'mocha'
import {
flushTests,
killallServers,
flushAndRunServer,
ServerInfo,
setAccessTokensToServers,
cleanupTests
} from '../../../../shared/extra-utils/index'
import { cleanupTests, flushAndRunServer, ServerInfo, setAccessTokensToServers } from '../../../../shared/extra-utils/index'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
import { uploadVideo } from '../../../../shared/extra-utils/videos/videos'
import { getLogs } from '../../../../shared/extra-utils/logs/logs'
import { getAuditLogs, getLogs } from '../../../../shared/extra-utils/logs/logs'
const expect = chai.expect
@ -26,69 +19,123 @@ describe('Test logs', function () {
await setAccessTokensToServers([ server ])
})
it('Should get logs with a start date', async function () {
this.timeout(10000)
describe('With the standard log file', function () {
it('Should get logs with a start date', async function () {
this.timeout(10000)
await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
await waitJobs([ server ])
await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
await waitJobs([ server ])
const now = new Date()
const now = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
await waitJobs([ server ])
await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
await waitJobs([ server ])
const res = await getLogs(server.url, server.accessToken, now)
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 1')).to.be.false
expect(logsString.includes('video 2')).to.be.true
})
it('Should get logs with an end date', async function () {
this.timeout(20000)
await uploadVideo(server.url, server.accessToken, { name: 'video 3' })
await waitJobs([ server ])
const now1 = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 4' })
await waitJobs([ server ])
const now2 = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 5' })
await waitJobs([ server ])
const res = await getLogs(server.url, server.accessToken, now1, now2)
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 3')).to.be.false
expect(logsString.includes('video 4')).to.be.true
expect(logsString.includes('video 5')).to.be.false
})
it('Should get filter by level', async function () {
this.timeout(10000)
const now = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 6' })
await waitJobs([ server ])
{
const res = await getLogs(server.url, server.accessToken, now, undefined, 'info')
const res = await getLogs(server.url, server.accessToken, now)
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 6')).to.be.true
}
expect(logsString.includes('video 1')).to.be.false
expect(logsString.includes('video 2')).to.be.true
})
{
const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn')
it('Should get logs with an end date', async function () {
this.timeout(20000)
await uploadVideo(server.url, server.accessToken, { name: 'video 3' })
await waitJobs([ server ])
const now1 = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 4' })
await waitJobs([ server ])
const now2 = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 5' })
await waitJobs([ server ])
const res = await getLogs(server.url, server.accessToken, now1, now2)
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 6')).to.be.false
}
expect(logsString.includes('video 3')).to.be.false
expect(logsString.includes('video 4')).to.be.true
expect(logsString.includes('video 5')).to.be.false
})
it('Should get filter by level', async function () {
this.timeout(10000)
const now = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 6' })
await waitJobs([ server ])
{
const res = await getLogs(server.url, server.accessToken, now, undefined, 'info')
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 6')).to.be.true
}
{
const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn')
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 6')).to.be.false
}
})
})
describe('With the audit log', function () {
it('Should get logs with a start date', async function () {
this.timeout(10000)
await uploadVideo(server.url, server.accessToken, { name: 'video 7' })
await waitJobs([ server ])
const now = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 8' })
await waitJobs([ server ])
const res = await getAuditLogs(server.url, server.accessToken, now)
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 7')).to.be.false
expect(logsString.includes('video 8')).to.be.true
expect(res.body).to.have.lengthOf(1)
const item = res.body[0]
const message = JSON.parse(item.message)
expect(message.domain).to.equal('videos')
expect(message.action).to.equal('create')
})
it('Should get logs with an end date', async function () {
this.timeout(20000)
await uploadVideo(server.url, server.accessToken, { name: 'video 9' })
await waitJobs([ server ])
const now1 = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 10' })
await waitJobs([ server ])
const now2 = new Date()
await uploadVideo(server.url, server.accessToken, { name: 'video 11' })
await waitJobs([ server ])
const res = await getAuditLogs(server.url, server.accessToken, now1, now2)
const logsString = JSON.stringify(res.body)
expect(logsString.includes('video 9')).to.be.false
expect(logsString.includes('video 10')).to.be.true
expect(logsString.includes('video 11')).to.be.false
})
})
after(async function () {

View File

@ -13,6 +13,19 @@ function getLogs (url: string, accessToken: string, startDate: Date, endDate?: D
})
}
export {
getLogs
function getAuditLogs (url: string, accessToken: string, startDate: Date, endDate?: Date) {
const path = '/api/v1/server/audit-logs'
return makeGetRequest({
url,
path,
token: accessToken,
query: { startDate, endDate },
statusCodeExpected: 200
})
}
export {
getLogs,
getAuditLogs
}

View File

@ -1 +1 @@
export type LogLevel = 'debug' | 'info' | 'warn' | 'error'
export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'audit'