mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2024-08-17 17:30:32 +03:00
Add sitemap
This commit is contained in:
parent
3b3b18203f
commit
2feebf3e6a
@ -148,6 +148,7 @@
|
||||
"sequelize": "4.41.2",
|
||||
"sequelize-typescript": "0.6.6",
|
||||
"sharp": "^0.21.0",
|
||||
"sitemap": "^2.1.0",
|
||||
"srt-to-vtt": "^1.1.2",
|
||||
"summon-install": "^0.4.3",
|
||||
"useragent": "^2.3.0",
|
||||
|
@ -18,6 +18,7 @@ removeFiles () {
|
||||
|
||||
dropRedis () {
|
||||
redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
|
||||
redis-cli KEYS "redis-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
|
||||
}
|
||||
|
||||
for i in $(seq 1 6); do
|
||||
|
@ -87,7 +87,7 @@ import {
|
||||
servicesRouter,
|
||||
webfingerRouter,
|
||||
trackerRouter,
|
||||
createWebsocketServer
|
||||
createWebsocketServer, botsRouter
|
||||
} from './server/controllers'
|
||||
import { advertiseDoNotTrack } from './server/middlewares/dnt'
|
||||
import { Redis } from './server/lib/redis'
|
||||
@ -156,6 +156,7 @@ app.use('/', activityPubRouter)
|
||||
app.use('/', feedsRouter)
|
||||
app.use('/', webfingerRouter)
|
||||
app.use('/', trackerRouter)
|
||||
app.use('/', botsRouter)
|
||||
|
||||
// Static files
|
||||
app.use('/', staticRouter)
|
||||
|
101
server/controllers/bots.ts
Normal file
101
server/controllers/bots.ts
Normal file
@ -0,0 +1,101 @@
|
||||
import * as express from 'express'
|
||||
import { asyncMiddleware } from '../middlewares'
|
||||
import { CONFIG, ROUTE_CACHE_LIFETIME } from '../initializers'
|
||||
import * as sitemapModule from 'sitemap'
|
||||
import { logger } from '../helpers/logger'
|
||||
import { VideoModel } from '../models/video/video'
|
||||
import { VideoChannelModel } from '../models/video/video-channel'
|
||||
import { AccountModel } from '../models/account/account'
|
||||
import { cacheRoute } from '../middlewares/cache'
|
||||
import { buildNSFWFilter } from '../helpers/express-utils'
|
||||
import { truncate } from 'lodash'
|
||||
|
||||
const botsRouter = express.Router()
|
||||
|
||||
// Special route that add OpenGraph and oEmbed tags
|
||||
// Do not use a template engine for a so little thing
|
||||
botsRouter.use('/sitemap.xml',
|
||||
asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP)),
|
||||
asyncMiddleware(getSitemap)
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
botsRouter
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function getSitemap (req: express.Request, res: express.Response) {
|
||||
let urls = getSitemapBasicUrls()
|
||||
|
||||
urls = urls.concat(await getSitemapLocalVideoUrls())
|
||||
urls = urls.concat(await getSitemapVideoChannelUrls())
|
||||
urls = urls.concat(await getSitemapAccountUrls())
|
||||
|
||||
const sitemap = sitemapModule.createSitemap({
|
||||
hostname: CONFIG.WEBSERVER.URL,
|
||||
urls: urls
|
||||
})
|
||||
|
||||
sitemap.toXML((err, xml) => {
|
||||
if (err) {
|
||||
logger.error('Cannot generate sitemap.', { err })
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
|
||||
res.header('Content-Type', 'application/xml')
|
||||
res.send(xml)
|
||||
})
|
||||
}
|
||||
|
||||
async function getSitemapVideoChannelUrls () {
|
||||
const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
|
||||
|
||||
return rows.map(channel => ({
|
||||
url: CONFIG.WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
|
||||
}))
|
||||
}
|
||||
|
||||
async function getSitemapAccountUrls () {
|
||||
const rows = await AccountModel.listLocalsForSitemap('createdAt')
|
||||
|
||||
return rows.map(channel => ({
|
||||
url: CONFIG.WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
|
||||
}))
|
||||
}
|
||||
|
||||
async function getSitemapLocalVideoUrls () {
|
||||
const resultList = await VideoModel.listForApi({
|
||||
start: 0,
|
||||
count: undefined,
|
||||
sort: 'createdAt',
|
||||
includeLocalVideos: true,
|
||||
nsfw: buildNSFWFilter(),
|
||||
filter: 'local',
|
||||
withFiles: false
|
||||
})
|
||||
|
||||
return resultList.data.map(v => ({
|
||||
url: CONFIG.WEBSERVER.URL + '/videos/watch/' + v.uuid,
|
||||
video: [
|
||||
{
|
||||
title: v.name,
|
||||
// Sitemap description should be < 2000 characters
|
||||
description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
|
||||
player_loc: CONFIG.WEBSERVER.URL + '/videos/embed/' + v.uuid,
|
||||
thumbnail_loc: v.getThumbnailStaticPath()
|
||||
}
|
||||
]
|
||||
}))
|
||||
}
|
||||
|
||||
function getSitemapBasicUrls () {
|
||||
const paths = [
|
||||
'/about/instance',
|
||||
'/videos/local'
|
||||
]
|
||||
|
||||
return paths.map(p => ({ url: CONFIG.WEBSERVER.URL + p }))
|
||||
}
|
@ -6,3 +6,4 @@ export * from './services'
|
||||
export * from './static'
|
||||
export * from './webfinger'
|
||||
export * from './tracker'
|
||||
export * from './bots'
|
||||
|
@ -7,12 +7,12 @@ import { extname } from 'path'
|
||||
import { isArray } from './custom-validators/misc'
|
||||
import { UserModel } from '../models/account/user'
|
||||
|
||||
function buildNSFWFilter (res: express.Response, paramNSFW?: string) {
|
||||
function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
|
||||
if (paramNSFW === 'true') return true
|
||||
if (paramNSFW === 'false') return false
|
||||
if (paramNSFW === 'both') return undefined
|
||||
|
||||
if (res.locals.oauth) {
|
||||
if (res && res.locals.oauth) {
|
||||
const user: UserModel = res.locals.oauth.token.User
|
||||
|
||||
// User does not want NSFW videos
|
||||
|
@ -61,6 +61,7 @@ const OAUTH_LIFETIME = {
|
||||
const ROUTE_CACHE_LIFETIME = {
|
||||
FEEDS: '15 minutes',
|
||||
ROBOTS: '2 hours',
|
||||
SITEMAP: '1 day',
|
||||
SECURITYTXT: '2 hours',
|
||||
NODEINFO: '10 minutes',
|
||||
DNT_POLICY: '1 week',
|
||||
|
@ -241,6 +241,27 @@ export class AccountModel extends Model<AccountModel> {
|
||||
})
|
||||
}
|
||||
|
||||
static listLocalsForSitemap (sort: string) {
|
||||
const query = {
|
||||
attributes: [ ],
|
||||
offset: 0,
|
||||
order: getSort(sort),
|
||||
include: [
|
||||
{
|
||||
attributes: [ 'preferredUsername', 'serverId' ],
|
||||
model: ActorModel.unscoped(),
|
||||
where: {
|
||||
serverId: null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
return AccountModel
|
||||
.unscoped()
|
||||
.findAll(query)
|
||||
}
|
||||
|
||||
toFormattedJSON (): Account {
|
||||
const actor = this.Actor.toFormattedJSON()
|
||||
const account = {
|
||||
|
@ -233,6 +233,27 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
|
||||
})
|
||||
}
|
||||
|
||||
static listLocalsForSitemap (sort: string) {
|
||||
const query = {
|
||||
attributes: [ ],
|
||||
offset: 0,
|
||||
order: getSort(sort),
|
||||
include: [
|
||||
{
|
||||
attributes: [ 'preferredUsername', 'serverId' ],
|
||||
model: ActorModel.unscoped(),
|
||||
where: {
|
||||
serverId: null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
return VideoChannelModel
|
||||
.unscoped()
|
||||
.findAll(query)
|
||||
}
|
||||
|
||||
static searchForApi (options: {
|
||||
actorId: number
|
||||
search: string
|
||||
|
@ -2,7 +2,18 @@
|
||||
|
||||
import 'mocha'
|
||||
import * as chai from 'chai'
|
||||
import { flushTests, killallServers, makeGetRequest, runServer, ServerInfo } from './utils'
|
||||
import {
|
||||
addVideoChannel,
|
||||
createUser,
|
||||
flushTests,
|
||||
killallServers,
|
||||
makeGetRequest,
|
||||
runServer,
|
||||
ServerInfo,
|
||||
setAccessTokensToServers,
|
||||
uploadVideo
|
||||
} from './utils'
|
||||
import { VideoPrivacy } from '../../shared/models/videos'
|
||||
|
||||
const expect = chai.expect
|
||||
|
||||
@ -15,6 +26,7 @@ describe('Test misc endpoints', function () {
|
||||
await flushTests()
|
||||
|
||||
server = await runServer(1)
|
||||
await setAccessTokensToServers([ server ])
|
||||
})
|
||||
|
||||
describe('Test a well known endpoints', function () {
|
||||
@ -93,6 +105,64 @@ describe('Test misc endpoints', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('Test bots endpoints', function () {
|
||||
|
||||
it('Should get the empty sitemap', async function () {
|
||||
const res = await makeGetRequest({
|
||||
url: server.url,
|
||||
path: '/sitemap.xml',
|
||||
statusCodeExpected: 200
|
||||
})
|
||||
|
||||
expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
|
||||
expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
|
||||
})
|
||||
|
||||
it('Should get the empty cached sitemap', async function () {
|
||||
const res = await makeGetRequest({
|
||||
url: server.url,
|
||||
path: '/sitemap.xml',
|
||||
statusCodeExpected: 200
|
||||
})
|
||||
|
||||
expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
|
||||
expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
|
||||
})
|
||||
|
||||
it('Should add videos, channel and accounts and get sitemap', async function () {
|
||||
this.timeout(35000)
|
||||
|
||||
await uploadVideo(server.url, server.accessToken, { name: 'video 1', nsfw: false })
|
||||
await uploadVideo(server.url, server.accessToken, { name: 'video 2', nsfw: false })
|
||||
await uploadVideo(server.url, server.accessToken, { name: 'video 3', privacy: VideoPrivacy.PRIVATE })
|
||||
|
||||
await addVideoChannel(server.url, server.accessToken, { name: 'channel1', displayName: 'channel 1' })
|
||||
await addVideoChannel(server.url, server.accessToken, { name: 'channel2', displayName: 'channel 2' })
|
||||
|
||||
await createUser(server.url, server.accessToken, 'user1', 'password')
|
||||
await createUser(server.url, server.accessToken, 'user2', 'password')
|
||||
|
||||
const res = await makeGetRequest({
|
||||
url: server.url,
|
||||
path: '/sitemap.xml?t=1', // avoid using cache
|
||||
statusCodeExpected: 200
|
||||
})
|
||||
|
||||
expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
|
||||
expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
|
||||
|
||||
expect(res.text).to.contain('<video:title><![CDATA[video 1]]></video:title>')
|
||||
expect(res.text).to.contain('<video:title><![CDATA[video 2]]></video:title>')
|
||||
expect(res.text).to.not.contain('<video:title><![CDATA[video 3]]></video:title>')
|
||||
|
||||
expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel1</loc></url>')
|
||||
expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel2</loc></url>')
|
||||
|
||||
expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user1</loc></url>')
|
||||
expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user2</loc></url>')
|
||||
})
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
killallServers([ server ])
|
||||
})
|
||||
|
19
yarn.lock
19
yarn.lock
@ -7457,6 +7457,15 @@ simple-websocket@^7.0.1:
|
||||
readable-stream "^2.0.5"
|
||||
ws "^6.0.0"
|
||||
|
||||
sitemap@^2.1.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-2.1.0.tgz#1633cb88c196d755ad94becfb1c1bcacc6d3425a"
|
||||
integrity sha512-AkfA7RDVCITQo+j5CpXsMJlZ/8ENO2NtgMHYIh+YMvex2Hao/oe3MQgNa03p0aWY6srCfUA1Q02OgiWCAiuccA==
|
||||
dependencies:
|
||||
lodash "^4.17.10"
|
||||
url-join "^4.0.0"
|
||||
xmlbuilder "^10.0.0"
|
||||
|
||||
slash@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"
|
||||
@ -8592,6 +8601,11 @@ urix@^0.1.0:
|
||||
resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
|
||||
integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
|
||||
|
||||
url-join@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.0.tgz#4d3340e807d3773bda9991f8305acdcc2a665d2a"
|
||||
integrity sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo=
|
||||
|
||||
url-parse-lax@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73"
|
||||
@ -9001,6 +9015,11 @@ xml@^1.0.1:
|
||||
resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
|
||||
integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
|
||||
|
||||
xmlbuilder@^10.0.0:
|
||||
version "10.1.1"
|
||||
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0"
|
||||
integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==
|
||||
|
||||
xmlbuilder@~9.0.1:
|
||||
version "9.0.7"
|
||||
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
|
||||
|
Loading…
Reference in New Issue
Block a user