From b522334f0e33dafadadba4356910998526a17d6d Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 25 Dec 2023 19:11:08 +0700 Subject: [PATCH] UBERF-4729: Fix front service (#4260) Signed-off-by: Andrey Sobolev --- common/config/rush/pnpm-lock.yaml | 26 +++++++++- server/backup/src/index.ts | 4 ++ server/core/src/indexer/content.ts | 1 + server/front/package.json | 6 ++- server/front/src/index.ts | 47 +++++++++++++------ server/minio/src/index.ts | 2 + tests/sanity/storage.json | 4 ++ tests/sanity/storageSecond-dev.json | 4 ++ .../sanity/tests/model/tracker/issues-page.ts | 2 +- 9 files changed, 76 insertions(+), 20 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index c308b140be..104b786024 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1022,6 +1022,9 @@ dependencies: mongodb: specifier: ^4.11.0 version: 4.17.1 + morgan: + specifier: ^1.10.0 + version: 1.10.0 msgpackr: specifier: ^1.8.5 version: 1.9.9 @@ -7259,6 +7262,12 @@ packages: '@types/node': 16.11.68 dev: false + /@types/morgan@1.9.9: + resolution: {integrity: sha512-iRYSDKVaC6FkGSpEVVIvrRGw0DfJMiQzIn3qr2G5B3C//AWkulhXgaBd7tS9/J79GWSYMTHGs7PfI5b3Y8m+RQ==} + dependencies: + '@types/node': 16.11.68 + dev: false + /@types/node-fetch@2.6.6: resolution: {integrity: sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==} dependencies: @@ -13356,6 +13365,17 @@ packages: resolution: {integrity: sha512-YRIr1exCIfBDLZle8WHOfSo7Xg3M+phcZfq9Fx1L6Abo+atGp7cge5pM7PjyBn4s1oZI/BRD4EMrzQBbPpVb5Q==} dev: false + /morgan@1.10.0: + resolution: {integrity: sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==} + engines: {node: '>= 0.8.0'} + dependencies: + basic-auth: 2.0.1 + debug: 2.6.9 + depd: 2.0.0 + on-finished: 2.3.0 + on-headers: 1.0.2 + dev: false + /mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -18613,7 +18633,7 @@ packages: dev: false file:projects/front.tgz(esbuild@0.16.17)(svelte@4.2.5): - resolution: {integrity: sha512-xBechy2SGLSNUUN/rM60kyFJOJA8edlX2f/gqHk272BIfcsGXtUlPl8yPXv/H9zciOKSAUSy/6fMDZVNATVuqA==, tarball: file:projects/front.tgz} + resolution: {integrity: sha512-G4bpLQ5WO+t2ZP5hbCRzS4Zpzi1bwLhpC0hdhLTNkwFw6GUj38chkzMtzroD/cExV30MrjtntL5yQJ2n3yvv2A==, tarball: file:projects/front.tgz} id: file:projects/front.tgz name: '@rush-temp/front' version: 0.0.0 @@ -18624,6 +18644,7 @@ packages: '@types/express': 4.17.18 '@types/express-fileupload': 1.4.2 '@types/jest': 29.5.5 + '@types/morgan': 1.9.9 '@types/node': 16.11.68 '@types/sharp': 0.32.0 '@types/uuid': 8.3.4 @@ -18641,6 +18662,7 @@ packages: express: 4.18.2 express-fileupload: 1.4.1 jest: 29.7.0(@types/node@16.11.68)(ts-node@10.9.1) + morgan: 1.10.0 prettier: 3.1.0 prettier-plugin-svelte: 3.1.0(prettier@3.1.0)(svelte@4.2.5) sharp: 0.32.6 @@ -23558,7 +23580,7 @@ packages: dev: false file:projects/telegram-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(ts-node@10.9.1): - resolution: {integrity: sha512-qrqvq8PmtA09Sn9AOCaId5Zupe804Fw4t6pRfrDPlmYm39aemhZ+QHAyUIvg2kr/yo0jNRwXg4g3DBpUbUTRhA==, tarball: file:projects/telegram-resources.tgz} + resolution: {integrity: sha512-xz5vevvHFXT/WIU4CCz9TPJUwGOgAUtiEgcJFYsWwzFUUL5Mz2ImUG0GkV9wQeAfkjaz0KiKD6YWe78Ir9WD5w==, tarball: file:projects/telegram-resources.tgz} id: file:projects/telegram-resources.tgz name: '@rush-temp/telegram-resources' version: 0.0.0 diff --git a/server/backup/src/index.ts b/server/backup/src/index.ts index 3f128e55ca..e016566dad 100644 --- a/server/backup/src/index.ts +++ b/server/backup/src/index.ts @@ -798,6 +798,10 @@ export async function restore ( }) }) const unzip = createGunzip() + + readStream.on('end', () => { + readStream.destroy() + }) readStream.pipe(unzip) unzip.pipe(ex) diff --git a/server/core/src/indexer/content.ts b/server/core/src/indexer/content.ts index bf64231997..0f374c5f50 100644 --- a/server/core/src/indexer/content.ts +++ b/server/core/src/indexer/content.ts @@ -118,6 +118,7 @@ export class ContentRetrievalStage implements FullTextPipelineStage { {}, async () => await this.contentAdapter.content(ref, contentType, readable) ) + readable?.destroy() textContent = textContent .split(/ +|\t+|\f+/) diff --git a/server/front/package.json b/server/front/package.json index 7ac9b4ff86..cdf3cd559e 100644 --- a/server/front/package.json +++ b/server/front/package.json @@ -36,7 +36,8 @@ "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", - "prettier-plugin-svelte": "^3.1.0" + "prettier-plugin-svelte": "^3.1.0", + "@types/morgan": "~1.9.9" }, "dependencies": { "@hcengineering/core": "^0.6.28", @@ -52,6 +53,7 @@ "body-parser": "~1.19.1", "compression": "~1.7.4", "sharp": "~0.32.0", - "@hcengineering/minio": "^0.6.0" + "@hcengineering/minio": "^0.6.0", + "morgan": "^1.10.0" } } diff --git a/server/front/src/index.ts b/server/front/src/index.ts index 0db06c5587..ec1ce187bc 100644 --- a/server/front/src/index.ts +++ b/server/front/src/index.ts @@ -26,6 +26,7 @@ import https from 'https' import { join, resolve } from 'path' import sharp from 'sharp' import { v4 as uuid } from 'uuid' +import morgan from 'morgan' async function minioUpload (minio: MinioService, workspace: WorkspaceId, file: UploadedFile): Promise { const id = uuid() @@ -85,7 +86,14 @@ async function getFileRange ( 'Content-Range': `bytes ${start}-${end}/${size}`, 'Accept-Ranges': 'bytes', 'Content-Length': end - start + 1, - 'Content-Type': stat.metaData['content-type'] + 'Content-Type': stat.metaData['content-type'], + Etag: stat.etag, + 'Last-Modified': stat.lastModified.toISOString() + }) + + dataStream.on('end', () => { + dataStream.destroy() + res.end() }) dataStream.pipe(res) @@ -100,19 +108,18 @@ async function getFile (client: MinioService, workspace: WorkspaceId, uuid: stri try { const dataStream = await client.get(workspace, uuid) - res.status(200) - res.set('Cache-Control', 'max-age=7d') - - const contentType = stat.metaData['content-type'] - if (contentType !== undefined) { - res.setHeader('Content-Type', contentType) - } + res.writeHead(200, { + 'Content-Type': stat.metaData['content-type'], + Etag: stat.etag, + 'Last-Modified': stat.lastModified.toISOString() + }) dataStream.on('data', function (chunk) { res.write(chunk) }) dataStream.on('end', function () { res.end() + dataStream.destroy() }) dataStream.on('error', function (err) { console.log(err) @@ -169,6 +176,8 @@ export function start ( app.use(bp.json()) app.use(bp.urlencoded({ extended: true })) + app.use(morgan('combined')) + // eslint-disable-next-line @typescript-eslint/no-misused-promises app.get('/config.json', async (req, res) => { res.status(200) @@ -215,11 +224,14 @@ export function start ( const fileSize = stat.size + res.writeHead(200, { + 'accept-ranges': 'bytes', + 'content-length': fileSize, + Etag: stat.etag, + 'Last-Modified': stat.lastModified.toISOString() + }) res.status(200) - res.setHeader('accept-ranges', 'bytes') - res.setHeader('content-length', fileSize) - res.end() } catch (error) { console.log(error) @@ -229,7 +241,6 @@ export function start ( const filesHandler = async (req: any, res: Response): Promise => { try { - console.log(req.headers) const cookies = ((req?.headers?.cookie as string) ?? '').split(';').map((it) => it.trim().split('=')) const token = cookies.find((it) => it[0] === 'presentation-metadata-Token')?.[1] @@ -262,8 +273,12 @@ export function start ( } else { await getFile(config.minio, payload.workspace, uuid, res) } - } catch (error) { - console.log(error) + } catch (error: any) { + if (error?.code === 'NoSuchKey' || error?.code === 'NotFound') { + console.log('No such key', req.query.file) + } else { + console.log(error) + } res.status(500).send() } } @@ -565,7 +580,9 @@ async function getResizeID ( const d = await config.minio.stat(payload.workspace, sizeId) hasSmall = d !== undefined && d.size > 0 } catch (err: any) { - console.error(err) + if (err.code !== 'NotFound') { + console.error(err) + } } if (hasSmall) { // We have cached small document, let's proceed with it. diff --git a/server/minio/src/index.ts b/server/minio/src/index.ts index af3e1c5461..3f33e4ec03 100644 --- a/server/minio/src/index.ts +++ b/server/minio/src/index.ts @@ -67,6 +67,7 @@ export class MinioService { } }) list.on('end', () => { + list.destroy() resolve(null) }) }) @@ -111,6 +112,7 @@ export class MinioService { }) data.on('end', () => { + data.destroy() resolve(null) }) }) diff --git a/tests/sanity/storage.json b/tests/sanity/storage.json index 93a58d3b93..1d332bbaa0 100644 --- a/tests/sanity/storage.json +++ b/tests/sanity/storage.json @@ -20,6 +20,10 @@ "name": "#platform.notification.logging", "value": "false" }, + { + "name": "#platform.notification.timeout", + "value": "0" + }, { "name": "#platform.lazy.loading", "value": "false" diff --git a/tests/sanity/storageSecond-dev.json b/tests/sanity/storageSecond-dev.json index 45daf4e950..36cfa821bc 100644 --- a/tests/sanity/storageSecond-dev.json +++ b/tests/sanity/storageSecond-dev.json @@ -20,6 +20,10 @@ "name": "#platform.notification.timeout", "value": "0" }, + { + "name": "#platform.notification.timeout", + "value": "0" + }, { "name": "#platform.testing.enabled", "value": "true" diff --git a/tests/sanity/tests/model/tracker/issues-page.ts b/tests/sanity/tests/model/tracker/issues-page.ts index 35e87f7eb5..1d76c33422 100644 --- a/tests/sanity/tests/model/tracker/issues-page.ts +++ b/tests/sanity/tests/model/tracker/issues-page.ts @@ -74,7 +74,7 @@ export class IssuesPage extends CommonTrackerPage { ) } - async createNewIssue (data: NewIssue, closeNotification: boolean = true): Promise { + async createNewIssue (data: NewIssue, closeNotification: boolean = false): Promise { await this.buttonCreateNewIssue.click() await this.fillNewIssueForm(data) await this.buttonCreateIssue.click()