hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-01-03 17:05:16 +03:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Reduce move tool memory (#6954)

Browse Source 
Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>
This commit is contained in:
Denis Bykhov 2024-10-16 16:12:17 +05:00 committed by GitHub
parent 1d3299e384
commit 23e8729b61
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 27 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
dev/tool/src/db.ts
View File

@ -86,30 +86,33 @@ async function moveWorkspace (
console.log('move domain', domain)
const docs: Doc[] = []
while (true) {
const doc = (await cursor.next()) as Doc | null
if (doc === null) break
if (currentIds.has(doc._id)) continue
docs.push(doc)
}
while (docs.length > 0) {
const part = docs.splice(0, 500)
const vals = part
.map((doc) => {
const d = convertDoc(doc, ws.workspace)
return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
}, '${escapeBackticks(JSON.stringify(d.data))}')`
})
.join(', ')
try {
await retryTxn(pgClient, async (client) => {
await client.query(
`INSERT INTO ${translateDomain(domain)} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ${vals}`
)
})
} catch (err) {
console.log('error when move doc to', domain, err)
continue
while (docs.length < 50000) {
const doc = (await cursor.next()) as Doc | null
if (doc === null) break
if (currentIds.has(doc._id)) continue
docs.push(doc)
}
if (docs.length === 0) break
while (docs.length > 0) {
const part = docs.splice(0, 500)
const vals = part
.map((doc) => {
const d = convertDoc(doc, ws.workspace)
return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
}, '${escapeBackticks(JSON.stringify(d.data))}')`
})
.join(', ')
try {
await retryTxn(pgClient, async (client) => {
await client.query(
`INSERT INTO ${translateDomain(domain)} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ${vals}`
)
})
} catch (err) {
console.log('error when move doc to', domain, err)
continue
}
}
}
}