Reusable Platform (#2374)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2022-11-15 09:51:46 +07:00 committed by GitHub
parent cd9c2ec928
commit 5326acaa79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
69 changed files with 6622 additions and 5225 deletions

View File

@ -82,6 +82,7 @@ jobs:
tests
rush.json
.prettierrc
tools
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
restore-keys: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
svelte-check:
@ -113,6 +114,7 @@ jobs:
tests
rush.json
.prettierrc
tools
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
restore-keys: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
@ -146,6 +148,7 @@ jobs:
tests
rush.json
.prettierrc
tools
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
restore-keys: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
@ -190,6 +193,7 @@ jobs:
tests
rush.json
.prettierrc
tools
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
restore-keys: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
@ -236,6 +240,7 @@ jobs:
tests
rush.json
.prettierrc
tools
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
restore-keys: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
@ -293,6 +298,7 @@ jobs:
tests
rush.json
.prettierrc
tools
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
restore-keys: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}

3
.gitignore vendored
View File

@ -76,4 +76,5 @@ dist
dist_cache
tsconfig.tsbuildinfo
ingest-attachment-*.zip
tsdoc-metadata.json
tsdoc-metadata.json
pods/front/dist

File diff suppressed because it is too large Load Diff

View File

@ -23,7 +23,7 @@
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.26.0",
"ts-node": "^10.8.0",
"esbuild": "^0.12.26",
"esbuild": "^0.15.13",
"@types/node": "~16.11.12",
"@typescript-eslint/parser": "^5.41.0",
"eslint-config-standard-with-typescript": "^23.0.0",

View File

@ -8,13 +8,13 @@
"build": "heft build",
"build:watch": "tsc",
"lint:fix": "eslint --fix src",
"start": "ts-node src/index.ts",
"bundle": "esbuild src/index.ts --bundle --minify --platform=node > bundle.js",
"start": "ts-node src/__start.ts",
"bundle": "esbuild src/__start.ts --bundle --minify --platform=node > bundle.js",
"docker:build": "docker build -t hardcoreeng/tool .",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging",
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool",
"run-local": "cross-env SERVER_SECRET=secret MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TRANSACTOR_URL=ws:/localhost:3333 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 ts-node ./src/index.ts",
"run": "cross-env ts-node ./src/index.ts",
"run-local": "cross-env SERVER_SECRET=secret MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TRANSACTOR_URL=ws:/localhost:3333 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 ts-node ./src/__start.ts",
"run": "cross-env ts-node ./src/__start.ts",
"upgrade": "rushx run-local upgrade",
"lint": "eslint src",
"format": "prettier --write src && eslint --fix src"
@ -29,7 +29,7 @@
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.26.0",
"ts-node": "^10.8.0",
"esbuild": "^0.12.26",
"esbuild": "^0.15.13",
"@types/minio": "~7.0.11",
"@types/node": "~16.11.12",
"@typescript-eslint/parser": "^5.41.0",

View File

@ -14,16 +14,23 @@
# limitations under the License.
#
export MONGO_URL=$(kubectl get secret mongodb -o jsonpath="{.data.url}" | base64 --decode)
export MINIO_ENDPOINT=$(kubectl get secret minio -o jsonpath="{.data.endpoint}" | base64 --decode)
export MINIO_ACCESS_KEY=$(kubectl get secret minio -o jsonpath="{.data.accessKey}" | base64 --decode)
export MINIO_SECRET_KEY=$(kubectl get secret minio -o jsonpath="{.data.secretKey}" | base64 --decode)
export MONGO_URL=$(kubectl get configmaps anticrm-config -o jsonpath="{.data.mongoDbUrl}")
export ELASTIC_URL=$(kubectl get configmaps anticrm-config -o jsonpath="{.data.elasticUrl}")
export MINIO_ENDPOINT=$(kubectl get configmaps anticrm-config -o jsonpath="{.data.minioEndpointUrl}")
export MINIO_ACCESS_KEY=$(kubectl get secret anticrm-secret -o jsonpath="{.data.minioAccessKey}" | base64 --decode)
export MINIO_SECRET_KEY=$(kubectl get secret anticrm-secret -o jsonpath="{.data.minioSecretKey}" | base64 --decode)
export SERVER_SECRET=$(kubectl get secret anticrm-secret -o jsonpath="{.data.serverSecret}" | base64 --decode)
kubectl run anticrm-tool --rm --tty -i --restart='Never' \
--env="MONGO_URL=$MONGO_URL" \
--env="TRANSACTOR_URL=ws://transactor/" \
--env="ELASTIC_URL=http://10.1.96.8:9200/" \
--env="ELASTIC_URL=$ELASTIC_URL" \
--env="TELEGRAM_DATABASE=telegram-service" \
--env="MINIO_ENDPOINT=$MINIO_ENDPOINT" \
--env="MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
--env="MINIO_SECRET_KEY=$MINIO_SECRET_KEY" --image hardcoreeng/tool --command -- bash
--env="MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
--env="SERVER_SECRET=$SERVER_SECRET" \
--image hardcoreeng/tool:v0.6.40 --command -- bash

35
dev/tool/src/__start.ts Normal file
View File

@ -0,0 +1,35 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { prepareTools as prepareToolsRaw } from '@hcengineering/server-tool'
import { Data, Tx, Version } from '@hcengineering/core'
import { MigrateOperation } from '@hcengineering/model'
import builder, { migrateOperations, version } from '@hcengineering/model-all'
import { Client } from 'minio'
import { devTool } from '.'
function prepareTools (): {
mongodbUri: string
minio: Client
txes: Tx[]
version: Data<Version>
migrateOperations: MigrateOperation[]
productId: string
} {
return { ...prepareToolsRaw(builder.getTxes()), version, migrateOperations, productId: '' }
}
devTool(prepareTools)

View File

@ -38,7 +38,8 @@ import {
restore
} from '@hcengineering/server-backup'
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import toolPlugin, { prepareTools, version } from '@hcengineering/server-tool'
import toolPlugin from '@hcengineering/server-tool'
import { program } from 'commander'
import { Db, MongoClient } from 'mongodb'
import { exit } from 'process'
@ -53,384 +54,411 @@ import { updateCandidates } from './recruit'
import { clearTelegramHistory } from './telegram'
import { diffWorkspace, dumpWorkspace, restoreWorkspace } from './workspace'
const serverSecret = process.env.SERVER_SECRET
if (serverSecret === undefined) {
console.error('please provide server secret')
process.exit(1)
}
import { Data, Tx, Version } from '@hcengineering/core'
import { MigrateOperation } from '@hcengineering/model'
import { Client } from 'minio'
const transactorUrl = process.env.TRANSACTOR_URL
if (transactorUrl === undefined) {
console.error('please provide transactor url.')
process.exit(1)
}
function getElasticUrl (): string {
const elasticUrl = process.env.ELASTIC_URL
if (elasticUrl === undefined) {
console.error('please provide elastic url')
/**
* @public
*/
export function devTool (
prepareTools: () => {
mongodbUri: string
minio: Client
txes: Tx[]
version: Data<Version>
migrateOperations: MigrateOperation[]
productId: string
}
): void {
const serverSecret = process.env.SERVER_SECRET
if (serverSecret === undefined) {
console.error('please provide server secret')
process.exit(1)
}
return elasticUrl
}
setMetadata(toolPlugin.metadata.Endpoint, transactorUrl)
setMetadata(toolPlugin.metadata.Transactor, transactorUrl)
setMetadata(serverToken.metadata.Secret, serverSecret)
const transactorUrl = process.env.TRANSACTOR_URL
if (transactorUrl === undefined) {
console.error('please provide transactor url.')
process.exit(1)
}
async function withDatabase (uri: string, f: (db: Db, client: MongoClient) => Promise<any>): Promise<void> {
console.log(`connecting to database '${uri}'...`)
function getElasticUrl (): string {
const elasticUrl = process.env.ELASTIC_URL
if (elasticUrl === undefined) {
console.error('please provide elastic url')
process.exit(1)
}
return elasticUrl
}
const client = await MongoClient.connect(uri)
await f(client.db(ACCOUNT_DB), client)
await client.close()
}
setMetadata(toolPlugin.metadata.Endpoint, transactorUrl)
setMetadata(toolPlugin.metadata.Transactor, transactorUrl)
setMetadata(serverToken.metadata.Secret, serverSecret)
program.version('0.0.1')
async function withDatabase (uri: string, f: (db: Db, client: MongoClient) => Promise<any>): Promise<void> {
console.log(`connecting to database '${uri}'...`)
// create-user john.appleseed@gmail.com --password 123 --workspace workspace --fullname "John Appleseed"
program
.command('create-account <email>')
.description('create user and corresponding account in master database')
.requiredOption('-p, --password <password>', 'user password')
.requiredOption('-f, --first <first>', 'first name')
.requiredOption('-l, --last <last>', 'last name')
.action(async (email: string, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
console.log(`creating account ${cmd.first as string} ${cmd.last as string} (${email})...`)
await createAccount(db, email, cmd.password, cmd.first, cmd.last)
const client = await MongoClient.connect(uri)
await f(client.db(ACCOUNT_DB), client)
await client.close()
}
program.version('0.0.1')
// create-user john.appleseed@gmail.com --password 123 --workspace workspace --fullname "John Appleseed"
program
.command('create-account <email>')
.description('create user and corresponding account in master database')
.requiredOption('-p, --password <password>', 'user password')
.requiredOption('-f, --first <first>', 'first name')
.requiredOption('-l, --last <last>', 'last name')
.action(async (email: string, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
console.log(`creating account ${cmd.first as string} ${cmd.last as string} (${email})...`)
await createAccount(db, email, cmd.password, cmd.first, cmd.last)
})
})
})
program
.command('reset-account <email>')
.description('create user and corresponding account in master database')
.option('-p, --password <password>', 'new user password')
.action(async (email: string, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`)
await replacePassword(db, email, cmd.password)
program
.command('reset-account <email>')
.description('create user and corresponding account in master database')
.option('-p, --password <password>', 'new user password')
.action(async (email: string, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`)
await replacePassword(db, email, cmd.password)
})
})
})
program
.command('assign-workspace <email> <workspace>')
.description('assign workspace')
.action(async (email: string, workspace: string, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db, client) => {
console.log(`assigning user ${email} to ${workspace}...`)
await assignWorkspace(db, email, workspace)
program
.command('assign-workspace <email> <workspace>')
.description('assign workspace')
.action(async (email: string, workspace: string, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db, client) => {
console.log(`assigning user ${email} to ${workspace}...`)
await assignWorkspace(db, email, workspace)
})
})
})
program
.command('show-user <email>')
.description('show user')
.action(async (email) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const info = await getAccount(db, email)
console.log(info)
program
.command('show-user <email>')
.description('show user')
.action(async (email) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const info = await getAccount(db, email)
console.log(info)
})
})
})
program
.command('create-workspace <name>')
.description('create workspace')
.requiredOption('-o, --organization <organization>', 'organization name')
.action(async (workspace, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
await createWorkspace(db, workspace, cmd.organization)
program
.command('create-workspace <name>')
.description('create workspace')
.requiredOption('-o, --organization <organization>', 'organization name')
.action(async (workspace, cmd) => {
const { mongodbUri, txes, version, migrateOperations, productId } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
await createWorkspace(version, txes, migrateOperations, productId, db, workspace, cmd.organization)
})
})
})
program
.command('set-user-role <email> <workspace> <role>')
.description('set user role')
.action(async (email: string, workspace: string, role: number, cmd) => {
console.log(`set user ${email} role for ${workspace}...`)
await setRole(email, workspace, role)
})
program
.command('upgrade-workspace <name>')
.description('upgrade workspace')
.action(async (workspace, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
await upgradeWorkspace(db, workspace)
program
.command('set-user-role <email> <workspace> <role>')
.description('set user role')
.action(async (email: string, workspace: string, role: number, cmd) => {
console.log(`set user ${email} role for ${workspace}...`)
await setRole(email, workspace, role)
})
})
program
.command('upgrade')
.description('upgrade')
.action(async (cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const workspaces = await listWorkspaces(db)
for (const ws of workspaces) {
console.log('---UPGRADING----', ws.workspace)
await upgradeWorkspace(db, ws.workspace)
}
program
.command('upgrade-workspace <name>')
.description('upgrade workspace')
.action(async (workspace, cmd) => {
const { mongodbUri, version, txes, migrateOperations, productId } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
await upgradeWorkspace(version, txes, migrateOperations, productId, db, workspace)
})
})
})
program
.command('drop-workspace <name>')
.description('drop workspace')
.action(async (workspace, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const ws = await getWorkspace(db, workspace)
if (ws === null) {
console.log('no workspace exists')
return
}
await dropWorkspace(db, workspace)
program
.command('upgrade')
.description('upgrade')
.action(async (cmd) => {
const { mongodbUri, version, txes, migrateOperations, productId } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const workspaces = await listWorkspaces(db, productId)
for (const ws of workspaces) {
console.log('---UPGRADING----', ws.workspace)
await upgradeWorkspace(version, txes, migrateOperations, productId, db, ws.workspace)
}
})
})
})
program
.command('list-workspaces')
.description('List workspaces')
.action(async () => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const workspacesJSON = JSON.stringify(await listWorkspaces(db), null, 2)
console.info(workspacesJSON)
console.log('latest model version:', JSON.stringify(version))
program
.command('drop-workspace <name>')
.description('drop workspace')
.action(async (workspace, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const ws = await getWorkspace(db, workspace)
if (ws === null) {
console.log('no workspace exists')
return
}
await dropWorkspace(db, workspace)
})
})
})
program
.command('show-accounts')
.description('Show accounts')
.action(async () => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const accountsJSON = JSON.stringify(await listAccounts(db), null, 2)
console.info(accountsJSON)
program
.command('list-workspaces')
.description('List workspaces')
.action(async () => {
const { mongodbUri, version, productId } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const workspacesJSON = JSON.stringify(await listWorkspaces(db, productId), null, 2)
console.info(workspacesJSON)
console.log('latest model version:', JSON.stringify(version))
})
})
})
program
.command('drop-account <name>')
.description('drop account')
.action(async (email, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
await dropAccount(db, email)
program
.command('show-accounts')
.description('Show accounts')
.action(async () => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const accountsJSON = JSON.stringify(await listAccounts(db), null, 2)
console.info(accountsJSON)
})
})
})
program
.command('dump-workspace <workspace> <dirName>')
.description('dump workspace transactions and minio resources')
.action(async (workspace, dirName, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await dumpWorkspace(mongodbUri, workspace, dirName, minio)
})
program
.command('backup <dirName> <workspace>')
.description('dump workspace transactions and minio resources')
.action(async (dirName, workspace, cmd) => {
const storage = await createFileBackupStorage(dirName)
return await backup(transactorUrl, workspace, storage)
})
program
.command('backup-restore <dirName> <workspace> [date]')
.description('dump workspace transactions and minio resources')
.action(async (dirName, workspace, date, cmd) => {
const storage = await createFileBackupStorage(dirName)
return await restore(transactorUrl, workspace, storage, parseInt(date ?? '-1'))
})
program
.command('backup-list <dirName>')
.description('list snaphost ids for backup')
.action(async (dirName, cmd) => {
const storage = await createFileBackupStorage(dirName)
return await backupList(storage)
})
program
.command('backup-s3 <bucketName> <dirName> <workspace>')
.description('dump workspace transactions and minio resources')
.action(async (bucketName, dirName, workspace, cmd) => {
const { minio } = prepareTools()
const storage = await createMinioBackupStorage(minio, bucketName, dirName)
return await backup(transactorUrl, workspace, storage)
})
program
.command('backup-s3-restore <bucketName>, <dirName> <workspace> [date]')
.description('dump workspace transactions and minio resources')
.action(async (bucketName, dirName, workspace, date, cmd) => {
const { minio } = prepareTools()
const storage = await createMinioBackupStorage(minio, bucketName, dirName)
return await restore(transactorUrl, workspace, storage, parseInt(date ?? '-1'))
})
program
.command('backup-s3-list <bucketName> <dirName>')
.description('list snaphost ids for backup')
.action(async (bucketName, dirName, cmd) => {
const { minio } = prepareTools()
const storage = await createMinioBackupStorage(minio, bucketName, dirName)
return await backupList(storage)
})
program
.command('restore-workspace <workspace> <dirName>')
.description('restore workspace transactions and minio resources from previous dump.')
.action(async (workspace, dirName, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await restoreWorkspace(mongodbUri, workspace, dirName, minio, getElasticUrl(), transactorUrl)
})
program
.command('diff-workspace <workspace>')
.description('restore workspace transactions and minio resources from previous dump.')
.action(async (workspace, cmd) => {
const { mongodbUri } = prepareTools()
return await diffWorkspace(mongodbUri, workspace)
})
program
.command('clear-telegram-history <workspace>')
.description('clear telegram history')
.option('-w, --workspace <workspace>', 'target workspace')
.action(async (workspace: string, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const telegramDB = process.env.TELEGRAM_DATABASE
if (telegramDB === undefined) {
console.error('please provide TELEGRAM_DATABASE.')
process.exit(1)
}
console.log(`clearing ${workspace} history:`)
await clearTelegramHistory(mongodbUri, workspace, telegramDB, minio)
program
.command('drop-account <name>')
.description('drop account')
.action(async (email, cmd) => {
const { mongodbUri } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
await dropAccount(db, email)
})
})
})
program
.command('clear-telegram-all-history')
.description('clear telegram history')
.action(async (cmd) => {
const { mongodbUri, minio } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const telegramDB = process.env.TELEGRAM_DATABASE
if (telegramDB === undefined) {
console.error('please provide TELEGRAM_DATABASE.')
process.exit(1)
}
const workspaces = await listWorkspaces(db)
for (const w of workspaces) {
console.log(`clearing ${w.workspace} history:`)
await clearTelegramHistory(mongodbUri, w.workspace, telegramDB, minio)
}
program
.command('dump-workspace <workspace> <dirName>')
.description('dump workspace transactions and minio resources')
.action(async (workspace, dirName, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await dumpWorkspace(mongodbUri, workspace, dirName, minio)
})
})
program
.command('rebuild-elastic [workspace]')
.description('rebuild elastic index')
.action(async (workspace, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
if (workspace === undefined) {
const workspaces = await listWorkspaces(db)
program
.command('backup <dirName> <workspace>')
.description('dump workspace transactions and minio resources')
.action(async (dirName, workspace, cmd) => {
const storage = await createFileBackupStorage(dirName)
return await backup(transactorUrl, workspace, storage)
})
program
.command('backup-restore <dirName> <workspace> [date]')
.description('dump workspace transactions and minio resources')
.action(async (dirName, workspace, date, cmd) => {
const storage = await createFileBackupStorage(dirName)
return await restore(transactorUrl, workspace, storage, parseInt(date ?? '-1'))
})
program
.command('backup-list <dirName>')
.description('list snaphost ids for backup')
.action(async (dirName, cmd) => {
const storage = await createFileBackupStorage(dirName)
return await backupList(storage)
})
program
.command('backup-s3 <bucketName> <dirName> <workspace>')
.description('dump workspace transactions and minio resources')
.action(async (bucketName, dirName, workspace, cmd) => {
const { minio } = prepareTools()
const storage = await createMinioBackupStorage(minio, bucketName, dirName)
return await backup(transactorUrl, workspace, storage)
})
program
.command('backup-s3-restore <bucketName>, <dirName> <workspace> [date]')
.description('dump workspace transactions and minio resources')
.action(async (bucketName, dirName, workspace, date, cmd) => {
const { minio } = prepareTools()
const storage = await createMinioBackupStorage(minio, bucketName, dirName)
return await restore(transactorUrl, workspace, storage, parseInt(date ?? '-1'))
})
program
.command('backup-s3-list <bucketName> <dirName>')
.description('list snaphost ids for backup')
.action(async (bucketName, dirName, cmd) => {
const { minio } = prepareTools()
const storage = await createMinioBackupStorage(minio, bucketName, dirName)
return await backupList(storage)
})
program
.command('restore-workspace <workspace> <dirName>')
.description('restore workspace transactions and minio resources from previous dump.')
.action(async (workspace, dirName, cmd) => {
const { mongodbUri, minio, txes, migrateOperations } = prepareTools()
return await restoreWorkspace(
mongodbUri,
workspace,
dirName,
minio,
getElasticUrl(),
transactorUrl,
txes,
migrateOperations
)
})
program
.command('diff-workspace <workspace>')
.description('restore workspace transactions and minio resources from previous dump.')
.action(async (workspace, cmd) => {
const { mongodbUri, txes } = prepareTools()
return await diffWorkspace(mongodbUri, workspace, txes)
})
program
.command('clear-telegram-history <workspace>')
.description('clear telegram history')
.option('-w, --workspace <workspace>', 'target workspace')
.action(async (workspace: string, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const telegramDB = process.env.TELEGRAM_DATABASE
if (telegramDB === undefined) {
console.error('please provide TELEGRAM_DATABASE.')
process.exit(1)
}
console.log(`clearing ${workspace} history:`)
await clearTelegramHistory(mongodbUri, workspace, telegramDB, minio)
})
})
program
.command('clear-telegram-all-history')
.description('clear telegram history')
.action(async (cmd) => {
const { mongodbUri, minio, productId } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
const telegramDB = process.env.TELEGRAM_DATABASE
if (telegramDB === undefined) {
console.error('please provide TELEGRAM_DATABASE.')
process.exit(1)
}
const workspaces = await listWorkspaces(db, productId)
for (const w of workspaces) {
await rebuildElastic(mongodbUri, w.workspace, minio, getElasticUrl())
console.log(`clearing ${w.workspace} history:`)
await clearTelegramHistory(mongodbUri, w.workspace, telegramDB, minio)
}
} else {
await rebuildElastic(mongodbUri, workspace, minio, getElasticUrl())
console.log('rebuild end')
}
})
})
})
program
.command('import-xml <workspace> <fileName>')
.description('Import Talants.')
.action(async (workspace, fileName, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await importXml(transactorUrl, workspace, minio, fileName, mongodbUri, getElasticUrl())
})
program
.command('rebuild-elastic [workspace]')
.description('rebuild elastic index')
.action(async (workspace, cmd) => {
const { mongodbUri, minio, productId } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
if (workspace === undefined) {
const workspaces = await listWorkspaces(db, productId)
program
.command('import-lead-csv <workspace> <fileName>')
.description('Import LEAD csv customer organizations')
.action(async (workspace, fileName, cmd) => {
return await importLead(transactorUrl, workspace, fileName)
})
for (const w of workspaces) {
await rebuildElastic(mongodbUri, w.workspace, minio, getElasticUrl())
}
} else {
await rebuildElastic(mongodbUri, workspace, minio, getElasticUrl())
console.log('rebuild end')
}
})
})
program
.command('import-lead-csv2 <workspace> <fileName>')
.description('Import LEAD csv customer organizations')
.action(async (workspace, fileName, cmd) => {
return await importLead2(transactorUrl, workspace, fileName)
})
program
.command('import-xml <workspace> <fileName>')
.description('Import Talants.')
.action(async (workspace, fileName, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await importXml(transactorUrl, workspace, minio, fileName, mongodbUri, getElasticUrl())
})
program
.command('import-talant-csv <workspace> <fileName>')
.description('Import Talant csv')
.action(async (workspace, fileName, cmd) => {
const rekoniUrl = process.env.REKONI_URL
if (rekoniUrl === undefined) {
console.log('Please provide REKONI_URL environment variable')
exit(1)
}
return await importTalants(transactorUrl, workspace, fileName, rekoniUrl)
})
program
.command('import-lead-csv <workspace> <fileName>')
.description('Import LEAD csv customer organizations')
.action(async (workspace, fileName, cmd) => {
return await importLead(transactorUrl, workspace, fileName)
})
program
.command('import-org-csv <workspace> <fileName>')
.description('Import Organizations csv')
.action(async (workspace, fileName, cmd) => {
return await importOrgs(transactorUrl, workspace, fileName)
})
program
.command('import-lead-csv2 <workspace> <fileName>')
.description('Import LEAD csv customer organizations')
.action(async (workspace, fileName, cmd) => {
return await importLead2(transactorUrl, workspace, fileName)
})
program
.command('lead-duplicates <workspace>')
.description('Find and remove duplicate organizations.')
.action(async (workspace, cmd) => {
return await removeDuplicates(transactorUrl, workspace)
})
program
.command('import-talant-csv <workspace> <fileName>')
.description('Import Talant csv')
.action(async (workspace, fileName, cmd) => {
const rekoniUrl = process.env.REKONI_URL
if (rekoniUrl === undefined) {
console.log('Please provide REKONI_URL environment variable')
exit(1)
}
return await importTalants(transactorUrl, workspace, fileName, rekoniUrl)
})
program
.command('generate-token <name> <workspace>')
.description('generate token')
.action(async (name, workspace) => {
console.log(generateToken(name, workspace))
})
program
.command('decode-token <token>')
.description('decode token')
.action(async (token) => {
console.log(decodeToken(token))
})
program
.command('update-recruit <workspace>')
.description('process pdf documents inside minio and update resumes with skills, etc.')
.action(async (workspace) => {
const rekoniUrl = process.env.REKONI_URL
if (rekoniUrl === undefined) {
console.log('Please provide REKONI_URL environment variable')
exit(1)
}
const { mongodbUri, minio } = prepareTools()
return await updateCandidates(transactorUrl, workspace, minio, mongodbUri, getElasticUrl(), rekoniUrl)
})
program
.command('import-org-csv <workspace> <fileName>')
.description('Import Organizations csv')
.action(async (workspace, fileName, cmd) => {
return await importOrgs(transactorUrl, workspace, fileName)
})
program.parse(process.argv)
program
.command('lead-duplicates <workspace>')
.description('Find and remove duplicate organizations.')
.action(async (workspace, cmd) => {
return await removeDuplicates(transactorUrl, workspace)
})
program
.command('generate-token <name> <workspace>')
.description('generate token')
.action(async (name, workspace) => {
console.log(generateToken(name, workspace))
})
program
.command('decode-token <token>')
.description('decode token')
.action(async (token) => {
console.log(decodeToken(token))
})
program
.command('update-recruit <workspace>')
.description('process pdf documents inside minio and update resumes with skills, etc.')
.action(async (workspace) => {
const rekoniUrl = process.env.REKONI_URL
if (rekoniUrl === undefined) {
console.log('Please provide REKONI_URL environment variable')
exit(1)
}
const { mongodbUri, minio } = prepareTools()
return await updateCandidates(transactorUrl, workspace, minio, mongodbUri, getElasticUrl(), rekoniUrl)
})
program.parse(process.argv)
}

View File

@ -16,7 +16,7 @@
import contact from '@hcengineering/contact'
import core, { DOMAIN_TX, Tx } from '@hcengineering/core'
import builder, { version } from '@hcengineering/model-all'
import { MigrateOperation } from '@hcengineering/model'
import { upgradeModel } from '@hcengineering/server-tool'
import { existsSync } from 'fs'
import { mkdir, open, readFile, writeFile } from 'fs/promises'
@ -54,7 +54,7 @@ export async function dumpWorkspace (mongoUrl: string, dbName: string, fileName:
}
const workspaceInfo: WorkspaceInfo = {
version: `${version.major}.${version.minor}.${version.patch}`,
version: '0.0.0',
collections: [],
minioData: []
}
@ -118,7 +118,9 @@ export async function restoreWorkspace (
fileName: string,
minio: Client,
elasticUrl: string,
transactorUrl: string
transactorUrl: string,
rawTxes: Tx[],
migrateOperations: MigrateOperation[]
): Promise<void> {
console.log('Restoring workspace', mongoUrl, dbName, fileName)
const client = new MongoClient(mongoUrl)
@ -170,7 +172,7 @@ export async function restoreWorkspace (
}
}
await upgradeModel(transactorUrl, dbName)
await upgradeModel(transactorUrl, dbName, rawTxes, migrateOperations)
await rebuildElastic(mongoUrl, dbName, minio, elasticUrl)
} finally {
@ -178,7 +180,7 @@ export async function restoreWorkspace (
}
}
export async function diffWorkspace (mongoUrl: string, dbName: string): Promise<void> {
export async function diffWorkspace (mongoUrl: string, dbName: string, rawTxes: Tx[]): Promise<void> {
const client = new MongoClient(mongoUrl)
try {
await client.connect()
@ -195,7 +197,7 @@ export async function diffWorkspace (mongoUrl: string, dbName: string): Promise<
})
.toArray()
const txes = builder.getTxes().filter((tx) => {
const txes = rawTxes.filter((tx) => {
return (
tx.objectSpace === core.space.Model &&
tx.modifiedBy === core.account.System &&

View File

@ -155,7 +155,7 @@ export class TSavedMessages extends TPreference implements SavedMessages {
attachedTo!: Ref<ChunterMessage>
}
export function createModel (builder: Builder): void {
export function createModel (builder: Builder, options = { addApplication: true }): void {
builder.createModel(
TChunterSpace,
TChannel,
@ -319,88 +319,90 @@ export function createModel (builder: Builder): void {
chunter.action.ConvertToPrivate
)
builder.createDoc(
workbench.class.Application,
core.space.Model,
{
label: chunter.string.ApplicationLabelChunter,
icon: chunter.icon.Chunter,
alias: chunterId,
hidden: false,
navigatorModel: {
specials: [
{
id: 'spaceBrowser',
component: workbench.component.SpaceBrowser,
icon: chunter.icon.ChannelBrowser,
label: chunter.string.ChannelBrowser,
position: 'top',
spaceClass: chunter.class.Channel,
componentProps: {
_class: chunter.class.Channel,
if (options.addApplication) {
builder.createDoc(
workbench.class.Application,
core.space.Model,
{
label: chunter.string.ApplicationLabelChunter,
icon: chunter.icon.Chunter,
alias: chunterId,
hidden: false,
navigatorModel: {
specials: [
{
id: 'spaceBrowser',
component: workbench.component.SpaceBrowser,
icon: chunter.icon.ChannelBrowser,
label: chunter.string.ChannelBrowser,
createItemDialog: chunter.component.CreateChannel,
createItemLabel: chunter.string.CreateChannel
position: 'top',
spaceClass: chunter.class.Channel,
componentProps: {
_class: chunter.class.Channel,
label: chunter.string.ChannelBrowser,
createItemDialog: chunter.component.CreateChannel,
createItemLabel: chunter.string.CreateChannel
}
},
{
id: 'archive',
component: workbench.component.Archive,
icon: view.icon.Archive,
label: workbench.string.Archive,
position: 'top',
visibleIf: workbench.function.HasArchiveSpaces,
spaceClass: chunter.class.Channel
},
{
id: 'threads',
label: chunter.string.Threads,
icon: chunter.icon.Thread,
component: chunter.component.Threads,
position: 'top'
},
{
id: 'savedItems',
label: chunter.string.SavedItems,
icon: chunter.icon.Bookmark,
component: chunter.component.SavedMessages
},
{
id: 'fileBrowser',
label: attachment.string.FileBrowser,
icon: attachment.icon.FileBrowser,
component: attachment.component.FileBrowser,
componentProps: {
requestedSpaceClasses: [chunter.class.Channel, chunter.class.DirectMessage]
}
},
{
id: 'chunterBrowser',
label: chunter.string.ChunterBrowser,
icon: workbench.icon.Search,
component: chunter.component.ChunterBrowser,
visibleIf: chunter.function.ChunterBrowserVisible
}
},
{
id: 'archive',
component: workbench.component.Archive,
icon: view.icon.Archive,
label: workbench.string.Archive,
position: 'top',
visibleIf: workbench.function.HasArchiveSpaces,
spaceClass: chunter.class.Channel
},
{
id: 'threads',
label: chunter.string.Threads,
icon: chunter.icon.Thread,
component: chunter.component.Threads,
position: 'top'
},
{
id: 'savedItems',
label: chunter.string.SavedItems,
icon: chunter.icon.Bookmark,
component: chunter.component.SavedMessages
},
{
id: 'fileBrowser',
label: attachment.string.FileBrowser,
icon: attachment.icon.FileBrowser,
component: attachment.component.FileBrowser,
componentProps: {
requestedSpaceClasses: [chunter.class.Channel, chunter.class.DirectMessage]
],
spaces: [
{
label: chunter.string.Channels,
spaceClass: chunter.class.Channel,
addSpaceLabel: chunter.string.CreateChannel,
createComponent: chunter.component.CreateChannel
},
{
label: chunter.string.DirectMessages,
spaceClass: chunter.class.DirectMessage,
addSpaceLabel: chunter.string.NewDirectMessage,
createComponent: chunter.component.CreateDirectMessage
}
},
{
id: 'chunterBrowser',
label: chunter.string.ChunterBrowser,
icon: workbench.icon.Search,
component: chunter.component.ChunterBrowser,
visibleIf: chunter.function.ChunterBrowserVisible
}
],
spaces: [
{
label: chunter.string.Channels,
spaceClass: chunter.class.Channel,
addSpaceLabel: chunter.string.CreateChannel,
createComponent: chunter.component.CreateChannel
},
{
label: chunter.string.DirectMessages,
spaceClass: chunter.class.DirectMessage,
addSpaceLabel: chunter.string.NewDirectMessage,
createComponent: chunter.component.CreateDirectMessage
}
],
aside: chunter.component.ThreadView
}
},
chunter.app.Chunter
)
],
aside: chunter.component.ThreadView
}
},
chunter.app.Chunter
)
}
builder.mixin(chunter.class.Comment, core.class.Class, view.mixin.AttributePresenter, {
presenter: chunter.component.CommentPresenter

View File

@ -25,6 +25,9 @@
export let id: string | undefined = undefined
</script>
/** eslint-disable a11y-click-events-have-key-events */
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div
{id}
class="flex-center icon-button icon-{size}"

View File

@ -40,7 +40,7 @@
let selection = 0
let list: ListView
async function handleSelection (evt: Event | undefined, selection: number): Promise<void> {
async function handleSelection (_: Event | undefined, selection: number): Promise<void> {
const space = objects[selection]
dispatch('close', space)
}
@ -76,14 +76,7 @@
<div class="selectPopup" on:keydown={onKeydown}>
{#if searchable}
<div class="header">
<input
bind:this={input}
type="text"
bind:value={search}
placeholder={phTraslate}
on:input={(ev) => {}}
on:change
/>
<input bind:this={input} type="text" bind:value={search} placeholder={phTraslate} on:input={() => {}} on:change />
</div>
{/if}
<div class="scroll">

View File

@ -16,7 +16,10 @@
import { getContext } from 'svelte'
import FontSize from './icons/FontSize.svelte'
const { currentFontSize, setFontSize } = getContext('fontsize')
const { currentFontSize, setFontSize } = getContext('fontsize') as {
currentFontSize: string
setFontSize: (size: string) => void
}
const fontsizes = ['small-font', 'normal-font']
@ -28,6 +31,7 @@
}
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div class="flex-center" on:click={changeFontSize}>
<FontSize size={'small'} />
</div>

View File

@ -20,7 +20,10 @@
import Flags from './icons/Flags.svelte'
const { currentLanguage, setLanguage } = getContext('lang')
const { currentLanguage, setLanguage } = getContext('lang') as {
currentLanguage: string
setLanguage: (lang: string) => void
}
const langs = [
{ id: 'en', label: ui.string.English },
{ id: 'ru', label: ui.string.Russian }
@ -41,6 +44,7 @@
<Flags />
{#if selected}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div bind:this={trigger} class="flex-center cursor-pointer" on:click={selectLanguage}>
<svg class="svg-small">
<use href="#{selected.id}-flag" />

View File

@ -58,8 +58,7 @@
let docWidth: number = window.innerWidth
let docHeight: number = window.innerHeight
let maxLenght: number
$: maxLenght = docWidth >= docHeight ? docWidth : docHeight
let isMobile: boolean
let alwaysMobile: boolean = false
$: isMobile = alwaysMobile || checkMobile()
@ -113,6 +112,7 @@
<div class="flex-center widget cursor-pointer mr-3">
<FontSizeSelector />
</div>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div
class="flex-center widget mr-3"
class:rotated={!isPortrait && isMobile}

View File

@ -3,7 +3,7 @@
import Mute from './icons/Mute.svelte'
import { deviceOptionsStore as deviceInfo } from '../../'
const { currentTheme, setTheme } = getContext('theme')
const { currentTheme, setTheme } = getContext('theme') as { currentTheme: string; setTheme: (theme: string) => void }
const themes = ['theme-light', 'theme-dark']
@ -17,6 +17,7 @@
}
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div class="flex-center" on:click={changeTheme}>
<Mute size={'small'} />
</div>

View File

@ -5,15 +5,15 @@
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
"scripts": {
"start": "ts-node src/index.ts",
"start": "ts-node src/__start.ts",
"build": "heft build",
"build:watch": "tsc",
"lint:fix": "eslint --fix src",
"bundle": "esbuild src/index.ts --bundle --minify --platform=node > bundle.js",
"bundle": "esbuild src/__start.ts --bundle --minify --platform=node > bundle.js",
"docker:build": "docker build -t hardcoreeng/account .",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/account staging",
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/account",
"run-local": "cross-env MONGO_URL=mongodb://localhost:27017 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' TRANSACTOR_URL=ws:/localhost:3333 ts-node src/index.ts",
"run-local": "cross-env MONGO_URL=mongodb://localhost:27017 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' TRANSACTOR_URL=ws:/localhost:3333 ts-node src/__start.ts",
"lint": "eslint src",
"format": "prettier --write src && eslint --fix src"
},
@ -27,7 +27,7 @@
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.26.0",
"esbuild": "^0.12.26",
"esbuild": "^0.15.13",
"@types/koa-bodyparser": "^4.3.3",
"@types/koa-router": "^7.4.4",
"@types/koa": "^2.13.4",
@ -42,12 +42,14 @@
"dependencies": {
"@hcengineering/account": "~0.6.0",
"@hcengineering/platform": "^0.6.7",
"@hcengineering/core": "^0.6.17",
"mongodb": "^4.9.0",
"koa": "^2.13.1",
"koa-router": "^10.1.1",
"koa-bodyparser": "^4.3.0",
"@koa/cors": "^3.1.0",
"@hcengineering/server-tool": "~0.6.0",
"@hcengineering/server-token": "~0.6.0"
"@hcengineering/server-token": "~0.6.0",
"@hcengineering/model-all": "~0.6.0"
}
}

View File

@ -0,0 +1,23 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { getMethods } from '@hcengineering/account'
import { Tx } from '@hcengineering/core'
import builder, { migrateOperations, version } from '@hcengineering/model-all'
import { serveAccount } from '.'
const txes = JSON.parse(JSON.stringify(builder.getTxes())) as Tx[]
serveAccount(getMethods(version, txes, migrateOperations, ''))

View File

@ -14,10 +14,10 @@
// limitations under the License.
//
import { ACCOUNT_DB, methods } from '@hcengineering/account'
import toolPlugin from '@hcengineering/server-tool'
import serverToken from '@hcengineering/server-token'
import { AccountMethod, ACCOUNT_DB } from '@hcengineering/account'
import platform, { Request, Response, serialize, setMetadata, Severity, Status } from '@hcengineering/platform'
import serverToken from '@hcengineering/server-token'
import toolPlugin from '@hcengineering/server-tool'
import cors from '@koa/cors'
import { IncomingHttpHeaders } from 'http'
import Koa from 'koa'
@ -25,80 +25,85 @@ import bodyParser from 'koa-bodyparser'
import Router from 'koa-router'
import { Db, MongoClient } from 'mongodb'
const ACCOUNT_PORT = parseInt(process.env.ACCOUNT_PORT ?? '3000')
const dbUri = process.env.MONGO_URL
if (dbUri === undefined) {
console.log('Please provide mongodb url')
process.exit(1)
}
const transactorUri = process.env.TRANSACTOR_URL
if (transactorUri === undefined) {
console.log('Please provide transactor url')
process.exit(1)
}
const endpointUri = process.env.ENDPOINT_URL ?? transactorUri
const serverSecret = process.env.SERVER_SECRET
if (serverSecret === undefined) {
console.log('Please provide server secret')
process.exit(1)
}
setMetadata(serverToken.metadata.Secret, serverSecret)
setMetadata(toolPlugin.metadata.Endpoint, endpointUri)
setMetadata(toolPlugin.metadata.Transactor, transactorUri)
let client: MongoClient
const app = new Koa()
const router = new Router()
const extractToken = (header: IncomingHttpHeaders): string | undefined => {
try {
return header.authorization?.slice(7) ?? undefined
} catch {
return undefined
/**
* @public
*/
export function serveAccount (methods: Record<string, AccountMethod>): void {
const ACCOUNT_PORT = parseInt(process.env.ACCOUNT_PORT ?? '3000')
const dbUri = process.env.MONGO_URL
if (dbUri === undefined) {
console.log('Please provide mongodb url')
process.exit(1)
}
}
router.post('rpc', '/', async (ctx) => {
const token = extractToken(ctx.request.headers)
const transactorUri = process.env.TRANSACTOR_URL
if (transactorUri === undefined) {
console.log('Please provide transactor url')
process.exit(1)
}
const request = ctx.request.body
const method = (methods as { [key: string]: (db: Db, request: Request<any>, token?: string) => Response<any> })[
request.method
]
if (method === undefined) {
const response: Response<void> = {
id: request.id,
error: new Status(Severity.ERROR, platform.status.UnknownMethod, { method: request.method })
const endpointUri = process.env.ENDPOINT_URL ?? transactorUri
const serverSecret = process.env.SERVER_SECRET
if (serverSecret === undefined) {
console.log('Please provide server secret')
process.exit(1)
}
setMetadata(serverToken.metadata.Secret, serverSecret)
setMetadata(toolPlugin.metadata.Endpoint, endpointUri)
setMetadata(toolPlugin.metadata.Transactor, transactorUri)
let client: MongoClient
const app = new Koa()
const router = new Router()
const extractToken = (header: IncomingHttpHeaders): string | undefined => {
try {
return header.authorization?.slice(7) ?? undefined
} catch {
return undefined
}
}
router.post('rpc', '/', async (ctx) => {
const token = extractToken(ctx.request.headers)
const request = ctx.request.body as any
const method = (methods as { [key: string]: (db: Db, request: Request<any>, token?: string) => Response<any> })[
request.method
]
if (method === undefined) {
const response: Response<void> = {
id: request.id,
error: new Status(Severity.ERROR, platform.status.UnknownMethod, { method: request.method })
}
ctx.body = serialize(response)
}
ctx.body = serialize(response)
if (client === undefined) {
client = await MongoClient.connect(dbUri)
}
const db = client.db(ACCOUNT_DB)
const result = await method(db, request, token)
console.log(result)
ctx.body = result
})
app.use(cors())
app.use(bodyParser())
app.use(router.routes()).use(router.allowedMethods())
const server = app.listen(ACCOUNT_PORT, () => {
console.log(`server started on port ${ACCOUNT_PORT}`)
})
const close = (): void => {
server.close()
}
if (client === undefined) {
client = await MongoClient.connect(dbUri)
}
const db = client.db(ACCOUNT_DB)
const result = await method(db, request, token)
console.log(result)
ctx.body = result
})
app.use(cors())
app.use(bodyParser())
app.use(router.routes()).use(router.allowedMethods())
const server = app.listen(ACCOUNT_PORT, () => {
console.log(`server started on port ${ACCOUNT_PORT}`)
})
const close = (): void => {
server.close()
process.on('SIGINT', close)
process.on('SIGTERM', close)
process.on('exit', close)
}
process.on('SIGINT', close)
process.on('SIGTERM', close)
process.on('exit', close)

View File

@ -27,7 +27,7 @@
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.26.0",
"esbuild": "^0.12.26",
"esbuild": "^0.15.13",
"@typescript-eslint/parser": "^5.41.0",
"eslint-config-standard-with-typescript": "^23.0.0",
"prettier": "^2.7.1",

View File

@ -27,7 +27,7 @@
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.26.0",
"esbuild": "^0.12.26",
"esbuild": "^0.15.13",
"@typescript-eslint/parser": "^5.41.0",
"eslint-config-standard-with-typescript": "^23.0.0",
"prettier": "^2.7.1",

7
pods/front/.eslintrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/config/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

4
pods/front/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

13
pods/front/Dockerfile Normal file
View File

@ -0,0 +1,13 @@
FROM node:16-alpine
RUN apk add dumb-init
ENV NODE_ENV production
WORKDIR /app
RUN npm install --ignore-scripts=false --verbose sharp --unsafe-perm
COPY bundle.js ./
COPY dist/ ./dist/
EXPOSE 8080
CMD [ "dumb-init", "node", "./bundle.js" ]

View File

@ -0,0 +1,18 @@
// The "rig.json" file directs tools to look for their config files in an external package.
// Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
/**
* (Required) The name of the rig package to inherit from.
* It should be an NPM package name with the "-rig" suffix.
*/
"rigPackageName": "@hcengineering/platform-rig"
/**
* (Optional) Selects a config profile from the rig package. The name must consist of
* lowercase alphanumeric words separated by hyphens, for example "sample-profile".
* If omitted, then the "default" profile will be used."
*/
// "rigProfile": "your-profile-name"
}

63
pods/front/package.json Normal file
View File

@ -0,0 +1,63 @@
{
"name": "@hcengineering/pod-front",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
"scripts": {
"build": "heft build",
"build:watch": "tsc",
"lint:fix": "eslint --fix src",
"bundle": "esbuild src/__start.ts --define:process.env.MODEL_VERSION=$(node ../../models/all/lib/__showversion.js) --bundle --minify --platform=node --external:sharp > bundle.js & rm -rf ./dist && cp -r ../../dev/prod/dist . && cp -r ../../dev/prod/public/* ./dist/ && rm ./dist/config.json",
"docker:build": "docker build -t hardcoreeng/front .",
"docker:staging": "../common/scripts/docker_tag.sh hardcoreeng/front staging",
"docker:push": "../common/scripts/docker_tag.sh hardcoreeng/front",
"lint": "eslint src",
"format": "prettier --write src && eslint --fix src",
"run-local": "cross-env MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost TRANSACTOR_URL=ws:/localhost:3333 SERVER_SECRET='secret' ACCOUNTS_URL=http://localhost:3000 UPLOAD_URL=/files ELASTIC_URL=http://localhost:9200 MODEL_VERSION=$(node ../../models/all/lib/__showversion.js) PUBLIC_DIR='.' ts-node ./src/__start.ts"
},
"devDependencies": {
"@hcengineering/platform-rig": "~0.6.0",
"@types/heft-jest": "^1.0.3",
"@types/node": "~16.11.12",
"@typescript-eslint/eslint-plugin": "^5.41.0",
"@typescript-eslint/parser": "^5.41.0",
"eslint-config-standard-with-typescript": "^23.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.4.0",
"eslint-plugin-promise": "^6.1.1",
"eslint": "^8.26.0",
"@types/express": "^4.17.13",
"@types/express-fileupload": "^1.1.7",
"@types/uuid": "^8.3.1",
"@types/cors": "^2.8.12",
"@types/minio": "~7.0.11",
"esbuild": "^0.15.13",
"prettier": "^2.7.1",
"@rushstack/heft": "^0.47.9",
"typescript": "^4.3.5",
"@types/body-parser": "~1.19.2",
"cross-env": "~7.0.3",
"ts-node": "^10.8.0",
"@types/compression": "~1.7.2",
"@types/sharp": "~0.30.4"
},
"dependencies": {
"@hcengineering/front": "^0.6.0",
"@hcengineering/core": "^0.6.17",
"@hcengineering/platform": "^0.6.7",
"express": "^4.17.1",
"express-fileupload": "^1.2.1",
"uuid": "^8.3.2",
"cors": "^2.8.5",
"@hcengineering/elastic": "~0.6.0",
"@hcengineering/server-core": "~0.6.1",
"@hcengineering/server-token": "~0.6.0",
"@hcengineering/attachment": "~0.6.1",
"@hcengineering/contrib": "~0.6.0",
"minio": "^7.0.26",
"body-parser": "~1.19.1",
"compression": "~1.7.4",
"sharp": "~0.30.7"
}
}

12
pods/front/run.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash
export ACCOUNTS_URL=http://localhost:3333
export COLLABORATOR_URL=ws://localhost:3078
export UPLOAD_URL=http://localhost:3333/files
export TRANSACTOR_URL=ws://localhost:3333
export ELASTIC_URL=http://elastic:9200
export MINIO_ENDPOINT=minio
export MINIO_ACCESS_KEY=minioadmin
export MINIO_SECRET_KEY=minioadmin
node ./bundle.js

17
pods/front/src/__start.ts Normal file
View File

@ -0,0 +1,17 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import '@hcengineering/front/src/__start'

1
pods/front/src/index.ts Normal file
View File

@ -0,0 +1 @@
export * from '@hcengineering/front'

10
pods/front/tsconfig.json Normal file
View File

@ -0,0 +1,10 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"target": "ES2019",
"rootDir": "./src",
"outDir": "./lib",
"esModuleInterop": true
}
}

7
pods/server/.eslintrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/config/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

4
pods/server/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

View File

@ -0,0 +1,37 @@
{
"name": "@hcengineering/server",
"entries": [
{
"version": "0.6.2",
"tag": "@hcengineering/server_v0.6.2",
"date": "Fri, 20 Aug 2021 16:21:03 GMT",
"comments": {
"patch": [
{
"comment": "Transaction ordering"
}
],
"dependency": [
{
"comment": "Updating dependency \"@hcengineering/core\" from `~0.6.10` to `~0.6.11`"
},
{
"comment": "Updating dependency \"@hcengineering/mongo\" from `~0.6.0` to `~0.6.1`"
}
]
}
},
{
"version": "0.6.0",
"tag": "@hcengineering/server_v0.6.0",
"date": "Sun, 08 Aug 2021 10:14:57 GMT",
"comments": {
"dependency": [
{
"comment": "Updating dependency \"@hcengineering/platform\" from `~0.6.3` to `~0.6.4`"
}
]
}
}
]
}

16
pods/server/CHANGELOG.md Normal file
View File

@ -0,0 +1,16 @@
# Change Log - @hcengineering/server
This log was last generated on Fri, 20 Aug 2021 16:21:03 GMT and should not be manually modified.
## 0.6.2
Fri, 20 Aug 2021 16:21:03 GMT
### Patches
- Transaction ordering
## 0.6.0
Sun, 08 Aug 2021 10:14:57 GMT
_Initial release_

20
pods/server/build.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
#
# Copyright © 2020, 2021 Anticrm Platform Contributors.
# Copyright © 2021 Hardcore Engineering Inc.
#
# Licensed under the Eclipse Public License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. You may
# obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
rushx bundle
rushx docker:build $@
rushx docker:push

View File

@ -0,0 +1,18 @@
// The "rig.json" file directs tools to look for their config files in an external package.
// Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
/**
* (Required) The name of the rig package to inherit from.
* It should be an NPM package name with the "-rig" suffix.
*/
"rigPackageName": "@hcengineering/platform-rig"
/**
* (Optional) Selects a config profile from the rig package. The name must consist of
* lowercase alphanumeric words separated by hyphens, for example "sample-profile".
* If omitted, then the "default" profile will be used."
*/
// "rigProfile": "your-profile-name"
}

83
pods/server/package.json Normal file
View File

@ -0,0 +1,83 @@
{
"name": "@hcengineering/pod-server",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
"scripts": {
"start": "cross-env MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret ts-node src/__start.ts",
"build": "heft build",
"lint:fix": "eslint --fix src",
"bundle": "esbuild src/__start.ts --bundle --platform=node > bundle.js",
"docker:build": "docker build -t hardcoreeng/transactor .",
"docker:staging": "../common/scripts/docker_tag.sh hardcoreeng/transactor staging",
"docker:push": "../common/scripts/docker_tag.sh hardcoreeng/transactor",
"build:watch": "tsc",
"lint": "eslint src",
"format": "prettier --write src && eslint --fix src"
},
"devDependencies": {
"cross-env": "~7.0.3",
"@hcengineering/platform-rig": "~0.6.0",
"@types/heft-jest": "^1.0.3",
"@types/node": "~16.11.12",
"@typescript-eslint/eslint-plugin": "^5.41.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.26.0",
"@types/ws": "^8.5.3",
"ts-node": "^10.8.0",
"esbuild": "^0.15.13",
"@typescript-eslint/parser": "^5.41.0",
"eslint-config-standard-with-typescript": "^23.0.0",
"prettier": "^2.7.1",
"@rushstack/heft": "^0.47.9",
"typescript": "^4.3.5",
"@types/minio": "~7.0.11"
},
"dependencies": {
"@hcengineering/core": "^0.6.17",
"@hcengineering/platform": "^0.6.7",
"@hcengineering/server-core": "~0.6.1",
"@hcengineering/server-ws": "~0.6.11",
"@hcengineering/server-attachment": "~0.6.1",
"@hcengineering/server-attachment-resources": "~0.6.0",
"@hcengineering/server": "~0.6.4",
"@hcengineering/mongo": "~0.6.1",
"@hcengineering/elastic": "~0.6.0",
"elastic-apm-node": "~3.26.0",
"minio": "^7.0.26",
"@hcengineering/server-contact": "~0.6.1",
"@hcengineering/server-contact-resources": "~0.6.0",
"@hcengineering/server-notification": "^0.6.0",
"@hcengineering/server-notification-resources": "~0.6.0",
"@hcengineering/server-setting": "~0.6.0",
"@hcengineering/server-setting-resources": "~0.6.0",
"@hcengineering/server-chunter": "~0.6.0",
"@hcengineering/server-chunter-resources": "~0.6.0",
"@hcengineering/server-inventory": "~0.6.0",
"@hcengineering/server-inventory-resources": "~0.6.0",
"@hcengineering/server-lead": "~0.6.0",
"@hcengineering/server-lead-resources": "~0.6.0",
"@hcengineering/server-recruit": "~0.6.0",
"@hcengineering/server-recruit-resources": "~0.6.0",
"@hcengineering/server-task": "~0.6.0",
"@hcengineering/server-task-resources": "~0.6.0",
"@hcengineering/server-tracker": "~0.6.0",
"@hcengineering/server-tracker-resources": "~0.6.0",
"@hcengineering/server-tags": "~0.6.0",
"@hcengineering/server-tags-resources": "~0.6.0",
"@hcengineering/server-calendar": "~0.6.0",
"@hcengineering/server-calendar-resources": "~0.6.0",
"@hcengineering/server-gmail": "~0.6.0",
"@hcengineering/server-gmail-resources": "~0.6.0",
"@hcengineering/server-preference": "~0.6.0",
"@hcengineering/server-telegram": "~0.6.0",
"@hcengineering/server-telegram-resources": "~0.6.0",
"@hcengineering/server-hr": "~0.6.0",
"@hcengineering/server-hr-resources": "~0.6.0",
"@hcengineering/server-token": "~0.6.0",
"@hcengineering/middleware": "~0.6.0"
}
}

17
pods/server/src/index.ts Normal file
View File

@ -0,0 +1,17 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021, 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export { start } from './server'

143
pods/server/src/server.ts Normal file
View File

@ -0,0 +1,143 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { DOMAIN_BLOB, DOMAIN_FULLTEXT_BLOB, DOMAIN_MODEL, DOMAIN_TRANSIENT, DOMAIN_TX } from '@hcengineering/core'
import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic'
import { ModifiedMiddleware, PrivateMiddleware } from '@hcengineering/middleware'
import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import { addLocation } from '@hcengineering/platform'
import {
BackupClientSession,
createMinioDataAdapter,
createNullAdapter,
getMetricsContext,
MinioConfig
} from '@hcengineering/server'
import { serverAttachmentId } from '@hcengineering/server-attachment'
import { serverCalendarId } from '@hcengineering/server-calendar'
import { serverChunterId } from '@hcengineering/server-chunter'
import { serverContactId } from '@hcengineering/server-contact'
import {
createInMemoryAdapter,
createPipeline,
DbConfiguration,
MiddlewareCreator,
Pipeline
} from '@hcengineering/server-core'
import { serverGmailId } from '@hcengineering/server-gmail'
import { serverHrId } from '@hcengineering/server-hr'
import { serverInventoryId } from '@hcengineering/server-inventory'
import { serverLeadId } from '@hcengineering/server-lead'
import { serverNotificationId } from '@hcengineering/server-notification'
import { serverRecruitId } from '@hcengineering/server-recruit'
import { serverSettingId } from '@hcengineering/server-setting'
import { serverTagsId } from '@hcengineering/server-tags'
import { serverTaskId } from '@hcengineering/server-task'
import { serverTelegramId } from '@hcengineering/server-telegram'
import { Token } from '@hcengineering/server-token'
import { serverTrackerId } from '@hcengineering/server-tracker'
import { BroadcastCall, ClientSession, start as startJsonRpc } from '@hcengineering/server-ws'
import { Client as MinioClient } from 'minio'
/**
* @public
*/
export function start (
dbUrl: string,
fullTextUrl: string,
minioConf: MinioConfig,
port: number,
host?: string
): () => void {
addLocation(serverAttachmentId, () => import('@hcengineering/server-attachment-resources'))
addLocation(serverContactId, () => import('@hcengineering/server-contact-resources'))
addLocation(serverNotificationId, () => import('@hcengineering/server-notification-resources'))
addLocation(serverSettingId, () => import('@hcengineering/server-setting-resources'))
addLocation(serverChunterId, () => import('@hcengineering/server-chunter-resources'))
addLocation(serverInventoryId, () => import('@hcengineering/server-inventory-resources'))
addLocation(serverLeadId, () => import('@hcengineering/server-lead-resources'))
addLocation(serverRecruitId, () => import('@hcengineering/server-recruit-resources'))
addLocation(serverTaskId, () => import('@hcengineering/server-task-resources'))
addLocation(serverTrackerId, () => import('@hcengineering/server-tracker-resources'))
addLocation(serverTagsId, () => import('@hcengineering/server-tags-resources'))
addLocation(serverCalendarId, () => import('@hcengineering/server-calendar-resources'))
addLocation(serverGmailId, () => import('@hcengineering/server-gmail-resources'))
addLocation(serverTelegramId, () => import('@hcengineering/server-telegram-resources'))
addLocation(serverHrId, () => import('@hcengineering/server-hr-resources'))
const middlewares: MiddlewareCreator[] = [ModifiedMiddleware.create, PrivateMiddleware.create]
return startJsonRpc(
getMetricsContext(),
(workspace: string) => {
const conf: DbConfiguration = {
domains: {
[DOMAIN_TX]: 'MongoTx',
[DOMAIN_TRANSIENT]: 'InMemory',
[DOMAIN_BLOB]: 'MinioData',
[DOMAIN_FULLTEXT_BLOB]: 'FullTextBlob',
[DOMAIN_MODEL]: 'Null'
},
defaultAdapter: 'Mongo',
adapters: {
MongoTx: {
factory: createMongoTxAdapter,
url: dbUrl
},
Mongo: {
factory: createMongoAdapter,
url: dbUrl
},
Null: {
factory: createNullAdapter,
url: ''
},
InMemory: {
factory: createInMemoryAdapter,
url: ''
},
MinioData: {
factory: createMinioDataAdapter,
url: ''
},
FullTextBlob: {
factory: createElasticBackupDataAdapter,
url: fullTextUrl
}
},
fulltextAdapter: {
factory: createElasticAdapter,
url: fullTextUrl
},
storageFactory: () =>
new MinioClient({
...minioConf,
port: 9000,
useSSL: false
}),
workspace
}
return createPipeline(conf, middlewares)
},
(token: Token, pipeline: Pipeline, broadcast: BroadcastCall) => {
if (token.extra?.mode === 'backup') {
return new BackupClientSession(broadcast, token, pipeline)
}
return new ClientSession(broadcast, token, pipeline)
},
port,
host
)
}

10
pods/server/tsconfig.json Normal file
View File

@ -0,0 +1,10 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"esModuleInterop": true,
"types": ["node"]
}
}

View File

@ -126,7 +126,7 @@
"@hcengineering/text-editor": "~0.6.0",
"@hcengineering/devmodel": "~0.6.0",
"@hcengineering/devmodel-resources": "~0.6.0",
"@hcengineering/front": "~0.6.0",
"@hcengineering/front": "^0.6.0",
"svelte-loader": "^3.1.3"
}
}

View File

@ -771,6 +771,16 @@
"projectFolder": "server/contrib",
"shouldPublish": true
},
{
"packageName": "@hcengineering/pod-front",
"projectFolder": "pods/front",
"shouldPublish": false
},
{
"packageName": "@hcengineering/pod-server",
"projectFolder": "pods/server",
"shouldPublish": false
},
{
"packageName": "@hcengineering/front",
"projectFolder": "server/front",
@ -1393,6 +1403,11 @@
"packageName": "@hcengineering/model-document",
"projectFolder": "models/document",
"shouldPublish": true
},
{
"packageName": "@hcengineering/apm",
"projectFolder": "tools/apm",
"shouldPublish": false
}
]
}

View File

@ -38,6 +38,7 @@
"ws": "^8.10.0",
"@hcengineering/model": "~0.6.0",
"@hcengineering/server-tool": "~0.6.0",
"@hcengineering/server-token": "~0.6.0"
"@hcengineering/server-token": "~0.6.0",
"@hcengineering/model-all": "~0.6.0"
}
}

View File

@ -14,11 +14,15 @@
// limitations under the License.
//
import { MongoClient, Db } from 'mongodb'
import { methods, getAccount, getWorkspace } from '..'
import builder, { migrateOperations, version } from '@hcengineering/model-all'
import { randomBytes } from 'crypto'
import { Db, MongoClient } from 'mongodb'
import { getAccount, getMethods, getWorkspace } from '..'
const DB_NAME = 'test_accounts'
const methods = getMethods(version, builder.getTxes(), migrateOperations, '')
describe('server', () => {
const dbUri = process.env.MONGODB_URI ?? 'mongodb://localhost:27017'
let conn: MongoClient

View File

@ -15,13 +15,14 @@
import contact, {
AvatarType,
combineName,
Employee,
buildGravatarId,
checkHasGravatar,
combineName,
Employee,
getAvatarColorForId
} from '@hcengineering/contact'
import core, { AccountRole, Ref, TxOperations } from '@hcengineering/core'
import core, { AccountRole, Data, Ref, Tx, TxOperations, Version } from '@hcengineering/core'
import { MigrateOperation } from '@hcengineering/model'
import platform, {
getMetadata,
PlatformError,
@ -34,7 +35,7 @@ import platform, {
StatusCode
} from '@hcengineering/platform'
import { decodeToken, generateToken } from '@hcengineering/server-token'
import toolPlugin, { connect, initModel, upgradeModel, version } from '@hcengineering/server-tool'
import toolPlugin, { connect, initModel, upgradeModel } from '@hcengineering/server-tool'
import { pbkdf2Sync, randomBytes } from 'crypto'
import { Binary, Db, ObjectId } from 'mongodb'
@ -61,7 +62,8 @@ const accountPlugin = plugin(accountId, {
WorkspaceNotFound: '' as StatusCode<{ workspace: string }>,
InvalidPassword: '' as StatusCode<{ account: string }>,
AccountAlreadyExists: '' as StatusCode<{ account: string }>,
WorkspaceAlreadyExists: '' as StatusCode<{ workspace: string }>
WorkspaceAlreadyExists: '' as StatusCode<{ workspace: string }>,
ProductIdMismatch: '' as StatusCode<{ productId: string }>
}
})
@ -102,6 +104,7 @@ export interface Workspace {
workspace: string
organisation: string
accounts: ObjectId[]
productId: string
}
/**
@ -333,8 +336,14 @@ export async function createAccount (
/**
* @public
*/
export async function listWorkspaces (db: Db): Promise<Workspace[]> {
return await db.collection<Workspace>(WORKSPACE_COLLECTION).find({}).toArray()
export async function listWorkspaces (db: Db, productId: string): Promise<Workspace[]> {
if (productId === '') {
return await db
.collection<Workspace>(WORKSPACE_COLLECTION)
.find({ productId: { $exits: false } })
.toArray()
}
return await db.collection<Workspace>(WORKSPACE_COLLECTION).find({ productId }).toArray()
}
/**
@ -347,7 +356,15 @@ export async function listAccounts (db: Db): Promise<Account[]> {
/**
* @public
*/
export async function createWorkspace (db: Db, workspace: string, organisation: string): Promise<string> {
export async function createWorkspace (
version: Data<Version>,
txes: Tx[],
migrationOperation: MigrateOperation[],
productId: string,
db: Db,
workspace: string,
organisation: string
): Promise<string> {
if ((await getWorkspace(db, workspace)) !== null) {
throw new PlatformError(new Status(Severity.ERROR, accountPlugin.status.WorkspaceAlreadyExists, { workspace }))
}
@ -356,45 +373,59 @@ export async function createWorkspace (db: Db, workspace: string, organisation:
.insertOne({
workspace,
organisation,
version
version,
productId
})
.then((e) => e.insertedId.toHexString())
await initModel(getTransactor(), workspace)
await initModel(getTransactor(), workspace, txes, migrationOperation)
return result
}
/**
* @public
*/
export async function upgradeWorkspace (db: Db, workspace: string): Promise<string> {
if ((await getWorkspace(db, workspace)) === null) {
export async function upgradeWorkspace (
version: Data<Version>,
txes: Tx[],
migrationOperation: MigrateOperation[],
productId: string,
db: Db,
workspace: string
): Promise<string> {
const ws = await getWorkspace(db, workspace)
if (ws === null) {
throw new PlatformError(new Status(Severity.ERROR, accountPlugin.status.WorkspaceNotFound, { workspace }))
}
if (ws.productId !== productId) {
throw new PlatformError(new Status(Severity.ERROR, accountPlugin.status.ProductIdMismatch, { productId }))
}
await db.collection(WORKSPACE_COLLECTION).updateOne(
{ workspace },
{
$set: { version }
}
)
await upgradeModel(getTransactor(), workspace)
await upgradeModel(getTransactor(), workspace, txes, migrationOperation)
return `${version.major}.${version.minor}.${version.patch}`
}
/**
* @public
*/
export async function createUserWorkspace (db: Db, token: string, workspace: string): Promise<LoginInfo> {
const { email } = decodeToken(token)
await createWorkspace(db, workspace, '')
await assignWorkspace(db, email, workspace)
await setRole(email, workspace, AccountRole.Owner)
const result = {
endpoint: getEndpoint(),
email,
token: generateToken(email, workspace)
}
return result
}
export const createUserWorkspace =
(version: Data<Version>, txes: Tx[], migrationOperation: MigrateOperation[], productId: string) =>
async (db: Db, token: string, workspace: string): Promise<LoginInfo> => {
const { email } = decodeToken(token)
await createWorkspace(version, txes, migrationOperation, productId, db, workspace, '')
await assignWorkspace(db, email, workspace)
await setRole(email, workspace, AccountRole.Owner)
const result = {
endpoint: getEndpoint(),
email,
token: generateToken(email, workspace)
}
return result
}
/**
* @public
@ -743,7 +774,12 @@ async function deactivateEmployeeAccount (email: string, workspace: string): Pro
}
}
function wrap (f: (db: Db, ...args: any[]) => Promise<any>) {
/**
* @public
*/
export type AccountMethod = (db: Db, request: Request<any[]>, token?: string) => Promise<Response<any>>
function wrap (f: (db: Db, ...args: any[]) => Promise<any>): AccountMethod {
return async function (db: Db, request: Request<any[]>, token?: string): Promise<Response<any>> {
if (token !== undefined) request.params.unshift(token)
return await f(db, ...request.params)
@ -760,24 +796,31 @@ function wrap (f: (db: Db, ...args: any[]) => Promise<any>) {
/**
* @public
*/
export const methods = {
login: wrap(login),
join: wrap(join),
checkJoin: wrap(checkJoin),
signUpJoin: wrap(signUpJoin),
selectWorkspace: wrap(selectWorkspace),
getUserWorkspaces: wrap(getUserWorkspaces),
getInviteLink: wrap(getInviteLink),
getAccountInfo: wrap(getAccountInfo),
createAccount: wrap(createAccount),
createWorkspace: wrap(createUserWorkspace),
assignWorkspace: wrap(assignWorkspace),
removeWorkspace: wrap(removeWorkspace),
leaveWorkspace: wrap(leaveWorkspace),
listWorkspaces: wrap(listWorkspaces),
changeName: wrap(changeName),
changePassword: wrap(changePassword)
// updateAccount: wrap(updateAccount)
export function getMethods (
version: Data<Version>,
txes: Tx[],
migrateOperations: MigrateOperation[],
productId: string
): Record<string, AccountMethod> {
return {
login: wrap(login),
join: wrap(join),
checkJoin: wrap(checkJoin),
signUpJoin: wrap(signUpJoin),
selectWorkspace: wrap(selectWorkspace),
getUserWorkspaces: wrap(getUserWorkspaces),
getInviteLink: wrap(getInviteLink),
getAccountInfo: wrap(getAccountInfo),
createAccount: wrap(createAccount),
createWorkspace: wrap(createUserWorkspace(version, txes, migrateOperations, productId)),
assignWorkspace: wrap(assignWorkspace),
removeWorkspace: wrap(removeWorkspace),
leaveWorkspace: wrap(leaveWorkspace),
listWorkspaces: wrap(listWorkspaces),
changeName: wrap(changeName),
changePassword: wrap(changePassword)
// updateAccount: wrap(updateAccount)
}
}
export default accountPlugin

View File

@ -8,13 +8,8 @@
"build": "heft build",
"build:watch": "tsc",
"lint:fix": "eslint --fix src",
"bundle": "esbuild src/__start.ts --define:process.env.MODEL_VERSION=$(node ../../models/all/lib/__showversion.js) --bundle --minify --platform=node --external:sharp > bundle.js & rm -rf ./dist && cp -r ../../dev/prod/dist . && cp -r ../../dev/prod/public/* ./dist/ && rm ./dist/config.json",
"docker:build": "docker build -t hardcoreeng/front .",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/front staging",
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/front",
"lint": "eslint src",
"format": "prettier --write src && eslint --fix src",
"run-local": "cross-env MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost TRANSACTOR_URL=ws:/localhost:3333 SERVER_SECRET='secret' ACCOUNTS_URL=http://localhost:3000 UPLOAD_URL=/files ELASTIC_URL=http://localhost:9200 MODEL_VERSION=$(node ../../models/all/lib/__showversion.js) PUBLIC_DIR='.' ts-node ./src/__start.ts"
"format": "prettier --write src && eslint --fix src"
},
"devDependencies": {
"@hcengineering/platform-rig": "~0.6.0",
@ -32,7 +27,6 @@
"@types/uuid": "^8.3.1",
"@types/cors": "^2.8.12",
"@types/minio": "~7.0.11",
"esbuild": "^0.12.26",
"prettier": "^2.7.1",
"@rushstack/heft": "^0.47.9",
"typescript": "^4.3.5",

View File

@ -17,7 +17,7 @@
import { Client } from 'minio'
import { setMetadata } from '@hcengineering/platform'
import serverToken from '@hcengineering/server-token'
import { start } from './app'
import { start } from '.'
const SERVER_PORT = parseInt(process.env.SERVER_PORT ?? '8080')

View File

@ -1,580 +0,0 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import attachment from '@hcengineering/attachment'
import { Account, Doc, Ref, Space } from '@hcengineering/core'
import { createElasticAdapter } from '@hcengineering/elastic'
import type { IndexedDoc } from '@hcengineering/server-core'
import { decodeToken, Token } from '@hcengineering/server-token'
import bp from 'body-parser'
import compression from 'compression'
import cors from 'cors'
import express, { Response } from 'express'
import fileUpload, { UploadedFile } from 'express-fileupload'
import https from 'https'
import { BucketItem, Client, ItemBucketMetadata } from 'minio'
import { join, resolve } from 'path'
import { v4 as uuid } from 'uuid'
import sharp from 'sharp'
async function minioUpload (minio: Client, workspace: string, file: UploadedFile): Promise<string> {
const id = uuid()
const meta: ItemBucketMetadata = {
'Content-Type': file.mimetype
}
const resp = await minio.putObject(workspace, id, file.data, file.size, meta)
console.log(resp)
return id
}
async function readMinioData (client: Client, db: string, name: string): Promise<Buffer[]> {
const data = await client.getObject(db, name)
const chunks: Buffer[] = []
await new Promise((resolve) => {
data.on('readable', () => {
let chunk
while ((chunk = data.read()) !== null) {
const b = chunk as Buffer
chunks.push(b)
}
})
data.on('end', () => {
resolve(null)
})
})
return chunks
}
function getRange (range: string, size: number): [number, number] {
const [startStr, endStr] = range.replace(/bytes=/, '').split('-')
let start = parseInt(startStr, 10)
let end = endStr !== undefined ? parseInt(endStr, 10) : size - 1
if (!isNaN(start) && isNaN(end)) {
end = size - 1
}
if (isNaN(start) && !isNaN(end)) {
start = size - end
end = size - 1
}
return [start, end]
}
async function getFileRange (
range: string,
client: Client,
workspace: string,
uuid: string,
res: Response
): Promise<void> {
const stat = await client.statObject(workspace, uuid)
const size = stat.size
const [start, end] = getRange(range, size)
if (start >= size || end >= size) {
res.writeHead(416, {
'Content-Range': `bytes */${size}`
})
res.end()
return
}
client.getPartialObject(workspace, uuid, start, end - start + 1, (err, dataStream) => {
if (err !== null) {
console.log(err)
res.status(500).send()
return
}
res.writeHead(206, {
Connection: 'keep-alive',
'Content-Range': `bytes ${start}-${end}/${size}`,
'Accept-Ranges': 'bytes',
'Content-Length': end - start + 1,
'Content-Type': stat.metaData['content-type']
})
dataStream.pipe(res)
})
}
async function getFile (client: Client, workspace: string, uuid: string, res: Response): Promise<void> {
const stat = await client.statObject(workspace, uuid)
client.getObject(workspace, uuid, (err, dataStream) => {
if (err !== null) {
console.log(err)
res.status(500).send()
return
}
res.status(200)
res.set('Cache-Control', 'max-age=604800')
const contentType = stat.metaData['content-type']
if (contentType !== undefined) {
res.setHeader('Content-Type', contentType)
}
dataStream.on('data', function (chunk) {
res.write(chunk)
})
dataStream.on('end', function () {
res.end()
})
dataStream.on('error', function (err) {
console.log(err)
res.status(500).send()
})
})
}
/**
* @public
* @param port -
*/
export function start (
config: {
transactorEndpoint: string
elasticUrl: string
minio: Client
accountsUrl: string
uploadUrl: string
modelVersion: string
collaboratorUrl: string
},
port: number
): () => void {
const app = express()
app.use(
compression({
filter: (req, res) => {
if (req.headers['x-no-compression'] != null) {
// don't compress responses with this request header
return false
}
// fallback to standard filter function
return compression.filter(req, res)
}
})
)
app.use(cors())
app.use(fileUpload())
app.use(bp.json())
app.use(bp.urlencoded({ extended: true }))
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.get('/config.json', async (req, res) => {
res.status(200)
res.set('Cache-Control', 'no-cache')
res.json({
ACCOUNTS_URL: config.accountsUrl,
UPLOAD_URL: config.uploadUrl,
MODEL_VERSION: config.modelVersion,
COLLABORATOR_URL: config.collaboratorUrl
})
})
const dist = resolve(process.env.PUBLIC_DIR ?? __dirname, 'dist')
console.log('serving static files from', dist)
app.use(express.static(dist, { maxAge: '168h' }))
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.head('/files', async (req, res: Response) => {
try {
const token = req.query.token as string
const payload = decodeToken(token)
let uuid = req.query.file as string
const size = req.query.size as 'inline' | 'tiny' | 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'
uuid = await getResizeID(size, uuid, config, payload)
const stat = await config.minio.statObject(payload.workspace, uuid)
const fileSize = stat.size
res.status(200)
res.setHeader('accept-ranges', 'bytes')
res.setHeader('content-length', fileSize)
res.end()
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.get('/files', async (req, res: Response) => {
try {
const token = req.query.token as string
const payload = decodeToken(token)
let uuid = req.query.file as string
const size = req.query.size as 'inline' | 'tiny' | 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'
uuid = await getResizeID(size, uuid, config, payload)
const range = req.headers.range
if (range !== undefined) {
await getFileRange(range, config.minio, payload.workspace, uuid, res)
} else {
await getFile(config.minio, payload.workspace, uuid, res)
}
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.post('/files', async (req, res) => {
const file = req.files?.file as UploadedFile
if (file === undefined) {
res.status(400).send()
return
}
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
try {
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const uuid = await minioUpload(config.minio, payload.workspace, file)
console.log('uploaded uuid', uuid)
const space = req.query.space as Ref<Space> | undefined
const attachedTo = req.query.attachedTo as Ref<Doc> | undefined
if (space !== undefined && attachedTo !== undefined) {
const elastic = await createElasticAdapter(config.elasticUrl, payload.workspace)
const indexedDoc: IndexedDoc = {
id: uuid as Ref<Doc>,
_class: attachment.class.Attachment,
space,
modifiedOn: Date.now(),
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo,
data: file.data.toString('base64')
}
await elastic.index(indexedDoc)
}
res.status(200).send(uuid)
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.delete('/files', async (req, res) => {
try {
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const uuid = req.query.file as string
await config.minio.removeObject(payload.workspace, uuid)
const extra = await listMinioObjects(config.minio, payload.workspace, uuid)
if (extra.size > 0) {
for (const e of extra.entries()) {
await config.minio.removeObject(payload.workspace, e[1].name)
}
}
res.status(200).send()
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// todo remove it after update all customers chrome extensions
app.get('/import', (req, res) => {
try {
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const url = req.query.url as string
const cookie = req.query.cookie as string | undefined
const attachedTo = req.query.attachedTo as Ref<Doc> | undefined
if (url === undefined) {
res.status(500).send('URL param is not defined')
return
}
console.log('importing from', url)
console.log('cookie', cookie)
const options =
cookie !== undefined
? {
headers: {
Cookie: cookie
}
}
: {}
https.get(url, options, (response) => {
console.log('status', response.statusCode)
if (response.statusCode !== 200) {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
res.status(500).send(`server returned ${response.statusCode}`)
return
}
const id = uuid()
const contentType = response.headers['content-type']
const meta: ItemBucketMetadata = {
'Content-Type': contentType
}
const data: Buffer[] = []
response
.on('data', function (chunk) {
data.push(chunk)
})
.on('end', function () {
const buffer = Buffer.concat(data)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
config.minio.putObject(payload.workspace, id, buffer, 0, meta, async (err, objInfo) => {
if (err !== null) {
console.log('minio putObject error', err)
res.status(500).send(err)
} else {
console.log('uploaded uuid', id)
if (attachedTo !== undefined) {
const space = req.query.space as Ref<Space>
const elastic = await createElasticAdapter(config.elasticUrl, payload.workspace)
const indexedDoc: IndexedDoc = {
id: id as Ref<Doc>,
_class: attachment.class.Attachment,
space,
modifiedOn: Date.now(),
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo,
data: buffer.toString('base64')
}
await elastic.index(indexedDoc)
}
res.status(200).send({
id,
contentType,
size: buffer.length
})
}
})
})
.on('error', function (err) {
res.status(500).send(err)
})
})
} catch (error) {
console.log(error)
res.status(500).send()
}
})
app.post('/import', (req, res) => {
try {
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const { url, cookie, attachedTo, space } = req.body
if (url === undefined) {
res.status(500).send('URL param is not defined')
return
}
console.log('importing from', url)
console.log('cookie', cookie)
const options =
cookie !== undefined
? {
headers: {
Cookie: cookie
}
}
: {}
https.get(url, options, (response) => {
console.log('status', response.statusCode)
if (response.statusCode !== 200) {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
res.status(500).send(`server returned ${response.statusCode}`)
return
}
const id = uuid()
const contentType = response.headers['content-type']
const meta: ItemBucketMetadata = {
'Content-Type': contentType
}
const data: Buffer[] = []
response
.on('data', function (chunk) {
data.push(chunk)
})
.on('end', function () {
const buffer = Buffer.concat(data)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
config.minio.putObject(payload.workspace, id, buffer, 0, meta, async (err, objInfo) => {
if (err !== null) {
console.log('minio putObject error', err)
res.status(500).send(err)
} else {
console.log('uploaded uuid', id)
if (attachedTo !== undefined) {
const elastic = await createElasticAdapter(config.elasticUrl, payload.workspace)
const indexedDoc: IndexedDoc = {
id: id as Ref<Doc>,
_class: attachment.class.Attachment,
space,
modifiedOn: Date.now(),
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo,
data: buffer.toString('base64')
}
await elastic.index(indexedDoc)
}
res.status(200).send({
id,
contentType,
size: buffer.length
})
}
})
})
.on('error', function (err) {
res.status(500).send(err)
})
})
} catch (error) {
console.log(error)
res.status(500).send()
}
})
app.get('*', function (request, response) {
response.sendFile(join(dist, 'index.html'))
})
const server = app.listen(port)
return () => {
server.close()
}
}
async function getResizeID (size: string, uuid: string, config: { minio: Client }, payload: Token): Promise<string> {
if (size !== undefined && size !== 'full') {
let width = 64
switch (size) {
case 'inline':
case 'tiny':
case 'x-small':
case 'small':
case 'medium':
width = 64
break
case 'large':
width = 256
break
case 'x-large':
width = 512
break
}
let hasSmall = false
const sizeId = uuid + `%size%${width}`
try {
const d = await config.minio.statObject(payload.workspace, sizeId)
hasSmall = d !== undefined && d.size > 0
} catch (err) {}
if (hasSmall) {
// We have cached small document, let's proceed with it.
uuid = sizeId
} else {
// Let's get data and resize it
const data = Buffer.concat(await readMinioData(config.minio, payload.workspace, uuid))
const dataBuff = await sharp(data)
.resize({
width
})
.jpeg()
.toBuffer()
await config.minio.putObject(payload.workspace, sizeId, dataBuff, {
'Content-Type': 'image/jpeg'
})
uuid = sizeId
}
}
return uuid
}
async function listMinioObjects (
client: Client,
db: string,
prefix: string
): Promise<Map<string, BucketItem & { metaData: ItemBucketMetadata }>> {
const items = new Map<string, BucketItem & { metaData: ItemBucketMetadata }>()
const list = await client.listObjects(db, prefix, true)
await new Promise((resolve) => {
list.on('data', (data) => {
items.set(data.name, { metaData: {}, ...data })
})
list.on('end', () => {
resolve(null)
})
})
return items
}

View File

@ -14,28 +14,567 @@
// limitations under the License.
//
import express from 'express'
import { resolve, join } from 'path'
import attachment from '@hcengineering/attachment'
import { Account, Doc, Ref, Space } from '@hcengineering/core'
import { createElasticAdapter } from '@hcengineering/elastic'
import type { IndexedDoc } from '@hcengineering/server-core'
import { decodeToken, Token } from '@hcengineering/server-token'
import bp from 'body-parser'
import compression from 'compression'
import cors from 'cors'
import express, { Response } from 'express'
import fileUpload, { UploadedFile } from 'express-fileupload'
import https from 'https'
import { BucketItem, Client, ItemBucketMetadata } from 'minio'
import { join, resolve } from 'path'
import { v4 as uuid } from 'uuid'
import sharp from 'sharp'
const port = process.env.PORT ?? 8080
const app = express()
async function minioUpload (minio: Client, workspace: string, file: UploadedFile): Promise<string> {
const id = uuid()
const meta: ItemBucketMetadata = {
'Content-Type': file.mimetype
}
const dist = resolve(__dirname, 'dist')
const resp = await minio.putObject(workspace, id, file.data, file.size, meta)
console.log('serving static files from', dist)
app.use(express.static(dist, { maxAge: '10m' }))
app.get('*', function (request, response) {
response.sendFile(join(dist, 'index.html'))
})
const server = app.listen(port)
console.log(`server started on port ${port}`)
const close = (): void => {
server.close()
console.log(resp)
return id
}
async function readMinioData (client: Client, db: string, name: string): Promise<Buffer[]> {
const data = await client.getObject(db, name)
const chunks: Buffer[] = []
await new Promise((resolve) => {
data.on('readable', () => {
let chunk
while ((chunk = data.read()) !== null) {
const b = chunk as Buffer
chunks.push(b)
}
})
data.on('end', () => {
resolve(null)
})
})
return chunks
}
function getRange (range: string, size: number): [number, number] {
const [startStr, endStr] = range.replace(/bytes=/, '').split('-')
let start = parseInt(startStr, 10)
let end = endStr !== undefined ? parseInt(endStr, 10) : size - 1
if (!isNaN(start) && isNaN(end)) {
end = size - 1
}
if (isNaN(start) && !isNaN(end)) {
start = size - end
end = size - 1
}
return [start, end]
}
async function getFileRange (
range: string,
client: Client,
workspace: string,
uuid: string,
res: Response
): Promise<void> {
const stat = await client.statObject(workspace, uuid)
const size = stat.size
const [start, end] = getRange(range, size)
if (start >= size || end >= size) {
res.writeHead(416, {
'Content-Range': `bytes */${size}`
})
res.end()
return
}
client.getPartialObject(workspace, uuid, start, end - start + 1, (err, dataStream) => {
if (err !== null) {
console.log(err)
res.status(500).send()
return
}
res.writeHead(206, {
Connection: 'keep-alive',
'Content-Range': `bytes ${start}-${end}/${size}`,
'Accept-Ranges': 'bytes',
'Content-Length': end - start + 1,
'Content-Type': stat.metaData['content-type']
})
dataStream.pipe(res)
})
}
async function getFile (client: Client, workspace: string, uuid: string, res: Response): Promise<void> {
const stat = await client.statObject(workspace, uuid)
client.getObject(workspace, uuid, (err, dataStream) => {
if (err !== null) {
console.log(err)
res.status(500).send()
return
}
res.status(200)
res.set('Cache-Control', 'max-age=604800')
const contentType = stat.metaData['content-type']
if (contentType !== undefined) {
res.setHeader('Content-Type', contentType)
}
dataStream.on('data', function (chunk) {
res.write(chunk)
})
dataStream.on('end', function () {
res.end()
})
dataStream.on('error', function (err) {
console.log(err)
res.status(500).send()
})
})
}
/**
* @public
* @param port -
*/
export function start (
config: {
transactorEndpoint: string
elasticUrl: string
minio: Client
accountsUrl: string
uploadUrl: string
modelVersion: string
collaboratorUrl: string
},
port: number
): () => void {
const app = express()
app.use(
compression({
filter: (req, res) => {
if (req.headers['x-no-compression'] != null) {
// don't compress responses with this request header
return false
}
// fallback to standard filter function
return compression.filter(req, res)
}
})
)
app.use(cors())
app.use(fileUpload())
app.use(bp.json())
app.use(bp.urlencoded({ extended: true }))
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.get('/config.json', async (req, res) => {
res.status(200)
res.set('Cache-Control', 'no-cache')
res.json({
ACCOUNTS_URL: config.accountsUrl,
UPLOAD_URL: config.uploadUrl,
MODEL_VERSION: config.modelVersion,
COLLABORATOR_URL: config.collaboratorUrl
})
})
const dist = resolve(process.env.PUBLIC_DIR ?? __dirname, 'dist')
console.log('serving static files from', dist)
app.use(express.static(dist, { maxAge: '168h' }))
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.head('/files', async (req, res: Response) => {
try {
const token = req.query.token as string
const payload = decodeToken(token)
let uuid = req.query.file as string
const size = req.query.size as 'inline' | 'tiny' | 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'
uuid = await getResizeID(size, uuid, config, payload)
const stat = await config.minio.statObject(payload.workspace, uuid)
const fileSize = stat.size
res.status(200)
res.setHeader('accept-ranges', 'bytes')
res.setHeader('content-length', fileSize)
res.end()
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.get('/files', async (req, res: Response) => {
try {
const token = req.query.token as string
const payload = decodeToken(token)
let uuid = req.query.file as string
const size = req.query.size as 'inline' | 'tiny' | 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'
uuid = await getResizeID(size, uuid, config, payload)
const range = req.headers.range
if (range !== undefined) {
await getFileRange(range, config.minio, payload.workspace, uuid, res)
} else {
await getFile(config.minio, payload.workspace, uuid, res)
}
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.post('/files', async (req, res) => {
const file = req.files?.file as UploadedFile
if (file === undefined) {
res.status(400).send()
return
}
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
try {
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const uuid = await minioUpload(config.minio, payload.workspace, file)
console.log('uploaded uuid', uuid)
const space = req.query.space as Ref<Space> | undefined
const attachedTo = req.query.attachedTo as Ref<Doc> | undefined
if (space !== undefined && attachedTo !== undefined) {
const elastic = await createElasticAdapter(config.elasticUrl, payload.workspace)
const indexedDoc: IndexedDoc = {
id: uuid as Ref<Doc>,
_class: attachment.class.Attachment,
space,
modifiedOn: Date.now(),
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo,
data: file.data.toString('base64')
}
await elastic.index(indexedDoc)
}
res.status(200).send(uuid)
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.delete('/files', async (req, res) => {
try {
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const uuid = req.query.file as string
await config.minio.removeObject(payload.workspace, uuid)
const extra = await listMinioObjects(config.minio, payload.workspace, uuid)
if (extra.size > 0) {
for (const e of extra.entries()) {
await config.minio.removeObject(payload.workspace, e[1].name)
}
}
res.status(200).send()
} catch (error) {
console.log(error)
res.status(500).send()
}
})
// todo remove it after update all customers chrome extensions
app.get('/import', (req, res) => {
try {
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const url = req.query.url as string
const cookie = req.query.cookie as string | undefined
const attachedTo = req.query.attachedTo as Ref<Doc> | undefined
if (url === undefined) {
res.status(500).send('URL param is not defined')
return
}
console.log('importing from', url)
console.log('cookie', cookie)
const options =
cookie !== undefined
? {
headers: {
Cookie: cookie
}
}
: {}
https.get(url, options, (response) => {
console.log('status', response.statusCode)
if (response.statusCode !== 200) {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
res.status(500).send(`server returned ${response.statusCode}`)
return
}
const id = uuid()
const contentType = response.headers['content-type']
const meta: ItemBucketMetadata = {
'Content-Type': contentType
}
const data: Buffer[] = []
response
.on('data', function (chunk) {
data.push(chunk)
})
.on('end', function () {
const buffer = Buffer.concat(data)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
config.minio.putObject(payload.workspace, id, buffer, 0, meta, async (err, objInfo) => {
if (err !== null) {
console.log('minio putObject error', err)
res.status(500).send(err)
} else {
console.log('uploaded uuid', id)
if (attachedTo !== undefined) {
const space = req.query.space as Ref<Space>
const elastic = await createElasticAdapter(config.elasticUrl, payload.workspace)
const indexedDoc: IndexedDoc = {
id: id as Ref<Doc>,
_class: attachment.class.Attachment,
space,
modifiedOn: Date.now(),
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo,
data: buffer.toString('base64')
}
await elastic.index(indexedDoc)
}
res.status(200).send({
id,
contentType,
size: buffer.length
})
}
})
})
.on('error', function (err) {
res.status(500).send(err)
})
})
} catch (error) {
console.log(error)
res.status(500).send()
}
})
app.post('/import', (req, res) => {
try {
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const payload = decodeToken(token)
const { url, cookie, attachedTo, space } = req.body
if (url === undefined) {
res.status(500).send('URL param is not defined')
return
}
console.log('importing from', url)
console.log('cookie', cookie)
const options =
cookie !== undefined
? {
headers: {
Cookie: cookie
}
}
: {}
https.get(url, options, (response) => {
console.log('status', response.statusCode)
if (response.statusCode !== 200) {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
res.status(500).send(`server returned ${response.statusCode}`)
return
}
const id = uuid()
const contentType = response.headers['content-type']
const meta: ItemBucketMetadata = {
'Content-Type': contentType
}
const data: Buffer[] = []
response
.on('data', function (chunk) {
data.push(chunk)
})
.on('end', function () {
const buffer = Buffer.concat(data)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
config.minio.putObject(payload.workspace, id, buffer, 0, meta, async (err, objInfo) => {
if (err !== null) {
console.log('minio putObject error', err)
res.status(500).send(err)
} else {
console.log('uploaded uuid', id)
if (attachedTo !== undefined) {
const elastic = await createElasticAdapter(config.elasticUrl, payload.workspace)
const indexedDoc: IndexedDoc = {
id: id as Ref<Doc>,
_class: attachment.class.Attachment,
space,
modifiedOn: Date.now(),
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo,
data: buffer.toString('base64')
}
await elastic.index(indexedDoc)
}
res.status(200).send({
id,
contentType,
size: buffer.length
})
}
})
})
.on('error', function (err) {
res.status(500).send(err)
})
})
} catch (error) {
console.log(error)
res.status(500).send()
}
})
app.get('*', function (request, response) {
response.sendFile(join(dist, 'index.html'))
})
const server = app.listen(port)
return () => {
server.close()
}
}
async function getResizeID (size: string, uuid: string, config: { minio: Client }, payload: Token): Promise<string> {
if (size !== undefined && size !== 'full') {
let width = 64
switch (size) {
case 'inline':
case 'tiny':
case 'x-small':
case 'small':
case 'medium':
width = 64
break
case 'large':
width = 256
break
case 'x-large':
width = 512
break
}
let hasSmall = false
const sizeId = uuid + `%size%${width}`
try {
const d = await config.minio.statObject(payload.workspace, sizeId)
hasSmall = d !== undefined && d.size > 0
} catch (err) {}
if (hasSmall) {
// We have cached small document, let's proceed with it.
uuid = sizeId
} else {
// Let's get data and resize it
const data = Buffer.concat(await readMinioData(config.minio, payload.workspace, uuid))
const dataBuff = await sharp(data)
.resize({
width
})
.jpeg()
.toBuffer()
await config.minio.putObject(payload.workspace, sizeId, dataBuff, {
'Content-Type': 'image/jpeg'
})
uuid = sizeId
}
}
return uuid
}
async function listMinioObjects (
client: Client,
db: string,
prefix: string
): Promise<Map<string, BucketItem & { metaData: ItemBucketMetadata }>> {
const items = new Map<string, BucketItem & { metaData: ItemBucketMetadata }>()
const list = await client.listObjects(db, prefix, true)
await new Promise((resolve) => {
list.on('data', (data) => {
items.set(data.name, { metaData: {}, ...data })
})
list.on('end', () => {
resolve(null)
})
})
return items
}
process.on('SIGINT', close)
process.on('SIGTERM', close)
process.on('exit', close)

View File

@ -1,26 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "nginx"
cert-manager.io/issuer: "letsencrypt-prod"
nginx.org/proxy-read-timeout: "3h"
nginx.org/proxy-send-timeout: "3h"
name: transactor-ingress
spec:
tls:
- hosts:
- transactor.hc.engineering
secretName: transactor-tls
rules:
- host: transactor.hc.engineering
http:
paths:
- backend:
service:
name: transactor
port:
number: 80
path: /
pathType: Prefix

View File

@ -1,60 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: transactor
spec:
replicas: 1
selector:
matchLabels:
app: transactor
template:
metadata:
labels:
app: transactor
spec:
containers:
- name: app
image: hardcoreeng/transactor
ports:
- containerPort: 3333
imagePullPolicy: Always
env:
- name: FRONT_URL
value: https://front.hc.engineering/
- name: MONGO_URL
valueFrom:
secretKeyRef:
name: mongodb
key: url
- name: ELASTIC_URL
valueFrom:
secretKeyRef:
name: elastic
key: url
- name: MINIO_ENDPOINT
valueFrom:
secretKeyRef:
name: minio
key: endpoint
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio
key: accessKey
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio
key: secretKey
---
apiVersion: v1
kind: Service
metadata:
name: transactor
spec:
selector:
app: transactor
ports:
- port: 80
targetPort: 3333

View File

@ -5,13 +5,8 @@
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
"scripts": {
"start": "cross-env MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret ts-node src/__start.ts",
"build": "heft build",
"lint:fix": "eslint --fix src",
"bundle": "esbuild src/__start.ts --bundle --platform=node > bundle.js",
"docker:build": "docker build -t hardcoreeng/transactor .",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/transactor staging",
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/transactor",
"build:watch": "tsc",
"lint": "eslint src",
"format": "prettier --write src && eslint --fix src"
@ -28,7 +23,6 @@
"eslint": "^8.26.0",
"@types/ws": "^8.5.3",
"ts-node": "^10.8.0",
"esbuild": "^0.12.26",
"@typescript-eslint/parser": "^5.41.0",
"eslint-config-standard-with-typescript": "^23.0.0",
"prettier": "^2.7.1",
@ -41,41 +35,10 @@
"@hcengineering/platform": "^0.6.7",
"@hcengineering/server-core": "~0.6.1",
"@hcengineering/server-ws": "~0.6.11",
"@hcengineering/server-attachment": "~0.6.1",
"@hcengineering/server-attachment-resources": "~0.6.0",
"@hcengineering/mongo": "~0.6.1",
"@hcengineering/elastic": "~0.6.0",
"elastic-apm-node": "~3.26.0",
"minio": "^7.0.26",
"@hcengineering/server-contact": "~0.6.1",
"@hcengineering/server-contact-resources": "~0.6.0",
"@hcengineering/server-notification": "^0.6.0",
"@hcengineering/server-notification-resources": "~0.6.0",
"@hcengineering/server-setting": "~0.6.0",
"@hcengineering/server-setting-resources": "~0.6.0",
"@hcengineering/server-chunter": "~0.6.0",
"@hcengineering/server-chunter-resources": "~0.6.0",
"@hcengineering/server-inventory": "~0.6.0",
"@hcengineering/server-inventory-resources": "~0.6.0",
"@hcengineering/server-lead": "~0.6.0",
"@hcengineering/server-lead-resources": "~0.6.0",
"@hcengineering/server-recruit": "~0.6.0",
"@hcengineering/server-recruit-resources": "~0.6.0",
"@hcengineering/server-task": "~0.6.0",
"@hcengineering/server-task-resources": "~0.6.0",
"@hcengineering/server-tracker": "~0.6.0",
"@hcengineering/server-tracker-resources": "~0.6.0",
"@hcengineering/server-tags": "~0.6.0",
"@hcengineering/server-tags-resources": "~0.6.0",
"@hcengineering/server-calendar": "~0.6.0",
"@hcengineering/server-calendar-resources": "~0.6.0",
"@hcengineering/server-gmail": "~0.6.0",
"@hcengineering/server-gmail-resources": "~0.6.0",
"@hcengineering/server-preference": "~0.6.0",
"@hcengineering/server-telegram": "~0.6.0",
"@hcengineering/server-telegram-resources": "~0.6.0",
"@hcengineering/server-hr": "~0.6.0",
"@hcengineering/server-hr-resources": "~0.6.0",
"minio": "^7.0.26",
"@hcengineering/server-token": "~0.6.0",
"@hcengineering/middleware": "~0.6.0"
}

View File

@ -1,8 +1,6 @@
import { MeasureContext, MeasureLogger, ParamType } from '@hcengineering/core'
import apm, { Agent, Span, Transaction } from 'elastic-apm-node'
export let metricsContext: MeasureContext
/**
* @public
*/

View File

@ -5,7 +5,10 @@ import { BroadcastCall, ClientSession, Session } from '@hcengineering/server-ws'
const chunkSize = 1024 * 1024
interface ChunkInfo {
/**
* @public
*/
export interface ChunkInfo {
idx: number
index: 0
finished: boolean
@ -21,6 +24,9 @@ export interface BackupSession extends Session {
loadDocs: (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]) => Promise<Doc[]>
}
/**
* @public
*/
export class BackupClientSession extends ClientSession implements BackupSession {
constructor (
protected readonly broadcast: BroadcastCall,

View File

@ -14,5 +14,8 @@
// limitations under the License.
//
export { start } from './server'
export type { MinioConfig } from './server'
export * from './server'
export * from './apm'
export * from './minio'
export * from './backup'
export * from './metrics'

View File

@ -8,38 +8,50 @@ const metricsConsole = (process.env.METRICS_CONSOLE ?? 'false') === 'true'
const METRICS_UPDATE_INTERVAL = 30000
export let metricsContext: MeasureContext
/**
* @public
*/
let metricsContext: MeasureContext | undefined
if (apmUrl === undefined) {
console.info('please provide apm server url for monitoring')
const metrics = newMetrics()
metricsContext = new MeasureMetricsContext('System', {}, metrics)
if (metricsFile !== undefined || metricsConsole) {
console.info('storing measurements into local file', metricsFile)
let oldMetricsValue = ''
const intTimer = setInterval(() => {
const val = metricsToString(metrics)
if (val !== oldMetricsValue) {
oldMetricsValue = val
if (metricsFile !== undefined) {
writeFile(metricsFile, val).catch((err) => console.error(err))
}
if (metricsConsole) {
console.info('METRICS:', val)
}
}
}, METRICS_UPDATE_INTERVAL)
const closeTimer = (): void => {
clearInterval(intTimer)
}
process.on('SIGINT', closeTimer)
process.on('SIGTERM', closeTimer)
/**
* @public
*/
export function getMetricsContext (): MeasureContext {
if (metricsContext !== undefined) {
return metricsContext
}
} else {
console.log('using APM', apmUrl)
metricsContext = new APMMeasureContext(createAPMAgent(apmUrl), 'root', {}, undefined, true)
if (apmUrl === undefined) {
console.info('please provide apm server url for monitoring')
const metrics = newMetrics()
metricsContext = new MeasureMetricsContext('System', {}, metrics)
if (metricsFile !== undefined || metricsConsole) {
console.info('storing measurements into local file', metricsFile)
let oldMetricsValue = ''
const intTimer = setInterval(() => {
const val = metricsToString(metrics)
if (val !== oldMetricsValue) {
oldMetricsValue = val
if (metricsFile !== undefined) {
writeFile(metricsFile, val).catch((err) => console.error(err))
}
if (metricsConsole) {
console.info('METRICS:', val)
}
}
}, METRICS_UPDATE_INTERVAL)
const closeTimer = (): void => {
clearInterval(intTimer)
}
process.on('SIGINT', closeTimer)
process.on('SIGTERM', closeTimer)
}
} else {
console.log('using APM', apmUrl)
metricsContext = new APMMeasureContext(createAPMAgent(apmUrl), 'root', {}, undefined, true)
}
return metricsContext
}

View File

@ -18,11 +18,6 @@ import {
Doc,
DocumentQuery,
Domain,
DOMAIN_BLOB,
DOMAIN_FULLTEXT_BLOB,
DOMAIN_MODEL,
DOMAIN_TRANSIENT,
DOMAIN_TX,
FindOptions,
FindResult,
Hierarchy,
@ -33,39 +28,7 @@ import {
Tx,
TxResult
} from '@hcengineering/core'
import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic'
import { ModifiedMiddleware, PrivateMiddleware } from '@hcengineering/middleware'
import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import { addLocation } from '@hcengineering/platform'
import { serverAttachmentId } from '@hcengineering/server-attachment'
import { serverCalendarId } from '@hcengineering/server-calendar'
import { serverChunterId } from '@hcengineering/server-chunter'
import { serverContactId } from '@hcengineering/server-contact'
import {
createInMemoryAdapter,
createPipeline,
DbAdapter,
DbConfiguration,
MiddlewareCreator,
Pipeline
} from '@hcengineering/server-core'
import { serverGmailId } from '@hcengineering/server-gmail'
import { serverInventoryId } from '@hcengineering/server-inventory'
import { serverLeadId } from '@hcengineering/server-lead'
import { serverNotificationId } from '@hcengineering/server-notification'
import { serverRecruitId } from '@hcengineering/server-recruit'
import { serverSettingId } from '@hcengineering/server-setting'
import { serverTagsId } from '@hcengineering/server-tags'
import { serverTaskId } from '@hcengineering/server-task'
import { serverTrackerId } from '@hcengineering/server-tracker'
import { serverTelegramId } from '@hcengineering/server-telegram'
import { serverHrId } from '@hcengineering/server-hr'
import { Token } from '@hcengineering/server-token'
import { BroadcastCall, ClientSession, start as startJsonRpc } from '@hcengineering/server-ws'
import { Client as MinioClient } from 'minio'
import { BackupClientSession } from './backup'
import { metricsContext } from './metrics'
import { createMinioDataAdapter } from './minio'
import { DbAdapter } from '@hcengineering/server-core'
class NullDbAdapter implements DbAdapter {
async init (model: Tx[]): Promise<void> {}
@ -99,7 +62,15 @@ class NullDbAdapter implements DbAdapter {
async clean (domain: Domain, docs: Ref<Doc>[]): Promise<void> {}
}
async function createNullAdapter (hierarchy: Hierarchy, url: string, db: string, modelDb: ModelDb): Promise<DbAdapter> {
/**
* @public
*/
export async function createNullAdapter (
hierarchy: Hierarchy,
url: string,
db: string,
modelDb: ModelDb
): Promise<DbAdapter> {
return new NullDbAdapter()
}
@ -111,94 +82,3 @@ export interface MinioConfig {
accessKey: string
secretKey: string
}
/**
* @public
*/
export function start (
dbUrl: string,
fullTextUrl: string,
minioConf: MinioConfig,
port: number,
host?: string
): () => void {
addLocation(serverAttachmentId, () => import('@hcengineering/server-attachment-resources'))
addLocation(serverContactId, () => import('@hcengineering/server-contact-resources'))
addLocation(serverNotificationId, () => import('@hcengineering/server-notification-resources'))
addLocation(serverSettingId, () => import('@hcengineering/server-setting-resources'))
addLocation(serverChunterId, () => import('@hcengineering/server-chunter-resources'))
addLocation(serverInventoryId, () => import('@hcengineering/server-inventory-resources'))
addLocation(serverLeadId, () => import('@hcengineering/server-lead-resources'))
addLocation(serverRecruitId, () => import('@hcengineering/server-recruit-resources'))
addLocation(serverTaskId, () => import('@hcengineering/server-task-resources'))
addLocation(serverTrackerId, () => import('@hcengineering/server-tracker-resources'))
addLocation(serverTagsId, () => import('@hcengineering/server-tags-resources'))
addLocation(serverCalendarId, () => import('@hcengineering/server-calendar-resources'))
addLocation(serverGmailId, () => import('@hcengineering/server-gmail-resources'))
addLocation(serverTelegramId, () => import('@hcengineering/server-telegram-resources'))
addLocation(serverHrId, () => import('@hcengineering/server-hr-resources'))
const middlewares: MiddlewareCreator[] = [ModifiedMiddleware.create, PrivateMiddleware.create]
return startJsonRpc(
metricsContext,
(workspace: string) => {
const conf: DbConfiguration = {
domains: {
[DOMAIN_TX]: 'MongoTx',
[DOMAIN_TRANSIENT]: 'InMemory',
[DOMAIN_BLOB]: 'MinioData',
[DOMAIN_FULLTEXT_BLOB]: 'FullTextBlob',
[DOMAIN_MODEL]: 'Null'
},
defaultAdapter: 'Mongo',
adapters: {
MongoTx: {
factory: createMongoTxAdapter,
url: dbUrl
},
Mongo: {
factory: createMongoAdapter,
url: dbUrl
},
Null: {
factory: createNullAdapter,
url: ''
},
InMemory: {
factory: createInMemoryAdapter,
url: ''
},
MinioData: {
factory: createMinioDataAdapter,
url: ''
},
FullTextBlob: {
factory: createElasticBackupDataAdapter,
url: fullTextUrl
}
},
fulltextAdapter: {
factory: createElasticAdapter,
url: fullTextUrl
},
storageFactory: () =>
new MinioClient({
...minioConf,
port: 9000,
useSSL: false
}),
workspace
}
return createPipeline(conf, middlewares)
},
(token: Token, pipeline: Pipeline, broadcast: BroadcastCall) => {
if (token.extra?.mode === 'backup') {
return new BackupClientSession(broadcast, token, pipeline)
}
return new ClientSession(broadcast, token, pipeline)
},
port,
host
)
}

View File

@ -29,8 +29,7 @@
},
"dependencies": {
"mongodb": "^4.9.0",
"@hcengineering/platform": "^0.6.7",
"@hcengineering/model-all": "~0.6.0",
"@hcengineering/platform": "^0.6.7",
"minio": "^7.0.26",
"@hcengineering/core": "^0.6.17",
"@hcengineering/contact": "~0.6.8",

View File

@ -14,15 +14,14 @@
//
import contact from '@hcengineering/contact'
import core, { DOMAIN_TX, Tx, Client as CoreClient, Domain, IndexKind, DOMAIN_MODEL } from '@hcengineering/core'
import builder, { migrateOperations } from '@hcengineering/model-all'
import core, { Client as CoreClient, Domain, DOMAIN_MODEL, DOMAIN_TX, IndexKind, Tx } from '@hcengineering/core'
import { MigrateOperation } from '@hcengineering/model'
import { Client } from 'minio'
import { Db, Document, MongoClient } from 'mongodb'
import { connect } from './connect'
import toolPlugin from './plugin'
import { MigrateClientImpl } from './upgrade'
export { version } from '@hcengineering/model-all'
export * from './connect'
export * from './plugin'
export { toolPlugin as default }
@ -30,7 +29,7 @@ export { toolPlugin as default }
/**
* @public
*/
export function prepareTools (): { mongodbUri: string, minio: Client, txes: Tx[] } {
export function prepareTools (rawTxes: Tx[]): { mongodbUri: string, minio: Client, txes: Tx[] } {
let minioEndpoint = process.env.MINIO_ENDPOINT
if (minioEndpoint === undefined) {
console.error('please provide minio endpoint')
@ -70,15 +69,19 @@ export function prepareTools (): { mongodbUri: string, minio: Client, txes: Tx[]
secretKey: minioSecretKey
})
const txes = JSON.parse(JSON.stringify(builder.getTxes())) as Tx[]
return { mongodbUri, minio, txes }
return { mongodbUri, minio, txes: JSON.parse(JSON.stringify(rawTxes)) as Tx[] }
}
/**
* @public
*/
export async function initModel (transactorUrl: string, dbName: string): Promise<void> {
const { mongodbUri, minio, txes } = prepareTools()
export async function initModel (
transactorUrl: string,
dbName: string,
rawTxes: Tx[],
migrateOperations: MigrateOperation[]
): Promise<void> {
const { mongodbUri, minio, txes } = prepareTools(rawTxes)
if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
throw Error('Model txes must target only core.space.Model')
}
@ -123,8 +126,13 @@ export async function initModel (transactorUrl: string, dbName: string): Promise
/**
* @public
*/
export async function upgradeModel (transactorUrl: string, dbName: string): Promise<void> {
const { mongodbUri, txes } = prepareTools()
export async function upgradeModel (
transactorUrl: string,
dbName: string,
rawTxes: Tx[],
migrateOperations: MigrateOperation[]
): Promise<void> {
const { mongodbUri, txes } = prepareTools(rawTxes)
if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
throw Error('Model txes must target only core.space.Model')

7
tools/apm/.eslintrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/config/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

30
tools/apm/apm.js Normal file

File diff suppressed because one or more lines are too long

35
tools/apm/package.json Normal file
View File

@ -0,0 +1,35 @@
{
"name": "@hcengineering/apm",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
"bin": "./lib/apm.js",
"scripts": {
"build": "heft build && esbuild src/index.ts --bundle --minify --platform=node > ./apm.js && echo 'build'",
"build:watch": "tsc",
"lint:fix": "eslint --fix src",
"lint": "eslint src",
"format": "prettier --write src && eslint --fix src"
},
"devDependencies": {
"@hcengineering/platform-rig": "~0.6.0",
"@types/heft-jest": "^1.0.3",
"@typescript-eslint/eslint-plugin": "^5.41.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.26.0",
"@typescript-eslint/parser": "^5.41.0",
"eslint-config-standard-with-typescript": "^23.0.0",
"prettier": "^2.7.1",
"@rushstack/heft": "^0.47.9",
"typescript": "^4.3.5",
"esbuild": "^0.15.13",
"@types/node": "~16.11.12"
},
"dependencies": {
"comment-json": "^4.2.2",
"commander": "^8.1.0"
}
}

24
tools/apm/src/index.ts Normal file
View File

@ -0,0 +1,24 @@
import { program } from 'commander'
import { syncRushFiles } from './sync'
import { createTemplate } from './template'
console.info('Anticrm Platform Manager')
program.version('0.6.0')
program
.command('rush-sync <root>')
.description('Synchronized rush.js files with platform.')
.action(async (root: string, cmd) => {
await syncRushFiles(process.cwd(), root)
})
program
.command('template-apply <root>')
.description('Create necessary startup packages')
.requiredOption('--root <root>', 'user password', 'platform')
.action(async (root: string, cmd) => {
await createTemplate(process.cwd(), root)
})
program.parse(process.argv)

48
tools/apm/src/sync.ts Normal file
View File

@ -0,0 +1,48 @@
import { CommentArray, CommentObject, parse, stringify } from 'comment-json'
import { readFile, writeFile } from 'fs/promises'
import { join } from 'path'
interface RushPackage {
packageName: string
projectFolder: string
shouldPublish: boolean
}
const ignoreProjects = new Set([
'@hcengineering/prod',
'@hcengineering/pod-front',
'@hcengineering/pod-server',
'@hcengineering/pod-account'
])
function platformFilter (root: string): (it: RushPackage) => boolean {
return (it) => {
return !it.projectFolder.startsWith(root)
}
}
export async function syncRushFiles (root: string, platformRoot: string): Promise<void> {
const platformJson: CommentObject = parse(
(await readFile(join(root, platformRoot, 'rush.json'))).toString()
) as CommentObject
const rushjs = join(root, 'rush.json')
const rushjsSource = join(root, 'rush_source.json')
const rushJson: CommentObject = parse((await readFile(rushjsSource)).toString()) as CommentObject
const platformProjecs = (platformJson.projects as unknown as CommentArray<RushPackage>).filter(
(it) => !ignoreProjects.has(it.packageName)
)
const projects = rushJson.projects as unknown as CommentArray<RushPackage>
const newProjects = projects.filter(platformFilter(platformRoot))
newProjects.push(
...platformProjecs.map((it) => ({
...it,
projectFolder: join(platformRoot, it.projectFolder),
shouldPublish: false
}))
)
rushJson.projects = newProjects as unknown as CommentArray<CommentObject>
await writeFile(rushjs, stringify(rushJson, undefined, 2))
}

View File

@ -0,0 +1 @@
export async function createTemplate (root: string, platformRoot: string): Promise<void> {}

10
tools/apm/tsconfig.json Normal file
View File

@ -0,0 +1,10 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"types": ["node"],
"esModuleInterop": true
}
}