1
1
mirror of https://github.com/leon-ai/leon.git synced 2024-12-21 15:51:49 +03:00
leon/scripts/check.js

604 lines
18 KiB
JavaScript
Raw Normal View History

import fs from 'node:fs'
import path from 'node:path'
import os from 'node:os'
import { spawn } from 'node:child_process'
2019-02-10 15:26:50 +03:00
import dotenv from 'dotenv'
import { command } from 'execa'
2019-02-10 15:26:50 +03:00
import semver from 'semver'
import kill from 'tree-kill'
2022-10-23 05:55:03 +03:00
import axios from 'axios'
import osName from 'os-name'
import getos from 'getos'
2019-02-10 15:26:50 +03:00
import { LogHelper } from '@/helpers/log-helper'
import { SystemHelper } from '@/helpers/system-helper'
import {
MINIMUM_REQUIRED_RAM,
LEON_VERSION,
NODEJS_BRIDGE_BIN_PATH,
PYTHON_BRIDGE_BIN_PATH,
TCP_SERVER_BIN_PATH,
TCP_SERVER_VERSION,
NODEJS_BRIDGE_VERSION,
2023-04-30 05:45:51 +03:00
PYTHON_BRIDGE_VERSION,
INSTANCE_ID
} from '@/constants'
2019-02-10 15:26:50 +03:00
dotenv.config()
/**
* Checking script
* Help to figure out the setup state
2019-02-10 15:26:50 +03:00
*/
;(async () => {
try {
const nodeMinRequiredVersion = '16'
2023-04-02 20:18:15 +03:00
const npmMinRequiredVersion = '8'
const flitePath = 'bin/flite/flite'
const coquiLanguageModelPath = 'bin/coqui/huge-vocabulary.scorer'
const amazonPath = 'core/config/voice/amazon.json'
const googleCloudPath = 'core/config/voice/google-cloud.json'
const watsonSttPath = 'core/config/voice/watson-stt.json'
const watsonTtsPath = 'core/config/voice/watson-tts.json'
const globalResolversNlpModelPath =
'core/data/models/leon-global-resolvers-model.nlp'
const skillsResolversNlpModelPath =
'core/data/models/leon-skills-resolvers-model.nlp'
const mainNlpModelPath = 'core/data/models/leon-main-model.nlp'
const report = {
can_run: { title: 'Run', type: 'error', v: true },
can_run_skill: { title: 'Run skills', type: 'error', v: true },
can_text: { title: 'Reply you by texting', type: 'error', v: true },
can_start_tcp_server: {
title: 'Start the TCP server',
type: 'error',
v: true
},
can_amazon_polly_tts: {
title: 'Amazon Polly text-to-speech',
type: 'warning',
v: true
},
can_google_cloud_tts: {
title: 'Google Cloud text-to-speech',
type: 'warning',
v: true
},
can_watson_tts: {
title: 'Watson text-to-speech',
type: 'warning',
v: true
},
can_offline_tts: {
title: 'Offline text-to-speech',
type: 'warning',
v: true
},
can_google_cloud_stt: {
title: 'Google Cloud speech-to-text',
type: 'warning',
v: true
},
can_watson_stt: {
title: 'Watson speech-to-text',
type: 'warning',
v: true
},
can_offline_stt: {
title: 'Offline speech-to-text',
type: 'warning',
v: true
2022-09-03 14:12:41 +03:00
}
}
2023-04-30 05:45:51 +03:00
let reportDataInput = {
leonVersion: null,
2023-04-30 05:45:51 +03:00
instanceID: INSTANCE_ID || null,
environment: {
osDetails: null,
nodeVersion: null,
npmVersion: null
},
nlpModels: {
globalResolversModelState: null,
skillsResolversModelState: null,
mainModelState: null
},
nodeJSBridge: {
version: null,
executionTime: null,
command: null,
output: null,
error: null
},
pythonBridge: {
version: null,
executionTime: null,
command: null,
output: null,
error: null
},
tcpServer: {
version: null,
startTime: null,
command: null,
output: null,
error: null
},
report: null
}
2019-02-10 15:26:50 +03:00
LogHelper.title('Checking')
/**
* Leon version checking
*/
LogHelper.info('Leon version')
LogHelper.success(`${LEON_VERSION}\n`)
2023-04-30 05:45:51 +03:00
reportDataInput.leonVersion = LEON_VERSION
2019-02-10 15:26:50 +03:00
/**
* Environment checking
*/
2022-10-16 18:25:58 +03:00
LogHelper.info('Environment')
const osInfo = {
type: os.type(),
platform: os.platform(),
arch: os.arch(),
cpus: os.cpus().length,
release: os.release(),
osName: osName(),
distro: null
}
const totalRAMInGB = SystemHelper.getTotalRAM()
2023-05-03 09:54:23 +03:00
const freeRAMInGB = SystemHelper.getFreeRAM()
2023-05-03 09:54:23 +03:00
if (Math.round(freeRAMInGB) < MINIMUM_REQUIRED_RAM) {
report.can_run.v = false
LogHelper.error(
2023-05-16 00:42:27 +03:00
`Free RAM: ${freeRAMInGB} GB | Total RAM: ${totalRAMInGB} GB. Leon needs at least ${MINIMUM_REQUIRED_RAM} GB of RAM`
)
} else {
2023-05-03 09:54:23 +03:00
LogHelper.success(
2023-05-16 00:42:27 +03:00
`Free RAM: ${freeRAMInGB} GB | Total RAM: ${totalRAMInGB} GB`
2023-05-03 09:54:23 +03:00
)
}
if (osInfo.platform === 'linux') {
getos((e, os) => {
osInfo.distro = os
LogHelper.success(`${JSON.stringify(osInfo)}\n`)
})
} else {
LogHelper.success(`${JSON.stringify(osInfo)}\n`)
}
2023-04-30 05:45:51 +03:00
reportDataInput.environment.osDetails = osInfo
reportDataInput.environment.totalRAMInGB = totalRAMInGB
2023-05-03 09:54:23 +03:00
reportDataInput.environment.freeRAMInGB = freeRAMInGB
;(
await Promise.all([
command('node --version', { shell: true }),
command('npm --version', { shell: true })
])
).forEach((p) => {
LogHelper.info(p.command)
2019-02-10 15:26:50 +03:00
if (
p.command.indexOf('node --version') !== -1 &&
!semver.satisfies(semver.clean(p.stdout), `>=${nodeMinRequiredVersion}`)
) {
Object.keys(report).forEach((item) => {
if (report[item].type === 'error') report[item].v = false
})
LogHelper.error(
`${p.stdout}\nThe Node.js version must be >=${nodeMinRequiredVersion}. Please install it: https://nodejs.org (or use nvm)\n`
)
} else if (
p.command.indexOf('npm --version') !== -1 &&
!semver.satisfies(semver.clean(p.stdout), `>=${npmMinRequiredVersion}`)
) {
Object.keys(report).forEach((item) => {
if (report[item].type === 'error') report[item].v = false
})
LogHelper.error(
`${p.stdout}\nThe npm version must be >=${npmMinRequiredVersion}. Please install it: https://www.npmjs.com/get-npm (or use nvm)\n`
)
} else {
LogHelper.success(`${p.stdout}\n`)
if (p.command.includes('node --version')) {
2023-04-30 05:45:51 +03:00
reportDataInput.environment.nodeVersion = p.stdout
} else if (p.command.includes('npm --version')) {
2023-04-30 05:45:51 +03:00
reportDataInput.environment.npmVersion = p.stdout
2022-09-03 14:12:41 +03:00
}
}
})
2019-02-10 15:26:50 +03:00
/**
* Skill execution checking with Node.js bridge
*/
LogHelper.success(`Node.js bridge version: ${NODEJS_BRIDGE_VERSION}`)
reportDataInput.nodeJSBridge.version = NODEJS_BRIDGE_VERSION
LogHelper.info('Executing a skill...')
try {
const executionStart = Date.now()
const p = await command(
`${NODEJS_BRIDGE_BIN_PATH} "${path.join(
process.cwd(),
'scripts',
'assets',
'nodejs-bridge-intent-object.json'
)}"`,
{ shell: true }
)
const executionEnd = Date.now()
const executionTime = executionEnd - executionStart
LogHelper.info(p.command)
reportDataInput.nodeJSBridge.command = p.command
LogHelper.success(p.stdout)
reportDataInput.nodeJSBridge.output = p.stdout
LogHelper.info(`Skill execution time: ${executionTime}ms\n`)
reportDataInput.nodeJSBridge.executionTime = `${executionTime}ms`
} catch (e) {
LogHelper.info(e.command)
report.can_run_skill.v = false
LogHelper.error(`${e}\n`)
reportDataInput.nodeJSBridge.error = JSON.stringify(e)
}
/**
* Skill execution checking with Python bridge
*/
2019-02-10 15:26:50 +03:00
LogHelper.success(`Python bridge version: ${PYTHON_BRIDGE_VERSION}`)
2023-04-30 05:45:51 +03:00
reportDataInput.pythonBridge.version = PYTHON_BRIDGE_VERSION
LogHelper.info('Executing a skill...')
try {
const executionStart = Date.now()
const p = await command(
`${PYTHON_BRIDGE_BIN_PATH} "${path.join(
process.cwd(),
2023-04-02 20:18:15 +03:00
'scripts',
'assets',
'python-bridge-intent-object.json'
)}"`,
{ shell: true }
)
const executionEnd = Date.now()
const executionTime = executionEnd - executionStart
LogHelper.info(p.command)
2023-04-30 05:45:51 +03:00
reportDataInput.pythonBridge.command = p.command
LogHelper.success(p.stdout)
2023-04-30 05:45:51 +03:00
reportDataInput.pythonBridge.output = p.stdout
LogHelper.info(`Skill execution time: ${executionTime}ms\n`)
2023-04-30 05:45:51 +03:00
reportDataInput.pythonBridge.executionTime = `${executionTime}ms`
} catch (e) {
LogHelper.info(e.command)
report.can_run_skill.v = false
LogHelper.error(`${e}\n`)
2023-04-30 05:45:51 +03:00
reportDataInput.pythonBridge.error = JSON.stringify(e)
}
/**
* TCP server startup checking
*/
LogHelper.success(`TCP server version: ${TCP_SERVER_VERSION}`)
2023-04-30 05:45:51 +03:00
reportDataInput.tcpServer.version = TCP_SERVER_VERSION
LogHelper.info('Starting the TCP server...')
const tcpServerCommand = `${TCP_SERVER_BIN_PATH} en`
const tcpServerStart = Date.now()
const p = spawn(tcpServerCommand, { shell: true })
const ignoredWarnings = [
'UserWarning: Unable to retrieve source for @torch.jit._overload function'
]
LogHelper.info(tcpServerCommand)
2023-04-30 05:45:51 +03:00
reportDataInput.tcpServer.command = tcpServerCommand
if (osInfo.platform === 'darwin') {
LogHelper.info(
'For the first start, it may take a few minutes to cold start the TCP server on macOS. No worries it is a one-time thing'
)
}
let tcpServerOutput = ''
p.stdout.on('data', (data) => {
const newData = data.toString()
tcpServerOutput += newData
if (newData?.toLowerCase().includes('waiting for')) {
kill(p.pid)
LogHelper.success('The TCP server can successfully start')
2022-09-03 14:12:41 +03:00
}
})
p.stderr.on('data', (data) => {
const newData = data.toString()
// Ignore given warnings on stderr output
if (!ignoredWarnings.some((w) => newData.includes(w))) {
tcpServerOutput += newData
report.can_start_tcp_server.v = false
2023-04-30 05:45:51 +03:00
reportDataInput.tcpServer.error = newData
LogHelper.error(`Cannot start the TCP server: ${newData}`)
}
})
const timeout = 3 * 60_000
// In case it takes too long, force kill
setTimeout(() => {
kill(p.pid)
const error = `The TCP server timed out after ${timeout}ms`
LogHelper.error(error)
2023-04-30 05:45:51 +03:00
reportDataInput.tcpServer.error = error
report.can_start_tcp_server.v = false
}, timeout)
p.stdout.on('end', async () => {
const tcpServerEnd = Date.now()
2023-04-30 05:45:51 +03:00
reportDataInput.tcpServer.output = tcpServerOutput
reportDataInput.tcpServer.startTime = `${tcpServerEnd - tcpServerStart}ms`
LogHelper.info(
2023-04-30 05:45:51 +03:00
`TCP server startup time: ${reportDataInput.tcpServer.startTime}\n`
)
2019-02-10 15:26:50 +03:00
2022-09-12 19:09:37 +03:00
/**
* Global resolvers NLP model checking
*/
2019-02-10 15:26:50 +03:00
LogHelper.info('Global resolvers NLP model state')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
if (
!fs.existsSync(globalResolversNlpModelPath) ||
2023-04-02 21:34:50 +03:00
!Object.keys(await fs.promises.readFile(globalResolversNlpModelPath))
.length
2022-09-03 14:12:41 +03:00
) {
const state = 'Global resolvers NLP model not found or broken'
2022-09-03 14:12:41 +03:00
report.can_text.v = false
Object.keys(report).forEach((item) => {
if (item.indexOf('stt') !== -1 || item.indexOf('tts') !== -1)
report[item].v = false
})
LogHelper.error(
`${state}. Try to generate a new one: "npm run train"\n`
2022-09-03 14:12:41 +03:00
)
2023-04-30 05:45:51 +03:00
reportDataInput.nlpModels.globalResolversModelState = state
2022-09-03 14:12:41 +03:00
} else {
const state = 'Found and valid'
LogHelper.success(`${state}\n`)
2023-04-30 05:45:51 +03:00
reportDataInput.nlpModels.globalResolversModelState = state
2022-09-03 14:12:41 +03:00
}
2022-07-19 16:57:00 +03:00
2022-09-12 19:09:37 +03:00
/**
* Skills resolvers NLP model checking
*/
2022-07-19 16:57:00 +03:00
LogHelper.info('Skills resolvers NLP model state')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
if (
!fs.existsSync(skillsResolversNlpModelPath) ||
2023-04-02 21:34:50 +03:00
!Object.keys(await fs.promises.readFile(skillsResolversNlpModelPath))
.length
2022-09-03 14:12:41 +03:00
) {
const state = 'Skills resolvers NLP model not found or broken'
2022-09-03 14:12:41 +03:00
report.can_text.v = false
Object.keys(report).forEach((item) => {
if (item.indexOf('stt') !== -1 || item.indexOf('tts') !== -1)
report[item].v = false
})
LogHelper.error(
`${state}. Try to generate a new one: "npm run train"\n`
2022-09-03 14:12:41 +03:00
)
2023-04-30 05:45:51 +03:00
reportDataInput.nlpModels.skillsResolversModelState = state
2022-09-03 14:12:41 +03:00
} else {
const state = 'Found and valid'
LogHelper.success(`${state}\n`)
2023-04-30 05:45:51 +03:00
reportDataInput.nlpModels.skillsResolversModelState = state
2022-09-03 14:12:41 +03:00
}
2022-09-12 19:09:37 +03:00
/**
* Main NLP model checking
*/
LogHelper.info('Main NLP model state')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
if (
!fs.existsSync(mainNlpModelPath) ||
2023-04-02 21:34:50 +03:00
!Object.keys(await fs.promises.readFile(mainNlpModelPath)).length
2022-09-03 14:12:41 +03:00
) {
const state = 'Main NLP model not found or broken'
2022-09-03 14:12:41 +03:00
report.can_text.v = false
Object.keys(report).forEach((item) => {
if (item.indexOf('stt') !== -1 || item.indexOf('tts') !== -1)
report[item].v = false
})
LogHelper.error(
`${state}. Try to generate a new one: "npm run train"\n`
2022-09-03 14:12:41 +03:00
)
2023-04-30 05:45:51 +03:00
reportDataInput.nlpModels.mainModelState = state
2022-09-03 14:12:41 +03:00
} else {
const state = 'Found and valid'
LogHelper.success(`${state}\n`)
2023-04-30 05:45:51 +03:00
reportDataInput.nlpModels.mainModelState = state
2022-09-03 14:12:41 +03:00
}
2019-02-10 15:26:50 +03:00
2022-09-12 19:09:37 +03:00
/**
* TTS/STT checking
*/
2019-02-10 15:26:50 +03:00
LogHelper.info('Amazon Polly TTS')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
try {
2023-04-02 21:34:50 +03:00
const json = JSON.parse(await fs.promises.readFile(amazonPath))
2022-09-03 14:12:41 +03:00
if (
json.credentials.accessKeyId === '' ||
json.credentials.secretAccessKey === ''
) {
report.can_amazon_polly_tts.v = false
LogHelper.warning('Amazon Polly TTS is not yet configured\n')
2022-09-03 14:12:41 +03:00
} else {
LogHelper.success('Configured\n')
2022-09-03 14:12:41 +03:00
}
} catch (e) {
2019-02-10 15:26:50 +03:00
report.can_amazon_polly_tts.v = false
LogHelper.warning(`Amazon Polly TTS is not yet configured: ${e}\n`)
2019-02-10 15:26:50 +03:00
}
LogHelper.info('Google Cloud TTS/STT')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
try {
2023-04-02 21:34:50 +03:00
const json = JSON.parse(await fs.promises.readFile(googleCloudPath))
2022-09-03 14:12:41 +03:00
const results = []
Object.keys(json).forEach((item) => {
if (json[item] === '') results.push(false)
})
if (results.includes(false)) {
report.can_google_cloud_tts.v = false
report.can_google_cloud_stt.v = false
LogHelper.warning('Google Cloud TTS/STT is not yet configured\n')
2022-09-03 14:12:41 +03:00
} else {
LogHelper.success('Configured\n')
2022-09-03 14:12:41 +03:00
}
} catch (e) {
2019-02-10 15:26:50 +03:00
report.can_google_cloud_tts.v = false
report.can_google_cloud_stt.v = false
LogHelper.warning(`Google Cloud TTS/STT is not yet configured: ${e}\n`)
2019-02-10 15:26:50 +03:00
}
LogHelper.info('Watson TTS')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
try {
2023-04-02 21:34:50 +03:00
const json = JSON.parse(await fs.promises.readFile(watsonTtsPath))
2022-09-03 14:12:41 +03:00
const results = []
Object.keys(json).forEach((item) => {
if (json[item] === '') results.push(false)
})
if (results.includes(false)) {
report.can_watson_tts.v = false
LogHelper.warning('Watson TTS is not yet configured\n')
2022-09-03 14:12:41 +03:00
} else {
LogHelper.success('Configured\n')
2022-09-03 14:12:41 +03:00
}
} catch (e) {
2019-02-10 15:26:50 +03:00
report.can_watson_tts.v = false
LogHelper.warning(`Watson TTS is not yet configured: ${e}\n`)
2019-02-10 15:26:50 +03:00
}
LogHelper.info('Offline TTS')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
if (!fs.existsSync(flitePath)) {
report.can_offline_tts.v = false
LogHelper.warning(
`Cannot find ${flitePath}. You can set up the offline TTS by running: "npm run setup:offline-tts"\n`
2022-09-03 14:12:41 +03:00
)
} else {
LogHelper.success(`Found Flite at ${flitePath}\n`)
2022-09-03 14:12:41 +03:00
}
2019-02-10 15:26:50 +03:00
LogHelper.info('Watson STT')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
try {
2023-04-02 21:34:50 +03:00
const json = JSON.parse(await fs.promises.readFile(watsonSttPath))
2022-09-03 14:12:41 +03:00
const results = []
Object.keys(json).forEach((item) => {
if (json[item] === '') results.push(false)
})
if (results.includes(false)) {
report.can_watson_stt.v = false
LogHelper.warning('Watson STT is not yet configured\n')
2022-09-03 14:12:41 +03:00
} else {
LogHelper.success('Configured\n')
2022-09-03 14:12:41 +03:00
}
} catch (e) {
2019-02-10 15:26:50 +03:00
report.can_watson_stt.v = false
LogHelper.warning(`Watson STT is not yet configured: ${e}`)
2022-09-03 14:12:41 +03:00
}
LogHelper.info('Offline STT')
2022-09-12 19:09:37 +03:00
2022-09-03 14:12:41 +03:00
if (!fs.existsSync(coquiLanguageModelPath)) {
report.can_offline_stt.v = false
LogHelper.warning(
2022-09-03 14:12:41 +03:00
`Cannot find ${coquiLanguageModelPath}. You can setup the offline STT by running: "npm run setup:offline-stt"`
)
2019-02-10 15:26:50 +03:00
} else {
LogHelper.success(
`Found Coqui language model at ${coquiLanguageModelPath}`
)
2019-02-10 15:26:50 +03:00
}
2022-09-12 19:09:37 +03:00
/**
* Report
*/
LogHelper.title('Report')
2019-02-10 15:26:50 +03:00
LogHelper.info('Here is the diagnosis about your current setup')
2022-09-03 14:12:41 +03:00
Object.keys(report).forEach((item) => {
if (report[item].v === true) {
LogHelper.success(report[item].title)
2022-09-03 14:12:41 +03:00
} else {
LogHelper[report[item].type](report[item].title)
2022-09-03 14:12:41 +03:00
}
})
2019-02-10 15:26:50 +03:00
LogHelper.default('')
if (
report.can_run.v &&
report.can_run_skill.v &&
report.can_text.v &&
report.can_start_tcp_server.v
) {
LogHelper.success('Hooray! Leon can run correctly')
LogHelper.info(
2022-09-03 14:12:41 +03:00
'If you have some yellow warnings, it is all good. It means some entities are not yet configured'
)
2019-02-10 15:26:50 +03:00
} else {
LogHelper.error('Please fix the errors above')
2019-02-10 15:26:50 +03:00
}
2023-04-30 05:45:51 +03:00
reportDataInput.report = report
2023-04-30 05:45:51 +03:00
reportDataInput = JSON.parse(
SystemHelper.sanitizeUsername(JSON.stringify(reportDataInput))
)
LogHelper.title('REPORT URL')
LogHelper.info('Sending report...')
try {
2022-10-23 05:55:03 +03:00
const { data } = await axios.post('https://getleon.ai/api/report', {
2023-04-30 05:45:51 +03:00
report: reportDataInput
2022-10-23 05:55:03 +03:00
})
2023-04-30 05:45:51 +03:00
const { data: responseReportData } = data
2023-04-30 05:45:51 +03:00
LogHelper.success(`Report URL: ${responseReportData.reportUrl}`)
} catch (e) {
LogHelper.error(`Failed to send report: ${e}`)
}
process.exit(0)
})
} catch (e) {
LogHelper.error(e)
}
})()