mirror of
https://github.com/leon-ai/leon.git
synced 2024-12-24 17:23:23 +03:00
refactor: remove training type
This commit is contained in:
louistiti
parent
a5769dce6e
commit
ddfebb0d74
10
package.json
10
package.json
@ -23,10 +23,10 @@
|
||||
"scripts": {
|
||||
"lint": "babel-node scripts/lint.js",
|
||||
"test": "npm run test:json && npm run test:unit && npm run test:e2e",
|
||||
"test:unit": "npm run train expressions:en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --projects test/unit/unit.jest.json && npm run train expressions",
|
||||
"test:unit": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --projects test/unit/unit.jest.json && npm run train",
|
||||
"test:e2e": "npm run test:e2e:nlp-modules && npm run test:e2e:modules",
|
||||
"test:e2e:modules": "babel-node scripts/run-clean-test-dbs.js && npm run train expressions:en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && babel-node scripts/run-clean-test-dbs.js && npm run train expressions",
|
||||
"test:e2e:nlp-modules": "npm run train expressions:en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train expressions",
|
||||
"test:e2e:modules": "babel-node scripts/run-clean-test-dbs.js && npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && babel-node scripts/run-clean-test-dbs.js && npm run train",
|
||||
"test:e2e:nlp-modules": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train",
|
||||
"test:json": "jest --silent --projects test/json/json.jest.json",
|
||||
"test:module": "babel-node scripts/test-module.js",
|
||||
"setup:offline": "babel-node scripts/setup-offline/setup-offline.js",
|
||||
@ -36,12 +36,12 @@
|
||||
"preinstall": "node scripts/setup/preinstall.js",
|
||||
"postinstall": "babel-node scripts/setup/setup.js",
|
||||
"dev:app": "vite --config app/vite.config.js",
|
||||
"dev:server": "npm run train expressions && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node",
|
||||
"dev:server": "npm run train && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node",
|
||||
"wake": "cross-env LEON_HOST=http://localhost LEON_PORT=1337 node hotword/index.js",
|
||||
"delete-dist:server": "shx rm -rf ./server/dist",
|
||||
"build": "npm run lint && npm run build:app && npm run build:server",
|
||||
"build:app": "cross-env LEON_NODE_ENV=production babel-node scripts/app/run-build-app.js",
|
||||
"build:server": "npm run delete-dist:server && npm run train expressions && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp",
|
||||
"build:server": "npm run delete-dist:server && npm run train && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp",
|
||||
"start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
|
||||
"train": "babel-node scripts/run-train.js",
|
||||
"prepare-release": "babel-node scripts/release/prepare-release.js",
|
||||
|
||||
@ -93,7 +93,7 @@ export default () => new Promise(async (resolve, reject) => {
|
||||
if (!fs.existsSync(nlpModelPath) || !Object.keys(fs.readFileSync(nlpModelPath)).length) {
|
||||
report.can_text.v = false
|
||||
Object.keys(report).forEach((item) => { if (item.indexOf('stt') !== -1 || item.indexOf('tts') !== -1) report[item].v = false })
|
||||
log.error('NLP model not found or broken. Try to generate a new one: "npm run train expressions"\n')
|
||||
log.error('NLP model not found or broken. Try to generate a new one: "npm run train"\n')
|
||||
} else {
|
||||
log.success('Found and valid\n')
|
||||
}
|
||||
|
||||
@ -16,8 +16,8 @@ import loader from '@/helpers/loader'
|
||||
|
||||
try {
|
||||
loader.start()
|
||||
await command('npm run train expressions:en', { shell: true })
|
||||
const cmd = await command(`cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --silent --config=./test/e2e/modules/e2e.modules.jest.json packages/${pkg}/test/${module}.spec.js && npm run train expressions`, { shell: true })
|
||||
await command('npm run train en', { shell: true })
|
||||
const cmd = await command(`cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --silent --config=./test/e2e/modules/e2e.modules.jest.json packages/${pkg}/test/${module}.spec.js && npm run train`, { shell: true })
|
||||
|
||||
log.default(cmd.stdout)
|
||||
log.default(cmd.stderr)
|
||||
|
||||
@ -11,74 +11,63 @@ import { langs } from '../core/langs.json'
|
||||
dotenv.config()
|
||||
|
||||
/**
|
||||
* Training script
|
||||
* Training expressions script
|
||||
*
|
||||
* npm run train expressions
|
||||
* npm run train expressions:en
|
||||
* npm run train [en or fr]
|
||||
*/
|
||||
export default () => new Promise(async (resolve, reject) => {
|
||||
const { argv } = process
|
||||
const packagesDir = 'packages'
|
||||
const modelFileName = 'server/src/data/leon-model.nlp'
|
||||
let type = (argv[2]) ? argv[2].toLowerCase() : 'expressions'
|
||||
let lang = ''
|
||||
|
||||
if (type.indexOf(':') !== -1) {
|
||||
[type, lang] = type.split(':')
|
||||
} else {
|
||||
lang = langs[process.env.LEON_LANG].short.toLowerCase().substr(0, 2)
|
||||
}
|
||||
const lang = argv[2]
|
||||
? argv[2].toLowerCase()
|
||||
: langs[process.env.LEON_LANG].short.toLowerCase().substr(0, 2)
|
||||
|
||||
try {
|
||||
if (type === 'expressions') {
|
||||
const dock = await dockStart({ use: ['Basic'] })
|
||||
const dock = await dockStart({ use: ['Basic'] })
|
||||
|
||||
const nlp = dock.get('nlp')
|
||||
nlp.settings.modelFileName = modelFileName
|
||||
nlp.settings.threshold = 0.8
|
||||
const nlp = dock.get('nlp')
|
||||
nlp.settings.modelFileName = modelFileName
|
||||
nlp.settings.threshold = 0.8
|
||||
|
||||
nlp.addLanguage(lang)
|
||||
nlp.addLanguage(lang)
|
||||
|
||||
const packages = fs.readdirSync(packagesDir)
|
||||
.filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory())
|
||||
let expressionsObj = { }
|
||||
const packages = fs.readdirSync(packagesDir)
|
||||
.filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory())
|
||||
let expressionsObj = { }
|
||||
|
||||
for (let i = 0; i < packages.length; i += 1) {
|
||||
log.info(`Training "${string.ucfirst(packages[i])}" package modules expressions...`)
|
||||
for (let i = 0; i < packages.length; i += 1) {
|
||||
log.info(`Training "${string.ucfirst(packages[i])}" package modules expressions...`)
|
||||
|
||||
expressionsObj = JSON.parse(fs.readFileSync(`${packagesDir}/${packages[i]}/data/expressions/${lang}.json`, 'utf8'))
|
||||
expressionsObj = JSON.parse(fs.readFileSync(`${packagesDir}/${packages[i]}/data/expressions/${lang}.json`, 'utf8'))
|
||||
|
||||
const modules = Object.keys(expressionsObj)
|
||||
for (let j = 0; j < modules.length; j += 1) {
|
||||
const module = modules[j]
|
||||
const actions = Object.keys(expressionsObj[module])
|
||||
const modules = Object.keys(expressionsObj)
|
||||
for (let j = 0; j < modules.length; j += 1) {
|
||||
const module = modules[j]
|
||||
const actions = Object.keys(expressionsObj[module])
|
||||
|
||||
for (let k = 0; k < actions.length; k += 1) {
|
||||
const action = actions[k]
|
||||
const exprs = expressionsObj[module][action].expressions
|
||||
for (let k = 0; k < actions.length; k += 1) {
|
||||
const action = actions[k]
|
||||
const exprs = expressionsObj[module][action].expressions
|
||||
|
||||
nlp.assignDomain(lang, `${module}.${action}`, packages[i])
|
||||
nlp.assignDomain(lang, `${module}.${action}`, packages[i])
|
||||
|
||||
for (let l = 0; l < exprs.length; l += 1) {
|
||||
nlp.addDocument(lang, exprs[l], `${module}.${action}`)
|
||||
}
|
||||
for (let l = 0; l < exprs.length; l += 1) {
|
||||
nlp.addDocument(lang, exprs[l], `${module}.${action}`)
|
||||
}
|
||||
|
||||
log.success(`"${string.ucfirst(module)}" module expressions trained`)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await nlp.train()
|
||||
|
||||
log.success(`NLP model saved in ${modelFileName}`)
|
||||
resolve()
|
||||
} catch (e) {
|
||||
log.error(`Failed to save NLP model: ${e}`)
|
||||
reject()
|
||||
log.success(`"${string.ucfirst(module)}" module expressions trained`)
|
||||
}
|
||||
} else {
|
||||
log.error(`"${type}" training type is unknown. Try "npm run train expressions"`)
|
||||
}
|
||||
|
||||
try {
|
||||
await nlp.train()
|
||||
|
||||
log.success(`NLP model saved in ${modelFileName}`)
|
||||
resolve()
|
||||
} catch (e) {
|
||||
log.error(`Failed to save NLP model: ${e}`)
|
||||
reject()
|
||||
}
|
||||
} catch (e) {
|
||||
|
||||
@ -28,7 +28,7 @@ class Nlu {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
if (!fs.existsSync(nlpModel)) {
|
||||
log.title('NLU')
|
||||
reject({ type: 'warning', obj: new Error('The NLP model does not exist, please run: npm run train expressions') })
|
||||
reject({ type: 'warning', obj: new Error('The NLP model does not exist, please run: npm run train') })
|
||||
} else {
|
||||
log.title('NLU')
|
||||
|
||||
@ -65,7 +65,7 @@ class Nlu {
|
||||
this.brain.talk(`${this.brain.wernicke('random_errors')}!`)
|
||||
this.brain.socket.emit('is-typing', false)
|
||||
|
||||
log.error('The NLP model is missing, please rebuild the project or if you are in dev run: npm run train expressions')
|
||||
log.error('The NLP model is missing, please rebuild the project or if you are in dev run: npm run train')
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@ -36,7 +36,7 @@ describe('NLU modules', () => {
|
||||
process.env.LEON_LANG = langKeys[i]
|
||||
|
||||
// Generate new NLP model for the tested language
|
||||
await command(`npm run train expressions:${lang.short}`, { shell: true })
|
||||
await command(`npm run train ${lang.short}`, { shell: true })
|
||||
// Load the new NLP model
|
||||
await nlu.loadModel(global.paths.nlp_model)
|
||||
})
|
||||
|
||||
@ -93,7 +93,7 @@ describe('server', () => {
|
||||
}, 50)
|
||||
|
||||
setTimeout(() => {
|
||||
expect(console.log.mock.calls[22][1]).toBe('Query found')
|
||||
expect(console.log.mock.calls[26][1]).toBe('Query found')
|
||||
console.log = jest.fn()
|
||||
}, 100)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user