refactor: remove training type

2024-09-11 10:25:40 +03:00 · 2021-12-27 17:09:12 +08:00 · 2021-12-27 17:09:12 +08:00 · ddfebb0d74
commit ddfebb0d74
parent a5769dce6e
7 changed files with 48 additions and 59 deletions
--- a/package.json
+++ b/package.json
@ -23,10 +23,10 @@
  "scripts": {
    "lint": "babel-node scripts/lint.js",
    "test": "npm run test:json && npm run test:unit && npm run test:e2e",
-    "test:unit": "npm run train expressions:en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --projects test/unit/unit.jest.json && npm run train expressions",
+    "test:unit": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --projects test/unit/unit.jest.json && npm run train",
    "test:e2e": "npm run test:e2e:nlp-modules && npm run test:e2e:modules",
-    "test:e2e:modules": "babel-node scripts/run-clean-test-dbs.js && npm run train expressions:en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && babel-node scripts/run-clean-test-dbs.js && npm run train expressions",
-    "test:e2e:nlp-modules": "npm run train expressions:en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train expressions",
+    "test:e2e:modules": "babel-node scripts/run-clean-test-dbs.js && npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && babel-node scripts/run-clean-test-dbs.js && npm run train",
+    "test:e2e:nlp-modules": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train",
    "test:json": "jest --silent --projects test/json/json.jest.json",
    "test:module": "babel-node scripts/test-module.js",
    "setup:offline": "babel-node scripts/setup-offline/setup-offline.js",
@ -36,12 +36,12 @@
    "preinstall": "node scripts/setup/preinstall.js",
    "postinstall": "babel-node scripts/setup/setup.js",
    "dev:app": "vite --config app/vite.config.js",
-    "dev:server": "npm run train expressions && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node",
+    "dev:server": "npm run train && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node",
    "wake": "cross-env LEON_HOST=http://localhost LEON_PORT=1337 node hotword/index.js",
    "delete-dist:server": "shx rm -rf ./server/dist",
    "build": "npm run lint && npm run build:app && npm run build:server",
    "build:app": "cross-env LEON_NODE_ENV=production babel-node scripts/app/run-build-app.js",
-    "build:server": "npm run delete-dist:server && npm run train expressions && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp",
+    "build:server": "npm run delete-dist:server && npm run train && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp",
    "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
    "train": "babel-node scripts/run-train.js",
    "prepare-release": "babel-node scripts/release/prepare-release.js",
--- a/scripts/check.js
+++ b/scripts/check.js
@ -93,7 +93,7 @@ export default () => new Promise(async (resolve, reject) => {
    if (!fs.existsSync(nlpModelPath) || !Object.keys(fs.readFileSync(nlpModelPath)).length) {
      report.can_text.v = false
      Object.keys(report).forEach((item) => { if (item.indexOf('stt') !== -1 || item.indexOf('tts') !== -1) report[item].v = false })
-      log.error('NLP model not found or broken. Try to generate a new one: "npm run train expressions"\n')
+      log.error('NLP model not found or broken. Try to generate a new one: "npm run train"\n')
    } else {
      log.success('Found and valid\n')
    }
--- a/scripts/test-module.js
+++ b/scripts/test-module.js
@ -16,8 +16,8 @@ import loader from '@/helpers/loader'

  try {
    loader.start()
-    await command('npm run train expressions:en', { shell: true })
-    const cmd = await command(`cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --silent --config=./test/e2e/modules/e2e.modules.jest.json packages/${pkg}/test/${module}.spec.js && npm run train expressions`, { shell: true })
+    await command('npm run train en', { shell: true })
+    const cmd = await command(`cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --silent --config=./test/e2e/modules/e2e.modules.jest.json packages/${pkg}/test/${module}.spec.js && npm run train`, { shell: true })

    log.default(cmd.stdout)
    log.default(cmd.stderr)
--- a/scripts/train.js
+++ b/scripts/train.js
@ -11,74 +11,63 @@ import { langs } from '../core/langs.json'
 dotenv.config()

 /**
- * Training script
+ * Training expressions script
 *
- * npm run train expressions
- * npm run train expressions:en
+ * npm run train [en or fr]
 */
 export default () => new Promise(async (resolve, reject) => {
  const { argv } = process
  const packagesDir = 'packages'
  const modelFileName = 'server/src/data/leon-model.nlp'
-  let type = (argv[2]) ? argv[2].toLowerCase() : 'expressions'
-  let lang = ''
-
-  if (type.indexOf(':') !== -1) {
-    [type, lang] = type.split(':')
-  } else {
-    lang = langs[process.env.LEON_LANG].short.toLowerCase().substr(0, 2)
-  }
+  const lang = argv[2]
+    ? argv[2].toLowerCase()
+    : langs[process.env.LEON_LANG].short.toLowerCase().substr(0, 2)

  try {
-    if (type === 'expressions') {
-      const dock = await dockStart({ use: ['Basic'] })
+    const dock = await dockStart({ use: ['Basic'] })

-      const nlp = dock.get('nlp')
-      nlp.settings.modelFileName = modelFileName
-      nlp.settings.threshold = 0.8
+    const nlp = dock.get('nlp')
+    nlp.settings.modelFileName = modelFileName
+    nlp.settings.threshold = 0.8

-      nlp.addLanguage(lang)
+    nlp.addLanguage(lang)

-      const packages = fs.readdirSync(packagesDir)
-        .filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory())
-      let expressionsObj = { }
+    const packages = fs.readdirSync(packagesDir)
+      .filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory())
+    let expressionsObj = { }

-      for (let i = 0; i < packages.length; i += 1) {
-        log.info(`Training "${string.ucfirst(packages[i])}" package modules expressions...`)
+    for (let i = 0; i < packages.length; i += 1) {
+      log.info(`Training "${string.ucfirst(packages[i])}" package modules expressions...`)

-        expressionsObj = JSON.parse(fs.readFileSync(`${packagesDir}/${packages[i]}/data/expressions/${lang}.json`, 'utf8'))
+      expressionsObj = JSON.parse(fs.readFileSync(`${packagesDir}/${packages[i]}/data/expressions/${lang}.json`, 'utf8'))

-        const modules = Object.keys(expressionsObj)
-        for (let j = 0; j < modules.length; j += 1) {
-          const module = modules[j]
-          const actions = Object.keys(expressionsObj[module])
+      const modules = Object.keys(expressionsObj)
+      for (let j = 0; j < modules.length; j += 1) {
+        const module = modules[j]
+        const actions = Object.keys(expressionsObj[module])

-          for (let k = 0; k < actions.length; k += 1) {
-            const action = actions[k]
-            const exprs = expressionsObj[module][action].expressions
+        for (let k = 0; k < actions.length; k += 1) {
+          const action = actions[k]
+          const exprs = expressionsObj[module][action].expressions

-            nlp.assignDomain(lang, `${module}.${action}`, packages[i])
+          nlp.assignDomain(lang, `${module}.${action}`, packages[i])

-            for (let l = 0; l < exprs.length; l += 1) {
-              nlp.addDocument(lang, exprs[l], `${module}.${action}`)
-            }
+          for (let l = 0; l < exprs.length; l += 1) {
+            nlp.addDocument(lang, exprs[l], `${module}.${action}`)
          }
-
-          log.success(`"${string.ucfirst(module)}" module expressions trained`)
        }
-      }

-      try {
-        await nlp.train()
-
-        log.success(`NLP model saved in ${modelFileName}`)
-        resolve()
-      } catch (e) {
-        log.error(`Failed to save NLP model: ${e}`)
-        reject()
+        log.success(`"${string.ucfirst(module)}" module expressions trained`)
      }
-    } else {
-      log.error(`"${type}" training type is unknown. Try "npm run train expressions"`)
+    }
+
+    try {
+      await nlp.train()
+
+      log.success(`NLP model saved in ${modelFileName}`)
+      resolve()
+    } catch (e) {
+      log.error(`Failed to save NLP model: ${e}`)
      reject()
    }
  } catch (e) {
--- a/server/src/core/nlu.js
+++ b/server/src/core/nlu.js
@ -28,7 +28,7 @@ class Nlu {
    return new Promise(async (resolve, reject) => {
      if (!fs.existsSync(nlpModel)) {
        log.title('NLU')
-        reject({ type: 'warning', obj: new Error('The NLP model does not exist, please run: npm run train expressions') })
+        reject({ type: 'warning', obj: new Error('The NLP model does not exist, please run: npm run train') })
      } else {
        log.title('NLU')

@ -65,7 +65,7 @@ class Nlu {
      this.brain.talk(`${this.brain.wernicke('random_errors')}!`)
      this.brain.socket.emit('is-typing', false)

-      log.error('The NLP model is missing, please rebuild the project or if you are in dev run: npm run train expressions')
+      log.error('The NLP model is missing, please rebuild the project or if you are in dev run: npm run train')

      return false
    }
--- a/test/e2e/nlp-modules.spec.js
+++ b/test/e2e/nlp-modules.spec.js
@ -36,7 +36,7 @@ describe('NLU modules', () => {
        process.env.LEON_LANG = langKeys[i]

        // Generate new NLP model for the tested language
-        await command(`npm run train expressions:${lang.short}`, { shell: true })
+        await command(`npm run train ${lang.short}`, { shell: true })
        // Load the new NLP model
        await nlu.loadModel(global.paths.nlp_model)
      })
--- a/test/unit/server/core/server.spec.js
+++ b/test/unit/server/core/server.spec.js
@ -93,7 +93,7 @@ describe('server', () => {
      }, 50)

      setTimeout(() => {
-        expect(console.log.mock.calls[22][1]).toBe('Query found')
+        expect(console.log.mock.calls[26][1]).toBe('Query found')
        console.log = jest.fn()
      }, 100)