From fb5c258cf6f81cc44224571a9d743468169a147d Mon Sep 17 00:00:00 2001 From: louistiti Date: Sun, 26 May 2024 09:47:13 +0800 Subject: [PATCH] refactor: unify NLP models paths --- scripts/check.js | 28 ++++++++++++------------- scripts/train/train.js | 25 +++++++++++----------- server/src/constants.ts | 13 ++++++++++++ server/src/core/nlp/nlu/model-loader.ts | 17 +++++++-------- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/scripts/check.js b/scripts/check.js index 014c5c0d..857f5763 100644 --- a/scripts/check.js +++ b/scripts/check.js @@ -22,7 +22,10 @@ import { PYTHON_TCP_SERVER_VERSION, NODEJS_BRIDGE_VERSION, PYTHON_BRIDGE_VERSION, - INSTANCE_ID + INSTANCE_ID, + SKILLS_RESOLVERS_NLP_MODEL_PATH, + GLOBAL_RESOLVERS_NLP_MODEL_PATH, + MAIN_NLP_MODEL_PATH } from '@/constants' dotenv.config() @@ -41,11 +44,6 @@ dotenv.config() const googleCloudPath = 'core/config/voice/google-cloud.json' const watsonSttPath = 'core/config/voice/watson-stt.json' const watsonTtsPath = 'core/config/voice/watson-tts.json' - const globalResolversNlpModelPath = - 'core/data/models/leon-global-resolvers-model.nlp' - const skillsResolversNlpModelPath = - 'core/data/models/leon-skills-resolvers-model.nlp' - const mainNlpModelPath = 'core/data/models/leon-main-model.nlp' const report = { can_run: { title: 'Run', type: 'error', v: true }, can_run_skill: { title: 'Run skills', type: 'error', v: true }, @@ -362,9 +360,10 @@ dotenv.config() LogHelper.info('Global resolvers NLP model state') if ( - !fs.existsSync(globalResolversNlpModelPath) || - !Object.keys(await fs.promises.readFile(globalResolversNlpModelPath)) - .length + !fs.existsSync(GLOBAL_RESOLVERS_NLP_MODEL_PATH) || + !Object.keys( + await fs.promises.readFile(GLOBAL_RESOLVERS_NLP_MODEL_PATH) + ).length ) { const state = 'Global resolvers NLP model not found or broken' @@ -391,9 +390,10 @@ dotenv.config() LogHelper.info('Skills resolvers NLP model state') if ( - !fs.existsSync(skillsResolversNlpModelPath) || - !Object.keys(await fs.promises.readFile(skillsResolversNlpModelPath)) - .length + !fs.existsSync(SKILLS_RESOLVERS_NLP_MODEL_PATH) || + !Object.keys( + await fs.promises.readFile(SKILLS_RESOLVERS_NLP_MODEL_PATH) + ).length ) { const state = 'Skills resolvers NLP model not found or broken' @@ -420,8 +420,8 @@ dotenv.config() LogHelper.info('Main NLP model state') if ( - !fs.existsSync(mainNlpModelPath) || - !Object.keys(await fs.promises.readFile(mainNlpModelPath)).length + !fs.existsSync(MAIN_NLP_MODEL_PATH) || + !Object.keys(await fs.promises.readFile(MAIN_NLP_MODEL_PATH)).length ) { const state = 'Main NLP model not found or broken' diff --git a/scripts/train/train.js b/scripts/train/train.js index e7f547d6..4d22e51a 100644 --- a/scripts/train/train.js +++ b/scripts/train/train.js @@ -3,6 +3,11 @@ import { Nlp } from '@nlpjs/nlp' import { LangAll } from '@nlpjs/lang-all' import dotenv from 'dotenv' +import { + MAIN_NLP_MODEL_PATH, + SKILLS_RESOLVERS_NLP_MODEL_PATH, + GLOBAL_RESOLVERS_NLP_MODEL_PATH +} from '@/constants' import { LogHelper } from '@/helpers/log-helper' import { LangHelper } from '@/helpers/lang-helper' @@ -20,12 +25,6 @@ dotenv.config() */ export default () => new Promise(async (resolve, reject) => { - const globalResolversModelFileName = - 'core/data/models/leon-global-resolvers-model.nlp' - const skillsResolversModelFileName = - 'core/data/models/leon-skills-resolvers-model.nlp' - const mainModelFileName = 'core/data/models/leon-main-model.nlp' - try { /** * Global resolvers NLP model configuration @@ -41,7 +40,8 @@ export default () => globalResolversNluManager.settings.log = false globalResolversNluManager.settings.trainByDomain = false - globalResolversNlp.settings.modelFileName = globalResolversModelFileName + globalResolversNlp.settings.modelFileName = + GLOBAL_RESOLVERS_NLP_MODEL_PATH globalResolversNlp.settings.threshold = 0.8 /** @@ -58,7 +58,8 @@ export default () => skillsResolversNluManager.settings.log = false skillsResolversNluManager.settings.trainByDomain = true - skillsResolversNlp.settings.modelFileName = skillsResolversModelFileName + skillsResolversNlp.settings.modelFileName = + SKILLS_RESOLVERS_NLP_MODEL_PATH skillsResolversNlp.settings.threshold = 0.8 /** @@ -79,7 +80,7 @@ export default () => mainNlp.settings.forceNER = true // https://github.com/axa-group/nlp.js/blob/master/examples/17-ner-nlg/index.js // mainNlp.settings.nlu = { useNoneFeature: true } mainNlp.settings.calculateSentiment = true - mainNlp.settings.modelFileName = mainModelFileName + mainNlp.settings.modelFileName = MAIN_NLP_MODEL_PATH mainNlp.settings.threshold = 0.8 /** @@ -104,7 +105,7 @@ export default () => await globalResolversNlp.train() LogHelper.success( - `Global resolvers NLP model saved in ${globalResolversModelFileName}` + `Global resolvers NLP model saved in ${GLOBAL_RESOLVERS_NLP_MODEL_PATH}` ) resolve() } catch (e) { @@ -116,7 +117,7 @@ export default () => await skillsResolversNlp.train() LogHelper.success( - `Skills resolvers NLP model saved in ${skillsResolversModelFileName}` + `Skills resolvers NLP model saved in ${SKILLS_RESOLVERS_NLP_MODEL_PATH}` ) resolve() } catch (e) { @@ -127,7 +128,7 @@ export default () => try { await mainNlp.train() - LogHelper.success(`Main NLP model saved in ${mainModelFileName}`) + LogHelper.success(`Main NLP model saved in ${MAIN_NLP_MODEL_PATH}`) resolve() } catch (e) { LogHelper.error(`Failed to save main NLP model: ${e}`) diff --git a/server/src/constants.ts b/server/src/constants.ts index bfea7197..1d091aec 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -218,6 +218,19 @@ export const SERVER_PATH = path.join( export const TMP_PATH = path.join(SERVER_PATH, 'tmp') export const LEON_FILE_PATH = path.join(process.cwd(), 'leon.json') +/** + * NLP models paths + */ +export const MAIN_NLP_MODEL_PATH = path.join(MODELS_PATH, 'leon-main-model.nlp') +export const GLOBAL_RESOLVERS_NLP_MODEL_PATH = path.join( + MODELS_PATH, + 'leon-global-resolvers-model.nlp' +) +export const SKILLS_RESOLVERS_NLP_MODEL_PATH = path.join( + MODELS_PATH, + 'leon-skills-resolvers-model.nlp' +) + /** * LLMs * @see k-quants comparison: https://github.com/ggerganov/llama.cpp/pull/1684 diff --git a/server/src/core/nlp/nlu/model-loader.ts b/server/src/core/nlp/nlu/model-loader.ts index 7bd1c574..ca794456 100644 --- a/server/src/core/nlp/nlu/model-loader.ts +++ b/server/src/core/nlp/nlu/model-loader.ts @@ -1,12 +1,15 @@ import fs from 'node:fs' -import path from 'node:path' import { containerBootstrap } from '@nlpjs/core-loader' import { Nlp } from '@nlpjs/nlp' import { BuiltinMicrosoft } from '@nlpjs/builtin-microsoft' import { LangAll } from '@nlpjs/lang-all' -import { MODELS_PATH } from '@/constants' +import { + MAIN_NLP_MODEL_PATH, + GLOBAL_RESOLVERS_NLP_MODEL_PATH, + SKILLS_RESOLVERS_NLP_MODEL_PATH +} from '@/constants' import { MICROSOFT_BUILT_IN_ENTITIES } from '@/core/nlp/nlu/ner' import { LogHelper } from '@/helpers/log-helper' @@ -44,13 +47,9 @@ export default class ModelLoader { */ public loadNLPModels(): Promise<[void, void, void]> { return Promise.all([ - this.loadGlobalResolversModel( - path.join(MODELS_PATH, 'leon-global-resolvers-model.nlp') - ), - this.loadSkillsResolversModel( - path.join(MODELS_PATH, 'leon-skills-resolvers-model.nlp') - ), - this.loadMainModel(path.join(MODELS_PATH, 'leon-main-model.nlp')) + this.loadGlobalResolversModel(GLOBAL_RESOLVERS_NLP_MODEL_PATH), + this.loadSkillsResolversModel(SKILLS_RESOLVERS_NLP_MODEL_PATH), + this.loadMainModel(MAIN_NLP_MODEL_PATH) ]) }