mirror of
https://github.com/leon-ai/leon.git
synced 2024-08-17 06:00:33 +03:00
refactor: unify NLP models paths
This commit is contained in:
parent
34170a38ac
commit
fb5c258cf6
@ -22,7 +22,10 @@ import {
|
||||
PYTHON_TCP_SERVER_VERSION,
|
||||
NODEJS_BRIDGE_VERSION,
|
||||
PYTHON_BRIDGE_VERSION,
|
||||
INSTANCE_ID
|
||||
INSTANCE_ID,
|
||||
SKILLS_RESOLVERS_NLP_MODEL_PATH,
|
||||
GLOBAL_RESOLVERS_NLP_MODEL_PATH,
|
||||
MAIN_NLP_MODEL_PATH
|
||||
} from '@/constants'
|
||||
|
||||
dotenv.config()
|
||||
@ -41,11 +44,6 @@ dotenv.config()
|
||||
const googleCloudPath = 'core/config/voice/google-cloud.json'
|
||||
const watsonSttPath = 'core/config/voice/watson-stt.json'
|
||||
const watsonTtsPath = 'core/config/voice/watson-tts.json'
|
||||
const globalResolversNlpModelPath =
|
||||
'core/data/models/leon-global-resolvers-model.nlp'
|
||||
const skillsResolversNlpModelPath =
|
||||
'core/data/models/leon-skills-resolvers-model.nlp'
|
||||
const mainNlpModelPath = 'core/data/models/leon-main-model.nlp'
|
||||
const report = {
|
||||
can_run: { title: 'Run', type: 'error', v: true },
|
||||
can_run_skill: { title: 'Run skills', type: 'error', v: true },
|
||||
@ -362,9 +360,10 @@ dotenv.config()
|
||||
LogHelper.info('Global resolvers NLP model state')
|
||||
|
||||
if (
|
||||
!fs.existsSync(globalResolversNlpModelPath) ||
|
||||
!Object.keys(await fs.promises.readFile(globalResolversNlpModelPath))
|
||||
.length
|
||||
!fs.existsSync(GLOBAL_RESOLVERS_NLP_MODEL_PATH) ||
|
||||
!Object.keys(
|
||||
await fs.promises.readFile(GLOBAL_RESOLVERS_NLP_MODEL_PATH)
|
||||
).length
|
||||
) {
|
||||
const state = 'Global resolvers NLP model not found or broken'
|
||||
|
||||
@ -391,9 +390,10 @@ dotenv.config()
|
||||
LogHelper.info('Skills resolvers NLP model state')
|
||||
|
||||
if (
|
||||
!fs.existsSync(skillsResolversNlpModelPath) ||
|
||||
!Object.keys(await fs.promises.readFile(skillsResolversNlpModelPath))
|
||||
.length
|
||||
!fs.existsSync(SKILLS_RESOLVERS_NLP_MODEL_PATH) ||
|
||||
!Object.keys(
|
||||
await fs.promises.readFile(SKILLS_RESOLVERS_NLP_MODEL_PATH)
|
||||
).length
|
||||
) {
|
||||
const state = 'Skills resolvers NLP model not found or broken'
|
||||
|
||||
@ -420,8 +420,8 @@ dotenv.config()
|
||||
LogHelper.info('Main NLP model state')
|
||||
|
||||
if (
|
||||
!fs.existsSync(mainNlpModelPath) ||
|
||||
!Object.keys(await fs.promises.readFile(mainNlpModelPath)).length
|
||||
!fs.existsSync(MAIN_NLP_MODEL_PATH) ||
|
||||
!Object.keys(await fs.promises.readFile(MAIN_NLP_MODEL_PATH)).length
|
||||
) {
|
||||
const state = 'Main NLP model not found or broken'
|
||||
|
||||
|
@ -3,6 +3,11 @@ import { Nlp } from '@nlpjs/nlp'
|
||||
import { LangAll } from '@nlpjs/lang-all'
|
||||
import dotenv from 'dotenv'
|
||||
|
||||
import {
|
||||
MAIN_NLP_MODEL_PATH,
|
||||
SKILLS_RESOLVERS_NLP_MODEL_PATH,
|
||||
GLOBAL_RESOLVERS_NLP_MODEL_PATH
|
||||
} from '@/constants'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
import { LangHelper } from '@/helpers/lang-helper'
|
||||
|
||||
@ -20,12 +25,6 @@ dotenv.config()
|
||||
*/
|
||||
export default () =>
|
||||
new Promise(async (resolve, reject) => {
|
||||
const globalResolversModelFileName =
|
||||
'core/data/models/leon-global-resolvers-model.nlp'
|
||||
const skillsResolversModelFileName =
|
||||
'core/data/models/leon-skills-resolvers-model.nlp'
|
||||
const mainModelFileName = 'core/data/models/leon-main-model.nlp'
|
||||
|
||||
try {
|
||||
/**
|
||||
* Global resolvers NLP model configuration
|
||||
@ -41,7 +40,8 @@ export default () =>
|
||||
|
||||
globalResolversNluManager.settings.log = false
|
||||
globalResolversNluManager.settings.trainByDomain = false
|
||||
globalResolversNlp.settings.modelFileName = globalResolversModelFileName
|
||||
globalResolversNlp.settings.modelFileName =
|
||||
GLOBAL_RESOLVERS_NLP_MODEL_PATH
|
||||
globalResolversNlp.settings.threshold = 0.8
|
||||
|
||||
/**
|
||||
@ -58,7 +58,8 @@ export default () =>
|
||||
|
||||
skillsResolversNluManager.settings.log = false
|
||||
skillsResolversNluManager.settings.trainByDomain = true
|
||||
skillsResolversNlp.settings.modelFileName = skillsResolversModelFileName
|
||||
skillsResolversNlp.settings.modelFileName =
|
||||
SKILLS_RESOLVERS_NLP_MODEL_PATH
|
||||
skillsResolversNlp.settings.threshold = 0.8
|
||||
|
||||
/**
|
||||
@ -79,7 +80,7 @@ export default () =>
|
||||
mainNlp.settings.forceNER = true // https://github.com/axa-group/nlp.js/blob/master/examples/17-ner-nlg/index.js
|
||||
// mainNlp.settings.nlu = { useNoneFeature: true }
|
||||
mainNlp.settings.calculateSentiment = true
|
||||
mainNlp.settings.modelFileName = mainModelFileName
|
||||
mainNlp.settings.modelFileName = MAIN_NLP_MODEL_PATH
|
||||
mainNlp.settings.threshold = 0.8
|
||||
|
||||
/**
|
||||
@ -104,7 +105,7 @@ export default () =>
|
||||
await globalResolversNlp.train()
|
||||
|
||||
LogHelper.success(
|
||||
`Global resolvers NLP model saved in ${globalResolversModelFileName}`
|
||||
`Global resolvers NLP model saved in ${GLOBAL_RESOLVERS_NLP_MODEL_PATH}`
|
||||
)
|
||||
resolve()
|
||||
} catch (e) {
|
||||
@ -116,7 +117,7 @@ export default () =>
|
||||
await skillsResolversNlp.train()
|
||||
|
||||
LogHelper.success(
|
||||
`Skills resolvers NLP model saved in ${skillsResolversModelFileName}`
|
||||
`Skills resolvers NLP model saved in ${SKILLS_RESOLVERS_NLP_MODEL_PATH}`
|
||||
)
|
||||
resolve()
|
||||
} catch (e) {
|
||||
@ -127,7 +128,7 @@ export default () =>
|
||||
try {
|
||||
await mainNlp.train()
|
||||
|
||||
LogHelper.success(`Main NLP model saved in ${mainModelFileName}`)
|
||||
LogHelper.success(`Main NLP model saved in ${MAIN_NLP_MODEL_PATH}`)
|
||||
resolve()
|
||||
} catch (e) {
|
||||
LogHelper.error(`Failed to save main NLP model: ${e}`)
|
||||
|
@ -218,6 +218,19 @@ export const SERVER_PATH = path.join(
|
||||
export const TMP_PATH = path.join(SERVER_PATH, 'tmp')
|
||||
export const LEON_FILE_PATH = path.join(process.cwd(), 'leon.json')
|
||||
|
||||
/**
|
||||
* NLP models paths
|
||||
*/
|
||||
export const MAIN_NLP_MODEL_PATH = path.join(MODELS_PATH, 'leon-main-model.nlp')
|
||||
export const GLOBAL_RESOLVERS_NLP_MODEL_PATH = path.join(
|
||||
MODELS_PATH,
|
||||
'leon-global-resolvers-model.nlp'
|
||||
)
|
||||
export const SKILLS_RESOLVERS_NLP_MODEL_PATH = path.join(
|
||||
MODELS_PATH,
|
||||
'leon-skills-resolvers-model.nlp'
|
||||
)
|
||||
|
||||
/**
|
||||
* LLMs
|
||||
* @see k-quants comparison: https://github.com/ggerganov/llama.cpp/pull/1684
|
||||
|
@ -1,12 +1,15 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
|
||||
import { containerBootstrap } from '@nlpjs/core-loader'
|
||||
import { Nlp } from '@nlpjs/nlp'
|
||||
import { BuiltinMicrosoft } from '@nlpjs/builtin-microsoft'
|
||||
import { LangAll } from '@nlpjs/lang-all'
|
||||
|
||||
import { MODELS_PATH } from '@/constants'
|
||||
import {
|
||||
MAIN_NLP_MODEL_PATH,
|
||||
GLOBAL_RESOLVERS_NLP_MODEL_PATH,
|
||||
SKILLS_RESOLVERS_NLP_MODEL_PATH
|
||||
} from '@/constants'
|
||||
import { MICROSOFT_BUILT_IN_ENTITIES } from '@/core/nlp/nlu/ner'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
|
||||
@ -44,13 +47,9 @@ export default class ModelLoader {
|
||||
*/
|
||||
public loadNLPModels(): Promise<[void, void, void]> {
|
||||
return Promise.all([
|
||||
this.loadGlobalResolversModel(
|
||||
path.join(MODELS_PATH, 'leon-global-resolvers-model.nlp')
|
||||
),
|
||||
this.loadSkillsResolversModel(
|
||||
path.join(MODELS_PATH, 'leon-skills-resolvers-model.nlp')
|
||||
),
|
||||
this.loadMainModel(path.join(MODELS_PATH, 'leon-main-model.nlp'))
|
||||
this.loadGlobalResolversModel(GLOBAL_RESOLVERS_NLP_MODEL_PATH),
|
||||
this.loadSkillsResolversModel(SKILLS_RESOLVERS_NLP_MODEL_PATH),
|
||||
this.loadMainModel(MAIN_NLP_MODEL_PATH)
|
||||
])
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user