mirror of
https://github.com/leon-ai/leon.git
synced 2024-09-11 18:27:21 +03:00
refactor: unify NLP models paths
This commit is contained in:
parent
34170a38ac
commit
fb5c258cf6
@ -22,7 +22,10 @@ import {
|
|||||||
PYTHON_TCP_SERVER_VERSION,
|
PYTHON_TCP_SERVER_VERSION,
|
||||||
NODEJS_BRIDGE_VERSION,
|
NODEJS_BRIDGE_VERSION,
|
||||||
PYTHON_BRIDGE_VERSION,
|
PYTHON_BRIDGE_VERSION,
|
||||||
INSTANCE_ID
|
INSTANCE_ID,
|
||||||
|
SKILLS_RESOLVERS_NLP_MODEL_PATH,
|
||||||
|
GLOBAL_RESOLVERS_NLP_MODEL_PATH,
|
||||||
|
MAIN_NLP_MODEL_PATH
|
||||||
} from '@/constants'
|
} from '@/constants'
|
||||||
|
|
||||||
dotenv.config()
|
dotenv.config()
|
||||||
@ -41,11 +44,6 @@ dotenv.config()
|
|||||||
const googleCloudPath = 'core/config/voice/google-cloud.json'
|
const googleCloudPath = 'core/config/voice/google-cloud.json'
|
||||||
const watsonSttPath = 'core/config/voice/watson-stt.json'
|
const watsonSttPath = 'core/config/voice/watson-stt.json'
|
||||||
const watsonTtsPath = 'core/config/voice/watson-tts.json'
|
const watsonTtsPath = 'core/config/voice/watson-tts.json'
|
||||||
const globalResolversNlpModelPath =
|
|
||||||
'core/data/models/leon-global-resolvers-model.nlp'
|
|
||||||
const skillsResolversNlpModelPath =
|
|
||||||
'core/data/models/leon-skills-resolvers-model.nlp'
|
|
||||||
const mainNlpModelPath = 'core/data/models/leon-main-model.nlp'
|
|
||||||
const report = {
|
const report = {
|
||||||
can_run: { title: 'Run', type: 'error', v: true },
|
can_run: { title: 'Run', type: 'error', v: true },
|
||||||
can_run_skill: { title: 'Run skills', type: 'error', v: true },
|
can_run_skill: { title: 'Run skills', type: 'error', v: true },
|
||||||
@ -362,9 +360,10 @@ dotenv.config()
|
|||||||
LogHelper.info('Global resolvers NLP model state')
|
LogHelper.info('Global resolvers NLP model state')
|
||||||
|
|
||||||
if (
|
if (
|
||||||
!fs.existsSync(globalResolversNlpModelPath) ||
|
!fs.existsSync(GLOBAL_RESOLVERS_NLP_MODEL_PATH) ||
|
||||||
!Object.keys(await fs.promises.readFile(globalResolversNlpModelPath))
|
!Object.keys(
|
||||||
.length
|
await fs.promises.readFile(GLOBAL_RESOLVERS_NLP_MODEL_PATH)
|
||||||
|
).length
|
||||||
) {
|
) {
|
||||||
const state = 'Global resolvers NLP model not found or broken'
|
const state = 'Global resolvers NLP model not found or broken'
|
||||||
|
|
||||||
@ -391,9 +390,10 @@ dotenv.config()
|
|||||||
LogHelper.info('Skills resolvers NLP model state')
|
LogHelper.info('Skills resolvers NLP model state')
|
||||||
|
|
||||||
if (
|
if (
|
||||||
!fs.existsSync(skillsResolversNlpModelPath) ||
|
!fs.existsSync(SKILLS_RESOLVERS_NLP_MODEL_PATH) ||
|
||||||
!Object.keys(await fs.promises.readFile(skillsResolversNlpModelPath))
|
!Object.keys(
|
||||||
.length
|
await fs.promises.readFile(SKILLS_RESOLVERS_NLP_MODEL_PATH)
|
||||||
|
).length
|
||||||
) {
|
) {
|
||||||
const state = 'Skills resolvers NLP model not found or broken'
|
const state = 'Skills resolvers NLP model not found or broken'
|
||||||
|
|
||||||
@ -420,8 +420,8 @@ dotenv.config()
|
|||||||
LogHelper.info('Main NLP model state')
|
LogHelper.info('Main NLP model state')
|
||||||
|
|
||||||
if (
|
if (
|
||||||
!fs.existsSync(mainNlpModelPath) ||
|
!fs.existsSync(MAIN_NLP_MODEL_PATH) ||
|
||||||
!Object.keys(await fs.promises.readFile(mainNlpModelPath)).length
|
!Object.keys(await fs.promises.readFile(MAIN_NLP_MODEL_PATH)).length
|
||||||
) {
|
) {
|
||||||
const state = 'Main NLP model not found or broken'
|
const state = 'Main NLP model not found or broken'
|
||||||
|
|
||||||
|
@ -3,6 +3,11 @@ import { Nlp } from '@nlpjs/nlp'
|
|||||||
import { LangAll } from '@nlpjs/lang-all'
|
import { LangAll } from '@nlpjs/lang-all'
|
||||||
import dotenv from 'dotenv'
|
import dotenv from 'dotenv'
|
||||||
|
|
||||||
|
import {
|
||||||
|
MAIN_NLP_MODEL_PATH,
|
||||||
|
SKILLS_RESOLVERS_NLP_MODEL_PATH,
|
||||||
|
GLOBAL_RESOLVERS_NLP_MODEL_PATH
|
||||||
|
} from '@/constants'
|
||||||
import { LogHelper } from '@/helpers/log-helper'
|
import { LogHelper } from '@/helpers/log-helper'
|
||||||
import { LangHelper } from '@/helpers/lang-helper'
|
import { LangHelper } from '@/helpers/lang-helper'
|
||||||
|
|
||||||
@ -20,12 +25,6 @@ dotenv.config()
|
|||||||
*/
|
*/
|
||||||
export default () =>
|
export default () =>
|
||||||
new Promise(async (resolve, reject) => {
|
new Promise(async (resolve, reject) => {
|
||||||
const globalResolversModelFileName =
|
|
||||||
'core/data/models/leon-global-resolvers-model.nlp'
|
|
||||||
const skillsResolversModelFileName =
|
|
||||||
'core/data/models/leon-skills-resolvers-model.nlp'
|
|
||||||
const mainModelFileName = 'core/data/models/leon-main-model.nlp'
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
/**
|
/**
|
||||||
* Global resolvers NLP model configuration
|
* Global resolvers NLP model configuration
|
||||||
@ -41,7 +40,8 @@ export default () =>
|
|||||||
|
|
||||||
globalResolversNluManager.settings.log = false
|
globalResolversNluManager.settings.log = false
|
||||||
globalResolversNluManager.settings.trainByDomain = false
|
globalResolversNluManager.settings.trainByDomain = false
|
||||||
globalResolversNlp.settings.modelFileName = globalResolversModelFileName
|
globalResolversNlp.settings.modelFileName =
|
||||||
|
GLOBAL_RESOLVERS_NLP_MODEL_PATH
|
||||||
globalResolversNlp.settings.threshold = 0.8
|
globalResolversNlp.settings.threshold = 0.8
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -58,7 +58,8 @@ export default () =>
|
|||||||
|
|
||||||
skillsResolversNluManager.settings.log = false
|
skillsResolversNluManager.settings.log = false
|
||||||
skillsResolversNluManager.settings.trainByDomain = true
|
skillsResolversNluManager.settings.trainByDomain = true
|
||||||
skillsResolversNlp.settings.modelFileName = skillsResolversModelFileName
|
skillsResolversNlp.settings.modelFileName =
|
||||||
|
SKILLS_RESOLVERS_NLP_MODEL_PATH
|
||||||
skillsResolversNlp.settings.threshold = 0.8
|
skillsResolversNlp.settings.threshold = 0.8
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -79,7 +80,7 @@ export default () =>
|
|||||||
mainNlp.settings.forceNER = true // https://github.com/axa-group/nlp.js/blob/master/examples/17-ner-nlg/index.js
|
mainNlp.settings.forceNER = true // https://github.com/axa-group/nlp.js/blob/master/examples/17-ner-nlg/index.js
|
||||||
// mainNlp.settings.nlu = { useNoneFeature: true }
|
// mainNlp.settings.nlu = { useNoneFeature: true }
|
||||||
mainNlp.settings.calculateSentiment = true
|
mainNlp.settings.calculateSentiment = true
|
||||||
mainNlp.settings.modelFileName = mainModelFileName
|
mainNlp.settings.modelFileName = MAIN_NLP_MODEL_PATH
|
||||||
mainNlp.settings.threshold = 0.8
|
mainNlp.settings.threshold = 0.8
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -104,7 +105,7 @@ export default () =>
|
|||||||
await globalResolversNlp.train()
|
await globalResolversNlp.train()
|
||||||
|
|
||||||
LogHelper.success(
|
LogHelper.success(
|
||||||
`Global resolvers NLP model saved in ${globalResolversModelFileName}`
|
`Global resolvers NLP model saved in ${GLOBAL_RESOLVERS_NLP_MODEL_PATH}`
|
||||||
)
|
)
|
||||||
resolve()
|
resolve()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@ -116,7 +117,7 @@ export default () =>
|
|||||||
await skillsResolversNlp.train()
|
await skillsResolversNlp.train()
|
||||||
|
|
||||||
LogHelper.success(
|
LogHelper.success(
|
||||||
`Skills resolvers NLP model saved in ${skillsResolversModelFileName}`
|
`Skills resolvers NLP model saved in ${SKILLS_RESOLVERS_NLP_MODEL_PATH}`
|
||||||
)
|
)
|
||||||
resolve()
|
resolve()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@ -127,7 +128,7 @@ export default () =>
|
|||||||
try {
|
try {
|
||||||
await mainNlp.train()
|
await mainNlp.train()
|
||||||
|
|
||||||
LogHelper.success(`Main NLP model saved in ${mainModelFileName}`)
|
LogHelper.success(`Main NLP model saved in ${MAIN_NLP_MODEL_PATH}`)
|
||||||
resolve()
|
resolve()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
LogHelper.error(`Failed to save main NLP model: ${e}`)
|
LogHelper.error(`Failed to save main NLP model: ${e}`)
|
||||||
|
@ -218,6 +218,19 @@ export const SERVER_PATH = path.join(
|
|||||||
export const TMP_PATH = path.join(SERVER_PATH, 'tmp')
|
export const TMP_PATH = path.join(SERVER_PATH, 'tmp')
|
||||||
export const LEON_FILE_PATH = path.join(process.cwd(), 'leon.json')
|
export const LEON_FILE_PATH = path.join(process.cwd(), 'leon.json')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* NLP models paths
|
||||||
|
*/
|
||||||
|
export const MAIN_NLP_MODEL_PATH = path.join(MODELS_PATH, 'leon-main-model.nlp')
|
||||||
|
export const GLOBAL_RESOLVERS_NLP_MODEL_PATH = path.join(
|
||||||
|
MODELS_PATH,
|
||||||
|
'leon-global-resolvers-model.nlp'
|
||||||
|
)
|
||||||
|
export const SKILLS_RESOLVERS_NLP_MODEL_PATH = path.join(
|
||||||
|
MODELS_PATH,
|
||||||
|
'leon-skills-resolvers-model.nlp'
|
||||||
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* LLMs
|
* LLMs
|
||||||
* @see k-quants comparison: https://github.com/ggerganov/llama.cpp/pull/1684
|
* @see k-quants comparison: https://github.com/ggerganov/llama.cpp/pull/1684
|
||||||
|
@ -1,12 +1,15 @@
|
|||||||
import fs from 'node:fs'
|
import fs from 'node:fs'
|
||||||
import path from 'node:path'
|
|
||||||
|
|
||||||
import { containerBootstrap } from '@nlpjs/core-loader'
|
import { containerBootstrap } from '@nlpjs/core-loader'
|
||||||
import { Nlp } from '@nlpjs/nlp'
|
import { Nlp } from '@nlpjs/nlp'
|
||||||
import { BuiltinMicrosoft } from '@nlpjs/builtin-microsoft'
|
import { BuiltinMicrosoft } from '@nlpjs/builtin-microsoft'
|
||||||
import { LangAll } from '@nlpjs/lang-all'
|
import { LangAll } from '@nlpjs/lang-all'
|
||||||
|
|
||||||
import { MODELS_PATH } from '@/constants'
|
import {
|
||||||
|
MAIN_NLP_MODEL_PATH,
|
||||||
|
GLOBAL_RESOLVERS_NLP_MODEL_PATH,
|
||||||
|
SKILLS_RESOLVERS_NLP_MODEL_PATH
|
||||||
|
} from '@/constants'
|
||||||
import { MICROSOFT_BUILT_IN_ENTITIES } from '@/core/nlp/nlu/ner'
|
import { MICROSOFT_BUILT_IN_ENTITIES } from '@/core/nlp/nlu/ner'
|
||||||
import { LogHelper } from '@/helpers/log-helper'
|
import { LogHelper } from '@/helpers/log-helper'
|
||||||
|
|
||||||
@ -44,13 +47,9 @@ export default class ModelLoader {
|
|||||||
*/
|
*/
|
||||||
public loadNLPModels(): Promise<[void, void, void]> {
|
public loadNLPModels(): Promise<[void, void, void]> {
|
||||||
return Promise.all([
|
return Promise.all([
|
||||||
this.loadGlobalResolversModel(
|
this.loadGlobalResolversModel(GLOBAL_RESOLVERS_NLP_MODEL_PATH),
|
||||||
path.join(MODELS_PATH, 'leon-global-resolvers-model.nlp')
|
this.loadSkillsResolversModel(SKILLS_RESOLVERS_NLP_MODEL_PATH),
|
||||||
),
|
this.loadMainModel(MAIN_NLP_MODEL_PATH)
|
||||||
this.loadSkillsResolversModel(
|
|
||||||
path.join(MODELS_PATH, 'leon-skills-resolvers-model.nlp')
|
|
||||||
),
|
|
||||||
this.loadMainModel(path.join(MODELS_PATH, 'leon-main-model.nlp'))
|
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user