mirror of
https://github.com/leon-ai/leon.git
synced 2024-09-11 10:25:40 +03:00
feat: train skills resolvers and remap as per changes
This commit is contained in:
parent
0fe0e9a717
commit
82df0a3c23
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "affirmation_denial",
|
||||
"intents": {
|
||||
"system.resolver.affirmation": {
|
||||
"affirmation": {
|
||||
"utterance_samples": [
|
||||
"Yes",
|
||||
"Yep",
|
||||
@ -21,11 +21,12 @@
|
||||
"That's right",
|
||||
"That works",
|
||||
"Go ahead",
|
||||
"Why not"
|
||||
"Why not",
|
||||
"Please"
|
||||
],
|
||||
"value": true
|
||||
},
|
||||
"system.resolver.denial": {
|
||||
"denial": {
|
||||
"utterance_samples": [
|
||||
"No",
|
||||
"No no don't",
|
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "affirmation_denial",
|
||||
"intents": {
|
||||
"system.resolver.affirmation": {
|
||||
"affirmation": {
|
||||
"utterance_samples": [
|
||||
"Oui",
|
||||
"Yep",
|
||||
@ -23,7 +23,7 @@
|
||||
],
|
||||
"value": true
|
||||
},
|
||||
"system.resolver.denial": {
|
||||
"denial": {
|
||||
"utterance_samples": [
|
||||
"Non",
|
||||
"Ne le fais pas",
|
@ -5,6 +5,7 @@ import log from '@/helpers/log'
|
||||
|
||||
/**
|
||||
* Train global entities
|
||||
* Add global entities annotations (@...)
|
||||
*/
|
||||
export default (lang, nlp) => new Promise((resolve) => {
|
||||
log.title('Global entities training')
|
||||
@ -13,7 +14,6 @@ export default (lang, nlp) => new Promise((resolve) => {
|
||||
const globalEntityFiles = fs.readdirSync(globalEntitiesPath)
|
||||
const newEntitiesObj = { }
|
||||
|
||||
// Add global entities annotations (@...)
|
||||
for (let i = 0; i < globalEntityFiles.length; i += 1) {
|
||||
const globalEntityFileName = globalEntityFiles[i]
|
||||
const [entityName] = globalEntityFileName.split('.')
|
||||
|
@ -8,7 +8,7 @@ import string from '@/helpers/string'
|
||||
import domain from '@/helpers/domain'
|
||||
|
||||
/**
|
||||
* Train global entities
|
||||
* Train skills actions
|
||||
*/
|
||||
export default (lang, nlp) => new Promise(async (resolve) => {
|
||||
log.title('Skills actions training')
|
||||
@ -16,7 +16,6 @@ export default (lang, nlp) => new Promise(async (resolve) => {
|
||||
const supportedActionTypes = ['dialog', 'logic']
|
||||
const [domainKeys, domains] = await Promise.all([domain.list(), domain.getDomainsObj()])
|
||||
|
||||
// Train skills actions
|
||||
for (let i = 0; i < domainKeys.length; i += 1) {
|
||||
const currentDomain = domains[domainKeys[i]]
|
||||
const skillKeys = Object.keys(currentDomain.skills)
|
||||
@ -49,7 +48,7 @@ export default (lang, nlp) => new Promise(async (resolve) => {
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
nlp.assignDomain(lang, `${skillName}.${actionName}`, currentDomain.name)
|
||||
nlp.assignDomain(lang, intent, currentDomain.name)
|
||||
|
||||
if (slots) {
|
||||
for (let l = 0; l < slots.length; l += 1) {
|
||||
|
@ -4,16 +4,14 @@ import fs from 'fs'
|
||||
import log from '@/helpers/log'
|
||||
|
||||
/**
|
||||
* Train global entities
|
||||
* Train global resolvers
|
||||
*/
|
||||
export default (lang, nlp) => new Promise((resolve) => {
|
||||
log.title('Global resolvers training')
|
||||
|
||||
const resolversPath = path.join(process.cwd(), 'core/data', lang, 'resolvers')
|
||||
const resolversPath = path.join(process.cwd(), 'core/data', lang, 'global-resolvers')
|
||||
const resolverFiles = fs.readdirSync(resolversPath)
|
||||
|
||||
// Add global entities annotations (@...)
|
||||
// Train resolvers
|
||||
for (let i = 0; i < resolverFiles.length; i += 1) {
|
||||
const resolverFileName = resolverFiles[i]
|
||||
const resolverPath = path.join(resolversPath, resolverFileName)
|
||||
@ -24,12 +22,13 @@ export default (lang, nlp) => new Promise((resolve) => {
|
||||
|
||||
for (let j = 0; j < intentKeys.length; j += 1) {
|
||||
const intentName = intentKeys[j]
|
||||
const intent = `resolver.global.${intentName}`
|
||||
const intentObj = resolverIntents[intentName]
|
||||
|
||||
nlp.assignDomain(lang, intentName, 'system')
|
||||
nlp.assignDomain(lang, intent, 'system')
|
||||
|
||||
for (let k = 0; k < intentObj.utterance_samples.length; k += 1) {
|
||||
nlp.addDocument(lang, intentObj.utterance_samples[k], intentName)
|
||||
nlp.addDocument(lang, intentObj.utterance_samples[k], intent)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -0,0 +1,55 @@
|
||||
import path from 'path'
|
||||
import fs from 'fs'
|
||||
|
||||
import log from '@/helpers/log'
|
||||
import domain from '@/helpers/domain'
|
||||
import json from '@/helpers/json'
|
||||
|
||||
/**
|
||||
* Train skills resolvers
|
||||
*/
|
||||
export default (lang, nlp) => new Promise(async (resolve) => {
|
||||
log.title('Skills resolvers training')
|
||||
|
||||
const [domainKeys, domains] = await Promise.all([domain.list(), domain.getDomainsObj()])
|
||||
|
||||
domainKeys.forEach((domainName) => {
|
||||
const currentDomain = domains[domainName]
|
||||
const skillKeys = Object.keys(currentDomain.skills)
|
||||
|
||||
skillKeys.forEach(async (skillName) => {
|
||||
const currentSkill = currentDomain.skills[skillName]
|
||||
const nluFilePath = path.join(currentSkill.path, 'nlu', `${lang}.json`)
|
||||
|
||||
if (fs.existsSync(nluFilePath)) {
|
||||
const { resolvers } = await json.loadNluData(nluFilePath, lang)
|
||||
|
||||
if (resolvers) {
|
||||
const resolversKeys = Object.keys(resolvers)
|
||||
|
||||
resolversKeys.forEach((resolverName) => {
|
||||
const resolver = resolvers[resolverName]
|
||||
const intentKeys = Object.keys(resolver.intents)
|
||||
|
||||
log.info(`[${lang}] Training "${resolverName}" resolver...`)
|
||||
|
||||
intentKeys.forEach((intentName) => {
|
||||
const intent = `resolver.${currentSkill.name}.${intentName}`
|
||||
const intentObj = resolver.intents[intentName]
|
||||
|
||||
nlp.assignDomain(lang, intent, currentDomain.name)
|
||||
|
||||
intentObj.utterance_samples.forEach((utteranceSample) => {
|
||||
nlp.addDocument(lang, utteranceSample, intent)
|
||||
})
|
||||
})
|
||||
|
||||
log.success(`[${lang}] "${resolverName}" resolver trained`)
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
resolve()
|
||||
})
|
@ -6,6 +6,7 @@ import dotenv from 'dotenv'
|
||||
import log from '@/helpers/log'
|
||||
import lang from '@/helpers/lang'
|
||||
import trainGlobalResolvers from './train-resolvers-model/train-global-resolvers'
|
||||
import trainSkillsResolvers from './train-resolvers-model/train-skills-resolvers'
|
||||
import trainGlobalEntities from './train-main-model/train-global-entities'
|
||||
import trainSkillsActions from './train-main-model/train-skills-actions'
|
||||
|
||||
@ -68,6 +69,8 @@ export default () => new Promise(async (resolve, reject) => {
|
||||
resolversNlp.addLanguage(lang)
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await trainGlobalResolvers(lang, resolversNlp)
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await trainSkillsResolvers(lang, resolversNlp)
|
||||
|
||||
mainNlp.addLanguage(lang)
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
|
@ -237,17 +237,20 @@ class Nlu {
|
||||
} else if (expectedItemType === 'resolver') {
|
||||
const { intent } = await this.resolversNlp.process(utterance)
|
||||
const resolveResolvers = (resolver, intent) => {
|
||||
const resolversPath = join(process.cwd(), 'core/data', this.brain.lang, 'resolvers')
|
||||
const resolversPath = join(process.cwd(), 'core/data', this.brain.lang, 'global-resolvers')
|
||||
const { intents } = JSON.parse(fs.readFileSync(join(resolversPath, `${resolver}.json`)))
|
||||
|
||||
// E.g. resolver.global.denial -> denial
|
||||
intent = intent.substring(intent.lastIndexOf('.') + 1)
|
||||
|
||||
return [{
|
||||
name: expectedItemName,
|
||||
value: intents[intent].value
|
||||
}]
|
||||
}
|
||||
|
||||
// Resolve resolver if one has been found
|
||||
if (intent.includes('system.resolver')) {
|
||||
// Resolve resolver if global resolver or skill resolver has been found
|
||||
if (intent.includes('resolver.global') || intent.includes(`resolver.${skillName}`)) {
|
||||
log.title('NLU')
|
||||
log.success('Resolvers resolved:')
|
||||
this.nluResultObj.resolvers = resolveResolvers(expectedItemName, intent)
|
||||
|
@ -23,7 +23,7 @@
|
||||
"resolvers": {
|
||||
"mbti_quiz": {
|
||||
"intents": {
|
||||
"resolver.1_a": {
|
||||
"1_a": {
|
||||
"utterance_samples": [
|
||||
"Interact with many",
|
||||
"Including strangers",
|
||||
@ -31,14 +31,14 @@
|
||||
],
|
||||
"value": "1_a"
|
||||
},
|
||||
"resolver.1_b": {
|
||||
"1_b": {
|
||||
"utterance_samples": [
|
||||
"Interact with a few",
|
||||
"Know to [me|you]"
|
||||
],
|
||||
"value": "1_b"
|
||||
},
|
||||
"resolver.2_a": {
|
||||
"2_a": {
|
||||
"utterance_samples": [
|
||||
"Realistic than speculative",
|
||||
"I'm not a dreamer",
|
||||
@ -46,7 +46,7 @@
|
||||
],
|
||||
"value": "2_a"
|
||||
},
|
||||
"resolver.2_b": {
|
||||
"2_b": {
|
||||
"utterance_samples": [
|
||||
"Speculative than realistic",
|
||||
"Speculative",
|
||||
|
Loading…
Reference in New Issue
Block a user