leon-ai/leon
1
1
Fork 0
mirror of https://github.com/leon-ai/leon.git synced 2024-09-11 10:25:40 +03:00
Code Issues Projects Releases Wiki Activity

feat: train skills resolvers and remap as per changes

Browse Source
This commit is contained in:
louistiti 2022-07-01 00:19:36 +08:00
parent 0fe0e9a717
commit 82df0a3c23
No known key found for this signature in database
GPG Key ID: 7ECA3DD523793FE6
9 changed files with 82 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/data/en/resolvers/affirmation_denial.json → core/data/en/global-resolvers/affirmation_denial.json
View File

@ -1,7 +1,7 @@
{
"name": "affirmation_denial",
"intents": {
"system.resolver.affirmation": {
"affirmation": {
"utterance_samples": [
"Yes",
"Yep",
@ -21,11 +21,12 @@
"That's right",
"That works",
"Go ahead",
"Why not"
"Why not",
"Please"
],
"value": true
},
"system.resolver.denial": {
"denial": {
"utterance_samples": [
"No",
"No no don't",

4
core/data/fr/resolvers/affirmation_denial.json → core/data/fr/global-resolvers/affirmation_denial.json
View File

@ -1,7 +1,7 @@
{
"name": "affirmation_denial",
"intents": {
"system.resolver.affirmation": {
"affirmation": {
"utterance_samples": [
"Oui",
"Yep",
@ -23,7 +23,7 @@
],
"value": true
},
"system.resolver.denial": {
"denial": {
"utterance_samples": [
"Non",
"Ne le fais pas",

2
scripts/train/train-main-model/train-global-entities.js
View File

@ -5,6 +5,7 @@ import log from '@/helpers/log'
/**
* Train global entities
* Add global entities annotations (@...)
*/
export default (lang, nlp) => new Promise((resolve) => {
log.title('Global entities training')
@ -13,7 +14,6 @@ export default (lang, nlp) => new Promise((resolve) => {
const globalEntityFiles = fs.readdirSync(globalEntitiesPath)
const newEntitiesObj = { }
// Add global entities annotations (@...)
for (let i = 0; i < globalEntityFiles.length; i += 1) {
const globalEntityFileName = globalEntityFiles[i]
const [entityName] = globalEntityFileName.split('.')

5
scripts/train/train-main-model/train-skills-actions.js
View File

@ -8,7 +8,7 @@ import string from '@/helpers/string'
import domain from '@/helpers/domain'
/**
* Train global entities
* Train skills actions
*/
export default (lang, nlp) => new Promise(async (resolve) => {
log.title('Skills actions training')
@ -16,7 +16,6 @@ export default (lang, nlp) => new Promise(async (resolve) => {
const supportedActionTypes = ['dialog', 'logic']
const [domainKeys, domains] = await Promise.all([domain.list(), domain.getDomainsObj()])
// Train skills actions
for (let i = 0; i < domainKeys.length; i += 1) {
const currentDomain = domains[domainKeys[i]]
const skillKeys = Object.keys(currentDomain.skills)
@ -49,7 +48,7 @@ export default (lang, nlp) => new Promise(async (resolve) => {
process.exit(1)
}
nlp.assignDomain(lang, `${skillName}.${actionName}`, currentDomain.name)
nlp.assignDomain(lang, intent, currentDomain.name)
if (slots) {
for (let l = 0; l < slots.length; l += 1) {

11
scripts/train/train-resolvers-model/train-global-resolvers.js
View File

@ -4,16 +4,14 @@ import fs from 'fs'
import log from '@/helpers/log'
/**
* Train global entities
* Train global resolvers
*/
export default (lang, nlp) => new Promise((resolve) => {
log.title('Global resolvers training')
const resolversPath = path.join(process.cwd(), 'core/data', lang, 'resolvers')
const resolversPath = path.join(process.cwd(), 'core/data', lang, 'global-resolvers')
const resolverFiles = fs.readdirSync(resolversPath)
// Add global entities annotations (@...)
// Train resolvers
for (let i = 0; i < resolverFiles.length; i += 1) {
const resolverFileName = resolverFiles[i]
const resolverPath = path.join(resolversPath, resolverFileName)
@ -24,12 +22,13 @@ export default (lang, nlp) => new Promise((resolve) => {
for (let j = 0; j < intentKeys.length; j += 1) {
const intentName = intentKeys[j]
const intent = `resolver.global.${intentName}`
const intentObj = resolverIntents[intentName]
nlp.assignDomain(lang, intentName, 'system')
nlp.assignDomain(lang, intent, 'system')
for (let k = 0; k < intentObj.utterance_samples.length; k += 1) {
nlp.addDocument(lang, intentObj.utterance_samples[k], intentName)
nlp.addDocument(lang, intentObj.utterance_samples[k], intent)
}
}

55
scripts/train/train-resolvers-model/train-skills-resolvers.js Normal file
View File

@ -0,0 +1,55 @@
import path from 'path'
import fs from 'fs'
import log from '@/helpers/log'
import domain from '@/helpers/domain'
import json from '@/helpers/json'
/**
* Train skills resolvers
*/
export default (lang, nlp) => new Promise(async (resolve) => {
log.title('Skills resolvers training')
const [domainKeys, domains] = await Promise.all([domain.list(), domain.getDomainsObj()])
domainKeys.forEach((domainName) => {
const currentDomain = domains[domainName]
const skillKeys = Object.keys(currentDomain.skills)
skillKeys.forEach(async (skillName) => {
const currentSkill = currentDomain.skills[skillName]
const nluFilePath = path.join(currentSkill.path, 'nlu', `${lang}.json`)
if (fs.existsSync(nluFilePath)) {
const { resolvers } = await json.loadNluData(nluFilePath, lang)
if (resolvers) {
const resolversKeys = Object.keys(resolvers)
resolversKeys.forEach((resolverName) => {
const resolver = resolvers[resolverName]
const intentKeys = Object.keys(resolver.intents)
log.info(`[${lang}] Training "${resolverName}" resolver...`)
intentKeys.forEach((intentName) => {
const intent = `resolver.${currentSkill.name}.${intentName}`
const intentObj = resolver.intents[intentName]
nlp.assignDomain(lang, intent, currentDomain.name)
intentObj.utterance_samples.forEach((utteranceSample) => {
nlp.addDocument(lang, utteranceSample, intent)
})
})
log.success(`[${lang}] "${resolverName}" resolver trained`)
})
}
}
})
})
resolve()
})

3
scripts/train/train.js
View File

@ -6,6 +6,7 @@ import dotenv from 'dotenv'
import log from '@/helpers/log'
import lang from '@/helpers/lang'
import trainGlobalResolvers from './train-resolvers-model/train-global-resolvers'
import trainSkillsResolvers from './train-resolvers-model/train-skills-resolvers'
import trainGlobalEntities from './train-main-model/train-global-entities'
import trainSkillsActions from './train-main-model/train-skills-actions'
@ -68,6 +69,8 @@ export default () => new Promise(async (resolve, reject) => {
resolversNlp.addLanguage(lang)
// eslint-disable-next-line no-await-in-loop
await trainGlobalResolvers(lang, resolversNlp)
// eslint-disable-next-line no-await-in-loop
await trainSkillsResolvers(lang, resolversNlp)
mainNlp.addLanguage(lang)
// eslint-disable-next-line no-await-in-loop

9
server/src/core/nlu.js
View File

@ -237,17 +237,20 @@ class Nlu {
} else if (expectedItemType === 'resolver') {
const { intent } = await this.resolversNlp.process(utterance)
const resolveResolvers = (resolver, intent) => {
const resolversPath = join(process.cwd(), 'core/data', this.brain.lang, 'resolvers')
const resolversPath = join(process.cwd(), 'core/data', this.brain.lang, 'global-resolvers')
const { intents } = JSON.parse(fs.readFileSync(join(resolversPath, `${resolver}.json`)))
// E.g. resolver.global.denial -> denial
intent = intent.substring(intent.lastIndexOf('.') + 1)
return [{
name: expectedItemName,
value: intents[intent].value
}]
}
// Resolve resolver if one has been found
if (intent.includes('system.resolver')) {
// Resolve resolver if global resolver or skill resolver has been found
if (intent.includes('resolver.global') || intent.includes(`resolver.${skillName}`)) {
log.title('NLU')
log.success('Resolvers resolved:')
this.nluResultObj.resolvers = resolveResolvers(expectedItemName, intent)

8
skills/social_communication/mbti/nlu/en.json
View File

@ -23,7 +23,7 @@
"resolvers": {
"mbti_quiz": {
"intents": {
"resolver.1_a": {
"1_a": {
"utterance_samples": [
"Interact with many",
"Including strangers",
@ -31,14 +31,14 @@
],
"value": "1_a"
},
"resolver.1_b": {
"1_b": {
"utterance_samples": [
"Interact with a few",
"Know to [me|you]"
],
"value": "1_b"
},
"resolver.2_a": {
"2_a": {
"utterance_samples": [
"Realistic than speculative",
"I'm not a dreamer",
@ -46,7 +46,7 @@
],
"value": "2_a"
},
"resolver.2_b": {
"2_b": {
"utterance_samples": [
"Speculative than realistic",
"Speculative",