From 623039415ad8aeadda2cbc249b0b6ffe475c1aea Mon Sep 17 00:00:00 2001 From: Divlo Date: Mon, 15 May 2023 21:18:54 +0200 Subject: [PATCH 01/10] feat(bridge/nodejs): add Widget abstract class --- app/src/js/client.js | 10 +++++++++- bridges/nodejs/src/sdk/aurora/button.ts | 14 ++++++------- bridges/nodejs/src/sdk/types.ts | 7 ------- bridges/nodejs/src/sdk/widget.ts | 26 ++++++------------------- skills/leon/age/src/actions/run.ts | 5 +---- 5 files changed, 22 insertions(+), 40 deletions(-) diff --git a/app/src/js/client.js b/app/src/js/client.js index 2d073e9c..c5a05b3c 100644 --- a/app/src/js/client.js +++ b/app/src/js/client.js @@ -79,7 +79,15 @@ export default class Client { const root = createRoot(container) // TODO: widget: pass props and dynamic component loading according to type - root.render(Button({ children: 'OK' })) + const widgets = { + Button: (options) => { + return Button({ + children: options.text + }) + } + } + + root.render(widgets[data.type](data.options)) }) this.socket.on('audio-forwarded', (data, cb) => { diff --git a/bridges/nodejs/src/sdk/aurora/button.ts b/bridges/nodejs/src/sdk/aurora/button.ts index f070453c..e4fcd2e1 100644 --- a/bridges/nodejs/src/sdk/aurora/button.ts +++ b/bridges/nodejs/src/sdk/aurora/button.ts @@ -1,14 +1,12 @@ +import { Widget } from '../widget' + // TODO: contains the button API. rendering engine <-> SDK -interface Options { +interface ButtonOptions { text: string } -export class Button { - private readonly text: string - - constructor(options: Options) { - this.text = options.text - - console.log('Button constructor', this.text) +export class Button extends Widget { + public constructor(options: ButtonOptions) { + super(options) } } diff --git a/bridges/nodejs/src/sdk/types.ts b/bridges/nodejs/src/sdk/types.ts index b56d86a9..f837c124 100644 --- a/bridges/nodejs/src/sdk/types.ts +++ b/bridges/nodejs/src/sdk/types.ts @@ -3,17 +3,10 @@ */ import type { ActionParams, IntentObject } from '@/core/brain/types' -import type { Button } from '@sdk/aurora/button' - export type { ActionParams, IntentObject } export type ActionFunction = (params: ActionParams) => Promise -/** - * Aurora - */ -export type AuroraComponent = Button // TODO - /** * Answer types */ diff --git a/bridges/nodejs/src/sdk/widget.ts b/bridges/nodejs/src/sdk/widget.ts index edc47fcb..2c6070b9 100644 --- a/bridges/nodejs/src/sdk/widget.ts +++ b/bridges/nodejs/src/sdk/widget.ts @@ -1,23 +1,9 @@ -import type { AuroraComponent } from '@sdk/types' +export abstract class Widget { + public readonly type: string + public readonly options: T -interface SerializedAuroraComponent { - type: string - props: Record -} - -export class Widget { - private readonly components: SerializedAuroraComponent[] - - constructor(components: AuroraComponent[]) { - this.components = components.map((component) => { - return { - type: component.constructor.name, - props: { - ...component - } - } - }) - - console.log('Widget constructor', this.components) + public constructor(options: T) { + this.type = this.constructor.name + this.options = options } } diff --git a/skills/leon/age/src/actions/run.ts b/skills/leon/age/src/actions/run.ts index aa2a060b..0d58bbd9 100644 --- a/skills/leon/age/src/actions/run.ts +++ b/skills/leon/age/src/actions/run.ts @@ -2,7 +2,6 @@ import utility from 'utility' // TODO import type { ActionFunction } from '@sdk/types' import { leon } from '@sdk/leon' -import { Widget } from '@sdk/widget' import { Network } from '@sdk/network' import { Button } from '@sdk/aurora/button' import { Memory } from '@sdk/memory' @@ -21,9 +20,7 @@ export const run: ActionFunction = async function () { const button = new Button({ text: 'Hello world from action skill' }) - const widget = new Widget([button]) - - await leon.answer({ widget }) + await leon.answer({ widget: button }) /// From 839c7bfc3c6e07003422814b3b67313c4bb7793e Mon Sep 17 00:00:00 2001 From: Divlo Date: Mon, 15 May 2023 22:58:40 +0200 Subject: [PATCH 02/10] fix(bridge/nodejs): improve Memory types with external skill memory --- bridges/nodejs/src/sdk/memory.ts | 13 +++++++------ skills/leon/age/src/actions/run.ts | 13 ++++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/bridges/nodejs/src/sdk/memory.ts b/bridges/nodejs/src/sdk/memory.ts index c13cdcc5..5181aaa0 100644 --- a/bridges/nodejs/src/sdk/memory.ts +++ b/bridges/nodejs/src/sdk/memory.ts @@ -8,7 +8,7 @@ interface MemoryOptions { defaultMemory?: T } -export class Memory { +export class Memory { private readonly memoryPath: string | undefined private readonly name: string private readonly defaultMemory: T | undefined @@ -55,12 +55,13 @@ export class Memory { * Read the memory * @example read() */ - public async read(): Promise { + public async read(): Promise { + if (!this.memoryPath) { + throw new Error( + `You cannot read the memory "${this.name}" as it belongs to another skill which haven't written to this memory yet` + ) + } try { - if (!this.memoryPath) { - return - } - if (!fs.existsSync(this.memoryPath)) { await this.clear() } diff --git a/skills/leon/age/src/actions/run.ts b/skills/leon/age/src/actions/run.ts index 0d58bbd9..fa4004ea 100644 --- a/skills/leon/age/src/actions/run.ts +++ b/skills/leon/age/src/actions/run.ts @@ -24,14 +24,17 @@ export const run: ActionFunction = async function () { /// - const otherSkillMemory = new Memory({ + const otherSkillMemory = new Memory({ name: 'productivity:todo_list:db' }) + try { + const todoLists = await otherSkillMemory.read() + console.log('todoLists', todoLists) + } catch { + console.log('todoLists', []) + } + const postsMemory = new Memory({ name: 'posts', defaultMemory: [] }) - const todoLists = await otherSkillMemory.read() - - console.log('todoLists', todoLists) - await postsMemory.write([ { id: 0, From efd1061f30063138a6c1f86971318f1891d6cd23 Mon Sep 17 00:00:00 2001 From: Divlo Date: Mon, 15 May 2023 23:42:27 +0200 Subject: [PATCH 03/10] fix: typo missing "GB" for Free RAM --- scripts/check.js | 4 ++-- server/src/pre-check.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/check.js b/scripts/check.js index b65876c7..5e99464e 100644 --- a/scripts/check.js +++ b/scripts/check.js @@ -159,11 +159,11 @@ dotenv.config() if (Math.round(freeRAMInGB) < MINIMUM_REQUIRED_RAM) { report.can_run.v = false LogHelper.error( - `Free RAM: ${freeRAMInGB} | Total RAM: ${totalRAMInGB} GB. Leon needs at least ${MINIMUM_REQUIRED_RAM} GB of RAM` + `Free RAM: ${freeRAMInGB} GB | Total RAM: ${totalRAMInGB} GB. Leon needs at least ${MINIMUM_REQUIRED_RAM} GB of RAM` ) } else { LogHelper.success( - `Free RAM: ${freeRAMInGB} | Total RAM: ${totalRAMInGB} GB` + `Free RAM: ${freeRAMInGB} GB | Total RAM: ${totalRAMInGB} GB` ) } diff --git a/server/src/pre-check.ts b/server/src/pre-check.ts index 263ed577..c97c8b40 100644 --- a/server/src/pre-check.ts +++ b/server/src/pre-check.ts @@ -101,11 +101,11 @@ const GLOBAL_DATA_SCHEMAS = { if (freeRAMInGB < MINIMUM_REQUIRED_RAM) { LogHelper.warning( - `Free RAM: ${freeRAMInGB} | Total RAM: ${totalRAMInGB} GB. Leon needs at least ${MINIMUM_REQUIRED_RAM} GB of RAM. It may not work as expected.` + `Free RAM: ${freeRAMInGB} GB | Total RAM: ${totalRAMInGB} GB. Leon needs at least ${MINIMUM_REQUIRED_RAM} GB of RAM. It may not work as expected.` ) } else { LogHelper.success( - `Minimum required RAM: ${MINIMUM_REQUIRED_RAM} GB | Free RAM: ${freeRAMInGB} | Total RAM: ${totalRAMInGB} GB` + `Minimum required RAM: ${MINIMUM_REQUIRED_RAM} GB | Free RAM: ${freeRAMInGB} GB | Total RAM: ${totalRAMInGB} GB` ) } From 01305af669ec117da2988acdad9e6a1da2593d3f Mon Sep 17 00:00:00 2001 From: louistiti Date: Tue, 16 May 2023 23:07:12 +0800 Subject: [PATCH 04/10] refactor(bridge/nodejs): `protected` memory constructor --- bridges/nodejs/src/sdk/memory.ts | 1 + bridges/nodejs/src/sdk/widget.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/nodejs/src/sdk/memory.ts b/bridges/nodejs/src/sdk/memory.ts index 5181aaa0..2f5e271d 100644 --- a/bridges/nodejs/src/sdk/memory.ts +++ b/bridges/nodejs/src/sdk/memory.ts @@ -61,6 +61,7 @@ export class Memory { `You cannot read the memory "${this.name}" as it belongs to another skill which haven't written to this memory yet` ) } + try { if (!fs.existsSync(this.memoryPath)) { await this.clear() diff --git a/bridges/nodejs/src/sdk/widget.ts b/bridges/nodejs/src/sdk/widget.ts index 2c6070b9..8a32e90c 100644 --- a/bridges/nodejs/src/sdk/widget.ts +++ b/bridges/nodejs/src/sdk/widget.ts @@ -2,7 +2,7 @@ export abstract class Widget { public readonly type: string public readonly options: T - public constructor(options: T) { + protected constructor(options: T) { this.type = this.constructor.name this.options = options } From f6300a3fb89b4c4d30c3359102b2769d58c6f798 Mon Sep 17 00:00:00 2001 From: Louis Grenard Date: Tue, 16 May 2023 23:14:02 +0800 Subject: [PATCH 05/10] docs: update `README.md` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 696ee295..68b3685c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ _

Your open-source personal assistant.

_ I'm taking a lot of time to work on the new core of Leon due to personal reasons. I can only work on it after work and on weekends. Hence, **I'm blocking any potential contribution as the whole core of Leon is coming with many breaking changes**. Many of you are willing to contribute in Leon (create new skills, help to improve the core, translations and so on...), a big thanks to every one of you! -I wish one day I could work on Leon full time, but it's not possible at the moment as I need to pay my bills. I have some ideas about how to monetize Leon in the future (Leon's core will always remain open source), but before to get there there is still a long way to go. +While I would love to devote more time to Leon, I'm currently unable to do so because I have bills to pay. I have some ideas about how to monetize Leon in the future (Leon's core will always remain open source), but before to get there there is still a long way to go. Until then, any financial support by [sponsoring Leon](http://sponsor.getleon.ai) is much appreciated 🙂 From cb6462b749dc363fa8d64c54ab4420d9600d2958 Mon Sep 17 00:00:00 2001 From: louistiti Date: Wed, 17 May 2023 18:31:39 +0800 Subject: [PATCH 06/10] feat: define new skill answer format --- server/src/schemas/skill-schemas.ts | 17 ++++++++++++----- skills/leon/age/config/en.json | 8 +++++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/server/src/schemas/skill-schemas.ts b/server/src/schemas/skill-schemas.ts index 58c02215..edfbe67d 100644 --- a/server/src/schemas/skill-schemas.ts +++ b/server/src/schemas/skill-schemas.ts @@ -23,6 +23,15 @@ const skillDataTypes = [ Type.Literal('global_resolver'), Type.Literal('entity') ] +const answerTypes = [ + Type.Array(Type.String()), + Type.Array( + Type.Object({ + speech: Type.String(), + text: Type.String() + }) + ) +] const skillCustomEnumEntityType = Type.Object( { type: Type.Literal('enum'), @@ -184,8 +193,8 @@ export const skillConfigSchemaObject = Type.Strict( ) ), utterance_samples: Type.Optional(Type.Array(Type.String())), - answers: Type.Optional(Type.Array(Type.String())), - unknown_answers: Type.Optional(Type.Array(Type.String())), + answers: Type.Optional(Type.Union(answerTypes)), + unknown_answers: Type.Optional(Type.Union(answerTypes)), suggestions: Type.Optional( Type.Array(Type.String(), { description: @@ -235,9 +244,7 @@ export const skillConfigSchemaObject = Type.Strict( { additionalProperties: false } ) ), - answers: Type.Optional( - Type.Record(Type.String(), Type.Array(Type.String())) - ), + answers: Type.Optional(Type.Record(Type.String(), Type.Union(answerTypes))), entities: Type.Optional(Type.Record(Type.String(), Type.String())), resolvers: Type.Optional( Type.Record( diff --git a/skills/leon/age/config/en.json b/skills/leon/age/config/en.json index 9341e8e8..4858acc5 100644 --- a/skills/leon/age/config/en.json +++ b/skills/leon/age/config/en.json @@ -9,6 +9,12 @@ "answers": { "default": ["I'm..."], "greet": ["Hey, just a try %name% again %name%", "Another try, hi"], - "answer": ["%answer%"] + "answer": ["%answer%"], + "test": [ + { + "speech": "This will be said out loud", + "text": "This will be shown in the chat" + } + ] } } From 9e77953976e627bd46d21181d1ef3d375d79a10a Mon Sep 17 00:00:00 2001 From: louistiti Date: Wed, 17 May 2023 22:21:11 +0800 Subject: [PATCH 07/10] refactor(bridge/nodejs): use `AnswerConfig` type from core schema --- bridges/nodejs/src/sdk/leon.ts | 9 +++++++-- bridges/nodejs/src/sdk/types.ts | 2 ++ server/src/schemas/skill-schemas.ts | 23 +++++++++++------------ skills/leon/age/src/actions/run.ts | 4 ++++ 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/bridges/nodejs/src/sdk/leon.ts b/bridges/nodejs/src/sdk/leon.ts index d4847ac7..79797887 100644 --- a/bridges/nodejs/src/sdk/leon.ts +++ b/bridges/nodejs/src/sdk/leon.ts @@ -1,4 +1,9 @@ -import type { AnswerData, AnswerInput, AnswerOutput } from '@sdk/types' +import type { + AnswerData, + AnswerInput, + AnswerOutput, + AnswerConfig +} from '@sdk/types' import { INTENT_OBJECT, SKILL_CONFIG, @@ -45,7 +50,7 @@ class Leon { public setAnswerData( answerKey: string | undefined, data: AnswerData = null - ): string | null | undefined { + ): AnswerConfig | null | undefined { if (answerKey) { try { // In case the answer key is a raw answer diff --git a/bridges/nodejs/src/sdk/types.ts b/bridges/nodejs/src/sdk/types.ts index f837c124..44330f53 100644 --- a/bridges/nodejs/src/sdk/types.ts +++ b/bridges/nodejs/src/sdk/types.ts @@ -2,6 +2,7 @@ * Action types */ import type { ActionParams, IntentObject } from '@/core/brain/types' +import type { AnswerConfigSchema } from '@/schemas/skill-schemas' export type { ActionParams, IntentObject } @@ -40,3 +41,4 @@ export interface WidgetAnswer extends Answer { } export type AnswerData = Record | null export type AnswerInput = TextAnswer | WidgetAnswer +export type AnswerConfig = AnswerConfigSchema diff --git a/server/src/schemas/skill-schemas.ts b/server/src/schemas/skill-schemas.ts index edfbe67d..e594ade0 100644 --- a/server/src/schemas/skill-schemas.ts +++ b/server/src/schemas/skill-schemas.ts @@ -23,15 +23,13 @@ const skillDataTypes = [ Type.Literal('global_resolver'), Type.Literal('entity') ] -const answerTypes = [ - Type.Array(Type.String()), - Type.Array( - Type.Object({ - speech: Type.String(), - text: Type.String() - }) - ) -] +const answerTypes = Type.Union([ + Type.String(), + Type.Object({ + speech: Type.String(), + text: Type.String() + }) +]) const skillCustomEnumEntityType = Type.Object( { type: Type.Literal('enum'), @@ -193,8 +191,8 @@ export const skillConfigSchemaObject = Type.Strict( ) ), utterance_samples: Type.Optional(Type.Array(Type.String())), - answers: Type.Optional(Type.Union(answerTypes)), - unknown_answers: Type.Optional(Type.Union(answerTypes)), + answers: Type.Optional(Type.Array(answerTypes)), + unknown_answers: Type.Optional(Type.Array(answerTypes)), suggestions: Type.Optional( Type.Array(Type.String(), { description: @@ -244,7 +242,7 @@ export const skillConfigSchemaObject = Type.Strict( { additionalProperties: false } ) ), - answers: Type.Optional(Type.Record(Type.String(), Type.Union(answerTypes))), + answers: Type.Optional(Type.Record(Type.String(), Type.Array(answerTypes))), entities: Type.Optional(Type.Record(Type.String(), Type.String())), resolvers: Type.Optional( Type.Record( @@ -277,3 +275,4 @@ export type SkillCustomRegexEntityTypeSchema = Static< export type SkillCustomEnumEntityTypeSchema = Static< typeof skillCustomEnumEntityType > +export type AnswerConfigSchema = Static diff --git a/skills/leon/age/src/actions/run.ts b/skills/leon/age/src/actions/run.ts index fa4004ea..995e42ee 100644 --- a/skills/leon/age/src/actions/run.ts +++ b/skills/leon/age/src/actions/run.ts @@ -17,6 +17,10 @@ interface Post { } export const run: ActionFunction = async function () { + await leon.answer({ key: 'test' }) + + /// + const button = new Button({ text: 'Hello world from action skill' }) From 7f9b60a6cd7d61a0d28b8ec1da9f0bdf7cd41d3e Mon Sep 17 00:00:00 2001 From: louistiti Date: Thu, 18 May 2023 00:00:40 +0800 Subject: [PATCH 08/10] feat(server): differentiate skill text answer from speech answer (WIP) --- bridges/nodejs/src/sdk/leon.ts | 38 +++++++- bridges/nodejs/src/sdk/types.ts | 29 ++---- core/skills-endpoints.json | 140 ++++++++++++++-------------- server/src/core/brain/brain.ts | 39 +++++--- server/src/core/brain/types.ts | 23 ++++- server/src/schemas/skill-schemas.ts | 2 +- skills/leon/good_bye/config/en.json | 13 +-- 7 files changed, 169 insertions(+), 115 deletions(-) diff --git a/bridges/nodejs/src/sdk/leon.ts b/bridges/nodejs/src/sdk/leon.ts index 79797887..dd1c527d 100644 --- a/bridges/nodejs/src/sdk/leon.ts +++ b/bridges/nodejs/src/sdk/leon.ts @@ -59,7 +59,7 @@ class Leon { } const answers = SKILL_CONFIG.answers[answerKey] ?? '' - let answer: string + let answer: AnswerConfig if (Array.isArray(answers)) { answer = answers[Math.floor(Math.random() * answers.length)] ?? '' @@ -69,7 +69,22 @@ class Leon { if (data) { for (const key in data) { - answer = answer.replaceAll(`%${key}%`, String(data[key])) + // In case the answer needs speech and text differentiation + if (typeof answer !== 'string' && answer.text) { + answer.text = answer.text.replaceAll( + `%${key}%`, + String(data[key]) + ) + answer.speech = answer.speech.replaceAll( + `%${key}%`, + String(data[key]) + ) + } else { + answer = (answer as string).replaceAll( + `%${key}%`, + String(data[key]) + ) + } } } @@ -77,7 +92,22 @@ class Leon { const { variables } = SKILL_CONFIG for (const key in variables) { - answer = answer.replaceAll(`%${key}%`, String(variables[key])) + // In case the answer needs speech and text differentiation + if (typeof answer !== 'string' && answer.text) { + answer.text = answer.text.replaceAll( + `%${key}%`, + String(variables[key]) + ) + answer.speech = answer.speech.replaceAll( + `%${key}%`, + String(variables[key]) + ) + } else { + answer = (answer as string).replaceAll( + `%${key}%`, + String(variables[key]) + ) + } } } @@ -108,7 +138,7 @@ class Leon { answerInput.widget && !answerInput.key ? 'widget' : (answerInput.key as string), - speech: this.setAnswerData(answerInput.key, answerInput.data) ?? '', + answer: this.setAnswerData(answerInput.key, answerInput.data) ?? '', core: answerInput.core, options: this.getSRCConfig('options') } diff --git a/bridges/nodejs/src/sdk/types.ts b/bridges/nodejs/src/sdk/types.ts index 44330f53..2ee8633a 100644 --- a/bridges/nodejs/src/sdk/types.ts +++ b/bridges/nodejs/src/sdk/types.ts @@ -1,8 +1,13 @@ /** * Action types */ -import type { ActionParams, IntentObject } from '@/core/brain/types' -import type { AnswerConfigSchema } from '@/schemas/skill-schemas' +import type { + ActionParams, + IntentObject, + SkillAnswerCoreData, + SkillAnswerOutput +} from '@/core/brain/types' +import type { SkillAnswerConfigSchema } from '@/schemas/skill-schemas' export type { ActionParams, IntentObject } @@ -11,26 +16,11 @@ export type ActionFunction = (params: ActionParams) => Promise /** * Answer types */ -export interface AnswerOutput extends IntentObject { - output: { - codes: string - speech: string - core?: AnswerCoreData - widget?: unknown // TODO - options: Record - } -} -export interface AnswerCoreData { - restart?: boolean - isInActionLoop?: boolean - showNextActionSuggestions?: boolean - showSuggestions?: boolean -} export interface Answer { key?: string widget?: unknown // TODO data?: AnswerData - core?: AnswerCoreData + core?: SkillAnswerCoreData } export interface TextAnswer extends Answer { key: string @@ -41,4 +31,5 @@ export interface WidgetAnswer extends Answer { } export type AnswerData = Record | null export type AnswerInput = TextAnswer | WidgetAnswer -export type AnswerConfig = AnswerConfigSchema +export type AnswerOutput = SkillAnswerOutput +export type AnswerConfig = SkillAnswerConfigSchema diff --git a/core/skills-endpoints.json b/core/skills-endpoints.json index 459c3b34..ad55eb08 100644 --- a/core/skills-endpoints.json +++ b/core/skills-endpoints.json @@ -1,5 +1,75 @@ { "endpoints": [ + { + "method": "GET", + "route": "/api/action/leon/age/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/color/favorite_color", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/color/why", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/color/color_hexadecimal", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/good_bye/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/greeting/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/introduction/introduce_leon", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/introduction/gather_basic_info", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/introduction/remember", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/joke/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/meaning_of_life/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/partner_assistant/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/random_number/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/welcome/run", + "params": [] + }, { "method": "POST", "route": "/api/action/news/github_trends/run", @@ -140,76 +210,6 @@ "method": "GET", "route": "/api/action/games/rochambeau/rematch", "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/age/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/color/favorite_color", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/color/why", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/color/color_hexadecimal", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/good_bye/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/greeting/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/introduction/introduce_leon", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/introduction/gather_basic_info", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/introduction/remember", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/joke/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/meaning_of_life/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/partner_assistant/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/random_number/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/welcome/run", - "params": [] } ] } diff --git a/server/src/core/brain/brain.ts b/server/src/core/brain/brain.ts index d9b3e809..6d21c773 100644 --- a/server/src/core/brain/brain.ts +++ b/server/src/core/brain/brain.ts @@ -9,7 +9,11 @@ import type { NERCustomEntity, NLUResult } from '@/core/nlp/types' -import type { SkillConfigSchema, SkillSchema } from '@/schemas/skill-schemas' +import type { + SkillAnswerConfigSchema, + SkillConfigSchema, + SkillSchema +} from '@/schemas/skill-schemas' import type { BrainProcessResult, IntentObject, @@ -29,6 +33,7 @@ import { LogHelper } from '@/helpers/log-helper' import { SkillDomainHelper } from '@/helpers/skill-domain-helper' import { StringHelper } from '@/helpers/string-helper' import Synchronizer from '@/core/synchronizer' +import type { AnswerOutput } from '@sdk/types' export default class Brain { private static instance: Brain @@ -43,7 +48,7 @@ export default class Brain { private domainFriendlyName = '' private skillFriendlyName = '' private skillOutput = '' - private speeches: string[] = [] + private answers: SkillAnswerConfigSchema[] = [] public isMuted = false // Close Leon mouth if true; e.g. over HTTP constructor() { @@ -97,19 +102,22 @@ export default class Brain { /** * Make Leon talk */ - public talk(rawSpeech: string, end = false): void { + public talk(answer: SkillAnswerConfigSchema, end = false): void { LogHelper.title('Brain') LogHelper.info('Talking...') - if (rawSpeech !== '') { + if (answer !== '') { + const textAnswer = typeof answer === 'string' ? answer : answer.text + const speechAnswer = typeof answer === 'string' ? answer : answer.speech + if (HAS_TTS) { // Stripe HTML to a whitespace. Whitespace to let the TTS respects punctuation - const speech = rawSpeech.replace(/<(?:.|\n)*?>/gm, ' ') + const speech = speechAnswer.replace(/<(?:.|\n)*?>/gm, ' ') TTS.add(speech, end) } - SOCKET_SERVER.socket?.emit('answer', rawSpeech) + SOCKET_SERVER.socket?.emit('answer', textAnswer) } } @@ -192,7 +200,7 @@ export default class Brain { data: Buffer ): Promise | void { try { - const obj = JSON.parse(data.toString()) + const obj = JSON.parse(data.toString()) as AnswerOutput if (typeof obj === 'object') { LogHelper.title(`${this.skillFriendlyName} skill (on data)`) @@ -202,11 +210,14 @@ export default class Brain { SOCKET_SERVER.socket?.emit('widget', obj.output.widget) } - const speech = obj.output.speech.toString() + // TODO: remove this condition when Python skills outputs are updated (replace "speech" with "answer") + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const { answer, speech } = obj.output if (!this.isMuted) { - this.talk(speech) + this.talk(answer || speech) } - this.speeches.push(speech) + this.answers.push(answer) this.skillOutput = data.toString() return Promise.resolve(null) @@ -231,11 +242,13 @@ export default class Brain { '%skill_name%': this.skillFriendlyName, '%domain_name%': this.domainFriendlyName })}!` + if (!this.isMuted) { this.talk(speech) SOCKET_SERVER.socket?.emit('is-typing', false) } - this.speeches.push(speech) + + this.answers.push(speech) } /** @@ -480,6 +493,8 @@ export default class Brain { await SkillDomainHelper.getSkillConfig(configFilePath, this._lang) const utteranceHasEntities = nluResult.entities.length > 0 const { answers: rawAnswers } = nluResult + // TODO: handle dialog action skill speech vs text + // let answers = rawAnswers as [{ answer: SkillAnswerConfigSchema }] let answers = rawAnswers let answer: string | undefined = '' @@ -505,6 +520,8 @@ export default class Brain { actions[nluResult.classification.action]?.unknown_answers if (unknownAnswers) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore answer = unknownAnswers[ Math.floor(Math.random() * unknownAnswers.length) diff --git a/server/src/core/brain/types.ts b/server/src/core/brain/types.ts index 9374fed0..e473f8d9 100644 --- a/server/src/core/brain/types.ts +++ b/server/src/core/brain/types.ts @@ -9,7 +9,10 @@ import type { NLUSlot, NLUSlots } from '@/core/nlp/types' -import type { SkillConfigSchema } from '@/schemas/skill-schemas' +import type { + SkillConfigSchema, + SkillAnswerConfigSchema +} from '@/schemas/skill-schemas' import type { ShortLanguageCode } from '@/types' interface SkillCoreData { @@ -29,7 +32,7 @@ export interface SkillResult { slots: NLUSlots output: { codes: string[] - speech: string + answer: string core: SkillCoreData | undefined // eslint-disable-next-line @typescript-eslint/no-explicit-any options: Record @@ -62,6 +65,22 @@ export interface IntentObject extends ActionParams { action: NLPAction } +export interface SkillAnswerCoreData { + restart?: boolean + isInActionLoop?: boolean + showNextActionSuggestions?: boolean + showSuggestions?: boolean +} +export interface SkillAnswerOutput extends IntentObject { + output: { + codes: string + answer: SkillAnswerConfigSchema + core?: SkillAnswerCoreData + widget?: unknown // TODO + options: Record + } +} + export interface BrainProcessResult extends NLUResult { speeches: string[] executionTime: number diff --git a/server/src/schemas/skill-schemas.ts b/server/src/schemas/skill-schemas.ts index e594ade0..96bb00b3 100644 --- a/server/src/schemas/skill-schemas.ts +++ b/server/src/schemas/skill-schemas.ts @@ -275,4 +275,4 @@ export type SkillCustomRegexEntityTypeSchema = Static< export type SkillCustomEnumEntityTypeSchema = Static< typeof skillCustomEnumEntityType > -export type AnswerConfigSchema = Static +export type SkillAnswerConfigSchema = Static diff --git a/skills/leon/good_bye/config/en.json b/skills/leon/good_bye/config/en.json index e0beadb6..7adba2c1 100644 --- a/skills/leon/good_bye/config/en.json +++ b/skills/leon/good_bye/config/en.json @@ -12,14 +12,11 @@ "I have to go" ], "answers": [ - "Bye!", - "Bye bye!", - "Good bye.", - "Bye! Take care.", - "Good bye, please, take care of yourself.", - "Bye! Enjoy your time!", - "See you!", - "See ya!" + { + "text": "text test", + "speech": "speech test" + }, + "Bye!" ] } } From e2899c8d9a716376f74a5f1a97f6b4a94b7e594b Mon Sep 17 00:00:00 2001 From: louistiti Date: Thu, 18 May 2023 20:38:26 +0800 Subject: [PATCH 09/10] refactor(skill/good_bye): rollback as speech/text answer is only supported in logic skills --- core/skills-endpoints.json | 140 ++++++++++++++-------------- skills/leon/good_bye/config/en.json | 13 ++- 2 files changed, 78 insertions(+), 75 deletions(-) diff --git a/core/skills-endpoints.json b/core/skills-endpoints.json index ad55eb08..459c3b34 100644 --- a/core/skills-endpoints.json +++ b/core/skills-endpoints.json @@ -1,75 +1,5 @@ { "endpoints": [ - { - "method": "GET", - "route": "/api/action/leon/age/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/color/favorite_color", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/color/why", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/color/color_hexadecimal", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/good_bye/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/greeting/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/introduction/introduce_leon", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/introduction/gather_basic_info", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/introduction/remember", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/joke/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/meaning_of_life/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/partner_assistant/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/random_number/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/action/leon/welcome/run", - "params": [] - }, { "method": "POST", "route": "/api/action/news/github_trends/run", @@ -210,6 +140,76 @@ "method": "GET", "route": "/api/action/games/rochambeau/rematch", "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/age/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/color/favorite_color", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/color/why", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/color/color_hexadecimal", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/good_bye/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/greeting/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/introduction/introduce_leon", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/introduction/gather_basic_info", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/introduction/remember", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/joke/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/meaning_of_life/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/partner_assistant/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/random_number/run", + "params": [] + }, + { + "method": "GET", + "route": "/api/action/leon/welcome/run", + "params": [] } ] } diff --git a/skills/leon/good_bye/config/en.json b/skills/leon/good_bye/config/en.json index 7adba2c1..e0beadb6 100644 --- a/skills/leon/good_bye/config/en.json +++ b/skills/leon/good_bye/config/en.json @@ -12,11 +12,14 @@ "I have to go" ], "answers": [ - { - "text": "text test", - "speech": "speech test" - }, - "Bye!" + "Bye!", + "Bye bye!", + "Good bye.", + "Bye! Take care.", + "Good bye, please, take care of yourself.", + "Bye! Enjoy your time!", + "See you!", + "See ya!" ] } } From d295105fa692f4ff918e5d69c5d99d313dba7066 Mon Sep 17 00:00:00 2001 From: louistiti Date: Thu, 18 May 2023 21:06:04 +0800 Subject: [PATCH 10/10] chore: disable `import/order` ESLint rule for skills --- .eslintrc.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.eslintrc.json b/.eslintrc.json index 0fdc8022..09b31dd6 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -67,6 +67,12 @@ ] }, "overrides": [ + { + "files": ["skills/**/*.ts"], + "rules": { + "import/order": "off" + } + }, { "files": ["*.ts"], "rules": {