chore: merge branch 'skill-text-n-speech-difference' into develop

.eslintrc.json

@@ -67,6 +67,12 @@
     ]
   },
   "overrides": [
+    {
+      "files": ["skills/**/*.ts"],
+      "rules": {
+        "import/order": "off"
+      }
+    },
     {
       "files": ["*.ts"],
       "rules": {

bridges/nodejs/src/sdk/leon.ts

@@ -1,4 +1,9 @@
-import type { AnswerData, AnswerInput, AnswerOutput } from '@sdk/types'
+import type {
+  AnswerData,
+  AnswerInput,
+  AnswerOutput,
+  AnswerConfig
+} from '@sdk/types'
 import {
   INTENT_OBJECT,
   SKILL_CONFIG,
@@ -45,7 +50,7 @@ class Leon {
   public setAnswerData(
     answerKey: string | undefined,
     data: AnswerData = null
-  ): string | null | undefined {
+  ): AnswerConfig | null | undefined {
     if (answerKey) {
       try {
         // In case the answer key is a raw answer
@@ -54,7 +59,7 @@ class Leon {
         }
 
         const answers = SKILL_CONFIG.answers[answerKey] ?? ''
-        let answer: string
+        let answer: AnswerConfig
 
         if (Array.isArray(answers)) {
           answer = answers[Math.floor(Math.random() * answers.length)] ?? ''
@@ -64,7 +69,22 @@ class Leon {
 
         if (data) {
           for (const key in data) {
-            answer = answer.replaceAll(`%${key}%`, String(data[key]))
+            // In case the answer needs speech and text differentiation
+            if (typeof answer !== 'string' && answer.text) {
+              answer.text = answer.text.replaceAll(
+                `%${key}%`,
+                String(data[key])
+              )
+              answer.speech = answer.speech.replaceAll(
+                `%${key}%`,
+                String(data[key])
+              )
+            } else {
+              answer = (answer as string).replaceAll(
+                `%${key}%`,
+                String(data[key])
+              )
+            }
           }
         }
 
@@ -72,7 +92,22 @@ class Leon {
           const { variables } = SKILL_CONFIG
 
           for (const key in variables) {
-            answer = answer.replaceAll(`%${key}%`, String(variables[key]))
+            // In case the answer needs speech and text differentiation
+            if (typeof answer !== 'string' && answer.text) {
+              answer.text = answer.text.replaceAll(
+                `%${key}%`,
+                String(variables[key])
+              )
+              answer.speech = answer.speech.replaceAll(
+                `%${key}%`,
+                String(variables[key])
+              )
+            } else {
+              answer = (answer as string).replaceAll(
+                `%${key}%`,
+                String(variables[key])
+              )
+            }
           }
         }
 
@@ -103,7 +138,7 @@ class Leon {
             answerInput.widget && !answerInput.key
               ? 'widget'
               : (answerInput.key as string),
-          speech: this.setAnswerData(answerInput.key, answerInput.data) ?? '',
+          answer: this.setAnswerData(answerInput.key, answerInput.data) ?? '',
           core: answerInput.core,
           options: this.getSRCConfig('options')
         }

bridges/nodejs/src/sdk/types.ts

@@ -1,7 +1,13 @@
 /**
  * Action types
  */
-import type { ActionParams, IntentObject } from '@/core/brain/types'
+import type {
+  ActionParams,
+  IntentObject,
+  SkillAnswerCoreData,
+  SkillAnswerOutput
+} from '@/core/brain/types'
+import type { SkillAnswerConfigSchema } from '@/schemas/skill-schemas'
 
 export type { ActionParams, IntentObject }
 
@@ -10,26 +16,11 @@ export type ActionFunction = (params: ActionParams) => Promise<void>
 /**
  * Answer types
  */
-export interface AnswerOutput extends IntentObject {
-  output: {
-    codes: string
-    speech: string
-    core?: AnswerCoreData
-    widget?: unknown // TODO
-    options: Record<string, string>
-  }
-}
-export interface AnswerCoreData {
-  restart?: boolean
-  isInActionLoop?: boolean
-  showNextActionSuggestions?: boolean
-  showSuggestions?: boolean
-}
 export interface Answer {
   key?: string
   widget?: unknown // TODO
   data?: AnswerData
-  core?: AnswerCoreData
+  core?: SkillAnswerCoreData
 }
 export interface TextAnswer extends Answer {
   key: string
@@ -40,3 +31,5 @@ export interface WidgetAnswer extends Answer {
 }
 export type AnswerData = Record<string, string | number> | null
 export type AnswerInput = TextAnswer | WidgetAnswer
+export type AnswerOutput = SkillAnswerOutput
+export type AnswerConfig = SkillAnswerConfigSchema

server/src/core/brain/brain.ts

@@ -9,7 +9,11 @@ import type {
   NERCustomEntity,
   NLUResult
 } from '@/core/nlp/types'
-import type { SkillConfigSchema, SkillSchema } from '@/schemas/skill-schemas'
+import type {
+  SkillAnswerConfigSchema,
+  SkillConfigSchema,
+  SkillSchema
+} from '@/schemas/skill-schemas'
 import type {
   BrainProcessResult,
   IntentObject,
@@ -29,6 +33,7 @@ import { LogHelper } from '@/helpers/log-helper'
 import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
 import { StringHelper } from '@/helpers/string-helper'
 import Synchronizer from '@/core/synchronizer'
+import type { AnswerOutput } from '@sdk/types'
 
 export default class Brain {
   private static instance: Brain
@@ -43,7 +48,7 @@ export default class Brain {
   private domainFriendlyName = ''
   private skillFriendlyName = ''
   private skillOutput = ''
-  private speeches: string[] = []
+  private answers: SkillAnswerConfigSchema[] = []
   public isMuted = false // Close Leon mouth if true; e.g. over HTTP
 
   constructor() {
@@ -97,19 +102,22 @@ export default class Brain {
   /**
    * Make Leon talk
    */
-  public talk(rawSpeech: string, end = false): void {
+  public talk(answer: SkillAnswerConfigSchema, end = false): void {
     LogHelper.title('Brain')
     LogHelper.info('Talking...')
 
-    if (rawSpeech !== '') {
+    if (answer !== '') {
+      const textAnswer = typeof answer === 'string' ? answer : answer.text
+      const speechAnswer = typeof answer === 'string' ? answer : answer.speech
+
       if (HAS_TTS) {
         // Stripe HTML to a whitespace. Whitespace to let the TTS respects punctuation
-        const speech = rawSpeech.replace(/<(?:.|\n)*?>/gm, ' ')
+        const speech = speechAnswer.replace(/<(?:.|\n)*?>/gm, ' ')
 
         TTS.add(speech, end)
       }
 
-      SOCKET_SERVER.socket?.emit('answer', rawSpeech)
+      SOCKET_SERVER.socket?.emit('answer', textAnswer)
     }
   }
 
@@ -192,7 +200,7 @@ export default class Brain {
     data: Buffer
   ): Promise<Error | null> | void {
     try {
-      const obj = JSON.parse(data.toString())
+      const obj = JSON.parse(data.toString()) as AnswerOutput
 
       if (typeof obj === 'object') {
         LogHelper.title(`${this.skillFriendlyName} skill (on data)`)
@@ -202,11 +210,14 @@ export default class Brain {
           SOCKET_SERVER.socket?.emit('widget', obj.output.widget)
         }
 
-        const speech = obj.output.speech.toString()
+        // TODO: remove this condition when Python skills outputs are updated (replace "speech" with "answer")
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore
+        const { answer, speech } = obj.output
         if (!this.isMuted) {
-          this.talk(speech)
+          this.talk(answer || speech)
         }
-        this.speeches.push(speech)
+        this.answers.push(answer)
         this.skillOutput = data.toString()
 
         return Promise.resolve(null)
@@ -231,11 +242,13 @@ export default class Brain {
       '%skill_name%': this.skillFriendlyName,
       '%domain_name%': this.domainFriendlyName
     })}!`
+
     if (!this.isMuted) {
       this.talk(speech)
       SOCKET_SERVER.socket?.emit('is-typing', false)
     }
-    this.speeches.push(speech)
+
+    this.answers.push(speech)
   }
 
   /**
@@ -480,6 +493,8 @@ export default class Brain {
             await SkillDomainHelper.getSkillConfig(configFilePath, this._lang)
           const utteranceHasEntities = nluResult.entities.length > 0
           const { answers: rawAnswers } = nluResult
+          // TODO: handle dialog action skill speech vs text
+          // let answers = rawAnswers as [{ answer: SkillAnswerConfigSchema }]
           let answers = rawAnswers
           let answer: string | undefined = ''
 
@@ -505,6 +520,8 @@ export default class Brain {
                 actions[nluResult.classification.action]?.unknown_answers
 
               if (unknownAnswers) {
+                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                // @ts-ignore
                 answer =
                   unknownAnswers[
                     Math.floor(Math.random() * unknownAnswers.length)

server/src/core/brain/types.ts

@@ -9,7 +9,10 @@ import type {
   NLUSlot,
   NLUSlots
 } from '@/core/nlp/types'
-import type { SkillConfigSchema } from '@/schemas/skill-schemas'
+import type {
+  SkillConfigSchema,
+  SkillAnswerConfigSchema
+} from '@/schemas/skill-schemas'
 import type { ShortLanguageCode } from '@/types'
 
 interface SkillCoreData {
@@ -29,7 +32,7 @@ export interface SkillResult {
   slots: NLUSlots
   output: {
     codes: string[]
-    speech: string
+    answer: string
     core: SkillCoreData | undefined
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     options: Record<string, any>
@@ -62,6 +65,22 @@ export interface IntentObject extends ActionParams {
   action: NLPAction
 }
 
+export interface SkillAnswerCoreData {
+  restart?: boolean
+  isInActionLoop?: boolean
+  showNextActionSuggestions?: boolean
+  showSuggestions?: boolean
+}
+export interface SkillAnswerOutput extends IntentObject {
+  output: {
+    codes: string
+    answer: SkillAnswerConfigSchema
+    core?: SkillAnswerCoreData
+    widget?: unknown // TODO
+    options: Record<string, string>
+  }
+}
+
 export interface BrainProcessResult extends NLUResult {
   speeches: string[]
   executionTime: number

server/src/schemas/skill-schemas.ts

@@ -23,6 +23,13 @@ const skillDataTypes = [
   Type.Literal('global_resolver'),
   Type.Literal('entity')
 ]
+const answerTypes = Type.Union([
+  Type.String(),
+  Type.Object({
+    speech: Type.String(),
+    text: Type.String()
+  })
+])
 const skillCustomEnumEntityType = Type.Object(
   {
     type: Type.Literal('enum'),
@@ -184,8 +191,8 @@ export const skillConfigSchemaObject = Type.Strict(
             )
           ),
           utterance_samples: Type.Optional(Type.Array(Type.String())),
-          answers: Type.Optional(Type.Array(Type.String())),
-          unknown_answers: Type.Optional(Type.Array(Type.String())),
+          answers: Type.Optional(Type.Array(answerTypes)),
+          unknown_answers: Type.Optional(Type.Array(answerTypes)),
           suggestions: Type.Optional(
             Type.Array(Type.String(), {
               description:
@@ -235,9 +242,7 @@ export const skillConfigSchemaObject = Type.Strict(
         { additionalProperties: false }
       )
     ),
-    answers: Type.Optional(
-      Type.Record(Type.String(), Type.Array(Type.String()))
-    ),
+    answers: Type.Optional(Type.Record(Type.String(), Type.Array(answerTypes))),
     entities: Type.Optional(Type.Record(Type.String(), Type.String())),
     resolvers: Type.Optional(
       Type.Record(
@@ -270,3 +275,4 @@ export type SkillCustomRegexEntityTypeSchema = Static<
 export type SkillCustomEnumEntityTypeSchema = Static<
   typeof skillCustomEnumEntityType
 >
+export type SkillAnswerConfigSchema = Static<typeof answerTypes>

skills/leon/age/config/en.json

@@ -9,6 +9,12 @@
   "answers": {
     "default": ["I'm..."],
     "greet": ["Hey, just a try %name% again %name%", "Another try, hi"],
-    "answer": ["%answer%"]
+    "answer": ["%answer%"],
+    "test": [
+      {
+        "speech": "This will be said out loud",
+        "text": "This will be shown in the chat"
+      }
+    ]
   }
 }

skills/leon/age/src/actions/run.ts

@@ -17,6 +17,10 @@ interface Post {
 }
 
 export const run: ActionFunction = async function () {
+  await leon.answer({ key: 'test' })
+
+  ///
+
   const button = new Button({
     text: 'Hello world from action skill'
   })