feat: widget fetching backbone (WIP)

app/src/custom-aurora-components/timer/timer.tsx

@@ -1,16 +1,12 @@
 import React, { useState, useEffect } from 'react'
-import { CircularProgress, Flexbox, Text, Loader } from '@leon-ai/aurora'
+import { CircularProgress, Flexbox, Text } from '@leon-ai/aurora'
 
 interface TimerProps {
   initialTime: number
   interval: number
   totalTimeContent: string
-  onFetch: () => void
   onEnd?: () => void
 }
-interface TimerFetchData {
-  initialTime: number
-}
 
 function formatTime(seconds: number): string {
   const minutes = seconds >= 60 ? Math.floor(seconds / 60) : 0
@@ -26,17 +22,12 @@ export function Timer({
   initialTime,
   interval,
   totalTimeContent,
-  onFetch,
   onEnd
 }: TimerProps) {
   const [progress, setProgress] = useState(0)
   const [timeLeft, setTimeLeft] = useState(initialTime)
-  const [isFetching, setIsFetching] = useState(false)
 
   useEffect(() => {
-    if (onFetch) {
-      onFetch()
-    }
     setTimeLeft(initialTime)
     setProgress(0)
   }, [initialTime])
@@ -65,18 +56,12 @@ export function Timer({
   return (
     <CircularProgress value={progress} size="lg">
       <Flexbox gap="xs" alignItems="center" justifyContent="center">
-        {!isFetching ? (
-          <>
-            <Text fontSize="lg" fontWeight="semi-bold">
-              {formatTime(timeLeft)}
-            </Text>
-            <Text fontSize="xs" secondary>
-              {totalTimeContent}
-            </Text>
-          </>
-        ) : (
-          <Loader />
-        )}
+        <Text fontSize="lg" fontWeight="semi-bold">
+          {formatTime(timeLeft)}
+        </Text>
+        <Text fontSize="xs" secondary>
+          {totalTimeContent}
+        </Text>
       </Flexbox>
     </CircularProgress>
   )

app/src/js/chatbot.js

@@ -1,10 +1,12 @@
 import { createRoot } from 'react-dom/client'
+import axios from 'axios'
 
 import renderAuroraComponent from './render-aurora-component'
 
 export default class Chatbot {
-  constructor(socket) {
+  constructor(socket, serverURL) {
     this.socket = socket
+    this.serverURL = serverURL
     this.et = new EventTarget()
     this.feed = document.querySelector('#feed')
     this.typing = document.querySelector('#is-typing')
@@ -104,7 +106,7 @@ export default class Chatbot {
 
     this.feed.appendChild(container).appendChild(bubble)
 
-    let widgetTree = null
+    let componentTree = null
     let widgetSupportedEvents = null
 
     /**
@@ -114,16 +116,34 @@ export default class Chatbot {
       const parsedWidget = JSON.parse(string)
       const root = createRoot(container)
 
-      widgetTree = parsedWidget.componentTree
+      componentTree = parsedWidget.componentTree
       widgetSupportedEvents = parsedWidget.supportedEvents
 
-      const reactNode = renderAuroraComponent(
-        this.socket,
-        widgetTree,
-        widgetSupportedEvents
-      )
+      if (parsedWidget.onFetch) {
+        // TODO: widget fetching
+        // TODO: inject Loader component in the componentTree to show loading state + refactor
+        axios.get(`${this.serverURL}/api/v1/fetch`).then((data) => {
+          componentTree = data.data.componentTree
 
-      root.render(reactNode)
+          console.log('componentTree', componentTree)
+
+          const reactNode = renderAuroraComponent(
+            this.socket,
+            componentTree,
+            widgetSupportedEvents
+          )
+
+          root.render(reactNode)
+        })
+      } else {
+        const reactNode = renderAuroraComponent(
+          this.socket,
+          componentTree,
+          widgetSupportedEvents
+        )
+
+        root.render(reactNode)
+      }
     }
 
     if (save) {

app/src/js/client.js

@@ -14,7 +14,7 @@ export default class Client {
     this.socket = io(this.serverUrl)
     this.history = localStorage.getItem('history')
     this.parsedHistory = []
-    this.chatbot = new Chatbot(this.socket)
+    this.chatbot = new Chatbot(this.socket, this.serverUrl)
     this.voiceEnergy = new VoiceEnergy(this)
     this._recorder = {}
     this._suggestions = []

app/src/js/render-aurora-component.js

@@ -48,16 +48,6 @@ export default function renderAuroraComponent(
       })
     }
 
-    // TODO: now!
-    // TODO: if onFetch, then set new values here, send generic fetch request to get skill -> widget id?
-    // TODO: need to create a standard on_fetch skill action that will be executed?
-    /*if (component.props.onFetch) {
-      console.log('component', component)
-      if (component.props.initialTime) {
-        component.props.initialTime = 0
-      }
-    }*/
-
     return createElement(reactComponent, component.props)
   }
 }

bridges/nodejs/src/sdk/leon.ts

@@ -126,6 +126,10 @@ class Leon {
           }),
           supportedEvents: SUPPORTED_WIDGET_EVENTS
         }
+
+        if (answerInput.widget.onFetch) {
+          answerObject.output.widget.onFetch = answerInput.widget.onFetch
+        }
       }
 
       // "Temporize" for the data buffer output on the core

bridges/nodejs/src/sdk/widget-component.ts

@@ -9,8 +9,7 @@ export const SUPPORTED_WIDGET_EVENTS = [
   'onSubmit',
   'onChange',
   'onStart',
-  'onEnd',
-  'onFetch'
+  'onEnd'
 ] as const
 
 function generateId(): string {

bridges/nodejs/src/sdk/widget.ts

@@ -24,7 +24,7 @@ interface SendUtteranceOptions {
 }
 
 export interface WidgetEventMethod {
-  methodName: 'send_utterance' | 'run_skill_action' | 'fetch_widget_data'
+  methodName: 'send_utterance' | 'run_skill_action'
   methodParams:
     | SendUtteranceWidgetEventMethodParams
     | RunSkillActionWidgetEventMethodParams
@@ -32,6 +32,8 @@ export interface WidgetEventMethod {
 }
 export interface WidgetOptions<T = unknown> {
   wrapperProps?: Omit<WidgetWrapperProps, 'children'>
+  // TODO: widget fetching
+  onFetch?: string
   params: T
 }
 
@@ -39,6 +41,8 @@ export abstract class Widget<T = unknown> {
   public actionName: string
   public id: string
   public widget: string
+  // TODO: widget fetching
+  public onFetch: string | null = null
   public wrapperProps: WidgetOptions<T>['wrapperProps']
   public params: WidgetOptions<T>['params']
 
@@ -46,6 +50,9 @@ export abstract class Widget<T = unknown> {
     if (options?.wrapperProps) {
       this.wrapperProps = options.wrapperProps
     }
+    if (options?.onFetch) {
+      this.onFetch = options.onFetch
+    }
     this.actionName = `${INTENT_OBJECT.domain}:${INTENT_OBJECT.skill}:${INTENT_OBJECT.action}`
     this.widget = this.constructor.name
     this.id = `${this.widget.toLowerCase()}-${Math.random()
@@ -100,22 +107,6 @@ export abstract class Widget<T = unknown> {
     }
   }
 
-  /**
-   * Indicate the core to fetch/set the data of a given widget
-   * @param dataToSet Data to set on fetch
-   * @example fetchWidgetData('timer-f42wa', { initialTime: 42 })
-   */
-  protected fetchWidgetData(dataToSet: string[]): WidgetEventMethod {
-    return {
-      methodName: 'fetch_widget_data',
-      methodParams: {
-        actionName: this.actionName,
-        widgetId: this.id,
-        dataToSet
-      }
-    }
-  }
-
   /**
    * Grab and compute the target content of the widget
    * @param key The key of the content

core/skills-endpoints.json

@@ -1,5 +1,10 @@
 {
   "endpoints": [
+    {
+      "method": "GET",
+      "route": "/api/action/unknown/widget-playground/run",
+      "params": []
+    },
     {
       "method": "POST",
       "route": "/api/action/games/akinator/choose_thematic",
@@ -63,53 +68,6 @@
       "route": "/api/action/news/product_hunt_trends/run",
       "params": []
     },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/create_list",
-      "params": ["list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "GET",
-      "route": "/api/action/productivity/todo_list/view_lists",
-      "params": []
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/view_list",
-      "params": ["list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/rename_list",
-      "params": ["old_list", "new_list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/delete_list",
-      "params": ["list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/add_todos",
-      "params": ["todos", "list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/complete_todos",
-      "params": ["todos", "list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/uncheck_todos",
-      "params": ["todos", "list"],
-      "entitiesType": "trim"
-    },
     {
       "method": "GET",
       "route": "/api/action/leon/age/run",
@@ -185,11 +143,53 @@
       "route": "/api/action/leon/thanks/run",
       "params": []
     },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/create_list",
+      "params": ["list"],
+      "entitiesType": "trim"
+    },
     {
       "method": "GET",
-      "route": "/api/action/unknown/widget-playground/run",
+      "route": "/api/action/productivity/todo_list/view_lists",
       "params": []
     },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/view_list",
+      "params": ["list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/rename_list",
+      "params": ["old_list", "new_list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/delete_list",
+      "params": ["list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/add_todos",
+      "params": ["todos", "list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/complete_todos",
+      "params": ["todos", "list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/uncheck_todos",
+      "params": ["todos", "list"],
+      "entitiesType": "trim"
+    },
     {
       "method": "GET",
       "route": "/api/action/utilities/date_time/current_date_time",
@@ -236,11 +236,6 @@
       "route": "/api/action/utilities/speed_test/run",
       "params": []
     },
-    {
-      "method": "GET",
-      "route": "/api/action/utilities/timer/configure_set_timer",
-      "params": []
-    },
     {
       "method": "GET",
       "route": "/api/action/utilities/timer/set_timer",

server/src/core/brain/brain.ts

@@ -64,7 +64,7 @@ export default class Brain {
   private skillProcess: ChildProcessWithoutNullStreams | undefined = undefined
   private domainFriendlyName = ''
   private skillFriendlyName = ''
-  private skillOutput = ''
+  public skillOutput = ''
   public isMuted = false // Close Leon mouth if true; e.g. over HTTP
 
   constructor() {

server/src/core/brain/types.ts

@@ -98,6 +98,8 @@ export interface SkillAnswerOutput extends IntentObject {
       id: string
       componentTree: WidgetWrapper
       supportedEvents: typeof SUPPORTED_WIDGET_EVENTS
+      // TODO: widget fetching
+      onFetch?: string
     }
   }
 }

server/src/core/http-server/http-server.ts

@@ -18,8 +18,10 @@ import { infoPlugin } from '@/core/http-server/api/info'
 import { llmInferencePlugin } from '@/core/http-server/api/llm-inference'
 import { keyMidd } from '@/core/http-server/plugins/key'
 import { utterancePlugin } from '@/core/http-server/api/utterance'
-import { LLM_MANAGER, PERSONA } from '@/core'
+import { BRAIN, LLM_MANAGER, PERSONA } from '@/core'
+import { DEFAULT_NLU_RESULT } from '@/core/nlp/nlu/nlu'
 import { SystemHelper } from '@/helpers/system-helper'
+import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
 
 const API_VERSION = 'v1'
 
@@ -101,6 +103,41 @@ export default class HTTPServer {
       reply.sendFile('index.html')
     })
 
+    this.fastify.get(`/api/${API_VERSION}/fetch`, async (_request, reply) => {
+      try {
+        BRAIN.isMuted = true
+        await BRAIN.execute({
+          ...DEFAULT_NLU_RESULT,
+          // TODO: widget fetching
+          skillConfigPath: SkillDomainHelper.getSkillConfigPath(
+            'utilities',
+            'timer'
+          ),
+          // TODO: widget fetching
+          classification: {
+            domain: 'utilities',
+            skill: 'timer',
+            action: 'check_timer',
+            confidence: 1
+          }
+        })
+
+        const parsedOutput = JSON.parse(BRAIN.skillOutput)
+
+        if (parsedOutput.output.widget) {
+          return reply.send({
+            componentTree: JSON.parse(BRAIN.skillOutput).output.widget
+              .componentTree
+          })
+        }
+
+        return reply.send({ componentTree: null })
+      } catch (e) {
+        LogHelper.title('HTTP Server')
+        LogHelper.error(`Failed to fetch widget component tree: ${e}`)
+      }
+    })
+
     this.fastify.register(infoPlugin, { apiVersion: API_VERSION })
 
     this.fastify.register(llmInferencePlugin, { apiVersion: API_VERSION })

server/src/core/nlp/nlu/action-loop.ts

@@ -22,13 +22,9 @@ export class ActionLoop {
   ): Promise<Partial<BrainProcessResult> | null> {
     const { domain, intent } = NLU.conversation.activeContext
     const [skillName, actionName] = intent.split('.') as [string, string]
-    const skillConfigPath = join(
-      process.cwd(),
-      'skills',
+    const skillConfigPath = SkillDomainHelper.getSkillConfigPath(
       domain,
-      skillName,
-      'config',
-      BRAIN.lang + '.json'
+      skillName
     )
     const newNLUResult = {
       ...DEFAULT_NLU_RESULT, // Reset entities, slots, etc.

server/src/core/nlp/nlu/nlu.ts

@@ -1,4 +1,3 @@
-import { join } from 'node:path'
 import { spawn } from 'node:child_process'
 
 import kill from 'tree-kill'
@@ -81,13 +80,9 @@ export default class NLU {
 
     const skillConfigPath = newNLUResult.skillConfigPath
       ? newNLUResult.skillConfigPath
-      : join(
-          process.cwd(),
-          'skills',
+      : SkillDomainHelper.getSkillConfigPath(
           newNLUResult.classification.domain,
-          newNLUResult.classification.skill,
-          'config',
-          BRAIN.lang + '.json'
+          newNLUResult.classification.skill
         )
     const { actions } = await SkillDomainHelper.getSkillConfig(
       skillConfigPath,
@@ -433,13 +428,9 @@ export default class NLU {
           )}`
         )
 
-        const skillConfigPath = join(
-          process.cwd(),
-          'skills',
+        const skillConfigPath = SkillDomainHelper.getSkillConfigPath(
           this._nluResult.classification.domain,
-          this._nluResult.classification.skill,
-          'config',
-          BRAIN.lang + '.json'
+          this._nluResult.classification.skill
         )
         this._nluResult.skillConfigPath = skillConfigPath
 

server/src/core/nlp/nlu/slot-filling.ts

@@ -1,5 +1,3 @@
-import { join } from 'node:path'
-
 import type { NLPUtterance } from '@/core/nlp/types'
 import type { BrainProcessResult } from '@/core/brain/types'
 import { BRAIN, MODEL_LOADER, NER, NLU, SOCKET_SERVER } from '@/core'
@@ -61,13 +59,9 @@ export class SlotFilling {
 
     const { domain, intent } = NLU.conversation.activeContext
     const [skillName, actionName] = intent.split('.') as [string, string]
-    const skillConfigPath = join(
-      process.cwd(),
-      'skills',
+    const skillConfigPath = SkillDomainHelper.getSkillConfigPath(
       domain,
-      skillName,
-      'config',
-      BRAIN.lang + '.json'
+      skillName
     )
 
     await NLU.setNLUResult({

server/src/core/socket-server.ts

@@ -196,12 +196,6 @@ export default class SocketServer {
               }
             } else if (method.methodName === 'run_skill_action') {
               this.socket?.emit('widget-run-skill-action', method.methodParams)
-            } else if (method.methodName === 'fetch_widget_data') {
-              console.log('method.methodParams', method.methodParams)
-              // TODO: get memory from domain:skill:action
-              // TODO: grab new data from the widget. E.g. initialTime
-              // TODO: get memory timestamp of the timer creation + initialTime
-              this.socket?.emit('widget-fetch-data', method.methodParams)
             }
           })
         }

server/src/helpers/skill-domain-helper.ts

@@ -10,6 +10,7 @@ import type {
   SkillBridgeSchema
 } from '@/schemas/skill-schemas'
 import { SKILLS_PATH } from '@/constants'
+import { BRAIN } from '@/core'
 
 interface SkillDomain {
   domainId: string
@@ -140,6 +141,18 @@ export class SkillDomainHelper {
     return path.join(SKILLS_PATH, domain, skill)
   }
 
+  /**
+   * Get skill config path
+   * @param domain Domain where the skill belongs
+   * @param skill Skill to get config path from
+   */
+  public static getSkillConfigPath(
+    domain: SkillDomain['name'],
+    skill: SkillSchema['name']
+  ): string {
+    return path.join(SKILLS_PATH, domain, skill, 'config', `${BRAIN.lang}.json`)
+  }
+
   /**
    * Get skill config
    * @param configFilePath Path of the skill config file

skills/utilities/timer/config/en.json

@@ -6,25 +6,11 @@
     "seconds": "seconds"
   },
   "actions": {
-    "configure_set_timer": {
-      "type": "dialog",
+    "set_timer": {
+      "type": "logic",
       "utterance_samples": [
         "[Set|Start|Create] a timer for @number [minutes|seconds]"
-      ],
-      "slots": [
-        {
-          "name": "duration",
-          "item": {
-            "type": "entity",
-            "name": "duration"
-          },
-          "questions": ["For how long time?"]
-        }
-      ],
-      "next_action": "set_timer"
-    },
-    "set_timer": {
-      "type": "logic"
+      ]
     },
     "cancel_timer": {
       "type": "logic",
@@ -52,8 +38,8 @@
   },
   "answers": {
     "cannot_get_duration": [
-      "Sorry, I can't get the duration of the timer.",
-      "I can't get the duration of the timer. Sorry."
+      "You should provide a duration for the timer.",
+      "You didn't provide a duration for the timer."
     ],
     "unit_not_supported": [
       "Sorry, I can't set a timer for this unit. Use %hours%, %minutes% or %seconds% instead.",

skills/utilities/timer/src/actions/check_timer.ts

@@ -14,14 +14,10 @@ export const run: ActionFunction = async function () {
   const { widgetId, interval, finishedAt, duration } = timerMemory
   const remainingTime = finishedAt - Math.floor(Date.now() / 1_000)
 
-  if (remainingTime <= 0) {
-    return await leon.answer({ key: 'no_timer_set' })
-  }
-
   const timerWidget = new TimerWidget({
     params: {
       id: widgetId,
-      seconds: remainingTime,
+      seconds: remainingTime <= 0 ? 0 : remainingTime,
       initialDuration: duration,
       interval
     }

skills/utilities/timer/src/actions/set_timer.ts

@@ -6,10 +6,10 @@ import { createTimerMemory } from '../lib/memory'
 
 export const run: ActionFunction = async function (params) {
   const supportedUnits = ['hours', 'minutes', 'seconds']
-  const durations = (
-    params.slots['duration']?.resolution as BuiltInDurationEntity['resolution']
+  const [duration] = (
+    params.current_entities.find((entity) => entity.type === 'duration')
+      ?.resolution as BuiltInDurationEntity['resolution']
   ).values
-  const [duration] = durations
 
   if (!duration) {
     return leon.answer({ key: 'cannot_get_duration' })
@@ -27,7 +27,10 @@ export const run: ActionFunction = async function (params) {
     params: {
       seconds,
       interval
-    }
+    },
+    // TODO: widget fetching
+    // TODO: better method
+    onFetch: 'check_timer'
   })
 
   await createTimerMemory(timerWidget.id, seconds, interval)

skills/utilities/timer/src/widgets/components/timer.ts

@@ -5,7 +5,6 @@ interface TimerProps {
   initialTime: number
   interval: number
   totalTimeContent: string
-  onFetch: () => WidgetEventMethod
   onEnd?: () => WidgetEventMethod
 }
 

skills/utilities/timer/src/widgets/timer-widget.ts

@@ -58,9 +58,6 @@ export class TimerWidget extends Widget<Params> {
       initialTime: seconds,
       interval,
       totalTimeContent,
-      onFetch: (): WidgetEventMethod => {
-        return this.fetchWidgetData(['initialTime'])
-      },
       onEnd: (): WidgetEventMethod => {
         return this.sendUtterance('times_up', {
           from: 'leon'