Merge branch 'feat/widgets' into develop

app/src/custom-aurora-components/index.ts

@@ -0,0 +1 @@
+export * from './timer'

app/src/custom-aurora-components/timer/index.ts

@@ -0,0 +1 @@
+export * from './timer'

app/src/custom-aurora-components/timer/timer.tsx

@@ -0,0 +1,68 @@
+import React, { useState, useEffect } from 'react'
+import { CircularProgress, Flexbox, Text } from '@leon-ai/aurora'
+
+interface TimerProps {
+  initialTime: number
+  interval: number
+  totalTimeContent: string
+  onEnd?: () => void
+}
+
+function formatTime(seconds: number): string {
+  const minutes = seconds >= 60 ? Math.floor(seconds / 60) : 0
+  const remainingSeconds = seconds % 60
+  const formattedMinutes = minutes < 10 ? `0${minutes}` : minutes
+  const formattedSeconds =
+    remainingSeconds < 10 ? `0${remainingSeconds}` : remainingSeconds
+
+  return `${formattedMinutes}:${formattedSeconds}`
+}
+
+export function Timer({
+  initialTime,
+  interval,
+  totalTimeContent,
+  onEnd
+}: TimerProps) {
+  const [progress, setProgress] = useState(0)
+  const [timeLeft, setTimeLeft] = useState(initialTime)
+
+  useEffect(() => {
+    setTimeLeft(initialTime)
+    setProgress(0)
+  }, [initialTime])
+
+  useEffect(() => {
+    if (timeLeft <= 0) {
+      return
+    }
+
+    const timer = setInterval(() => {
+      setTimeLeft((prevTime) => {
+        const newTime = prevTime - 1
+
+        if (newTime <= 0 && onEnd) {
+          onEnd()
+        }
+
+        return newTime
+      })
+      setProgress((prevProgress) => prevProgress + 100 / initialTime)
+    }, interval)
+
+    return () => clearInterval(timer)
+  }, [initialTime, interval, timeLeft])
+
+  return (
+    <CircularProgress value={progress} size="lg">
+      <Flexbox gap="xs" alignItems="center" justifyContent="center">
+        <Text fontSize="lg" fontWeight="semi-bold">
+          {formatTime(timeLeft)}
+        </Text>
+        <Text fontSize="xs" secondary>
+          {totalTimeContent}
+        </Text>
+      </Flexbox>
+    </CircularProgress>
+  )
+}

app/src/js/render-aurora-component.js

@@ -1,6 +1,8 @@
 import { createElement } from 'react'
 import * as auroraComponents from '@leon-ai/aurora'
 
+import * as customAuroraComponents from '../custom-aurora-components'
+
 export default function renderAuroraComponent(
   socket,
   component,
@@ -8,7 +10,18 @@ export default function renderAuroraComponent(
 ) {
   if (component) {
     // eslint-disable-next-line import/namespace
-    const reactComponent = auroraComponents[component.component]
+    let reactComponent = auroraComponents[component.component]
+    /**
+     * Find custom component if a former component is not found
+     */
+    if (!reactComponent) {
+      // eslint-disable-next-line import/namespace
+      reactComponent = customAuroraComponents[component.component]
+    }
+
+    if (!reactComponent) {
+      console.error(`Component ${component} not found`)
+    }
 
     // Check if the browsed component has a supported event and bind it
     if (

bridges/nodejs/src/sdk/leon.ts

@@ -116,7 +116,6 @@ class Leon {
       }
 
       if (answerInput.widget) {
-        console.log('render', answerInput.widget.render())
         answerObject.output.widget = {
           tree: new WidgetWrapper({
             ...answerInput.widget.wrapperProps,
@@ -124,35 +123,6 @@ class Leon {
           }),
           supportedEvents: SUPPORTED_WIDGET_EVENTS
         }
-        // dynamically import the TSX component
-        /*const { default: tsxComponent } = await import(
-          // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-          // @ts-expect-error
-          '@@/skills/unknown/widget-playground/src/widgets/my-component.tsx'
-        )
-        console.log('tsxComponent', tsxComponent)
-        const componentString = ReactDOMServer.renderToString(
-          React.createElement(tsxComponent)
-        )
-        const componentEventHandlers = {
-          onClick: () => {}
-        }
-        // const componentString = ReactDOMServer.renderToString(tsxComponent)
-
-        // Collect event handlers from the component
-        React.Children.forEach(React.createElement(tsxComponent), (child) => {
-          if (child.props && child.props.onClick) {
-            componentEventHandlers.onClick = child.props.onClick.toString()
-          }
-        })
-        const response = {
-          componentString,
-          componentEventHandlers
-        }
-
-        console.log('componentString', componentString)
-
-        answerObject.output.widgetWithHandlers = response*/
       }
 
       // "Temporize" for the data buffer output on the core

bridges/nodejs/src/sdk/widget-component.ts

@@ -7,7 +7,10 @@ interface WidgetEvent {
 export const SUPPORTED_WIDGET_EVENTS = [
   'onClick',
   'onSubmit',
-  'onChange'
+  'onChange',
+  'onStart',
+  'onEnd',
+  'onFetch'
 ] as const
 
 function generateId(): string {

bridges/nodejs/src/sdk/widget.ts

@@ -3,13 +3,20 @@ import { type WidgetWrapperProps } from '@leon-ai/aurora'
 import { SKILL_CONFIG } from '@bridge/constants'
 import { WidgetComponent } from '@sdk/widget-component'
 
+type UtteranceSender = 'leon' | 'owner'
+
 interface SendUtteranceWidgetEventMethodParams {
+  from: UtteranceSender
   utterance: string
 }
 interface RunSkillActionWidgetEventMethodParams {
   actionName: string
   params: Record<string, unknown>
 }
+interface SendUtteranceOptions {
+  from?: UtteranceSender
+  data?: Record<string, unknown>
+}
 
 export interface WidgetEventMethod {
   methodName: 'send_utterance' | 'run_skill_action'
@@ -41,18 +48,20 @@ export abstract class Widget<T = unknown> {
   /**
    * Indicate the core to send a given utterance
    * @param key The key of the content
-   * @param data The data to apply
+   * @param options The options of the utterance
-   * @example content('provider_selected', { provider: 'Spotify' }) // 'I chose the Spotify provider'
+   * @example content('provider_selected', { data: { provider: 'Spotify' } }) // 'I chose the Spotify provider'
    */
   protected sendUtterance(
     key: string,
-    data?: Record<string, unknown>
+    options?: SendUtteranceOptions
   ): WidgetEventMethod {
-    const utteranceContent = this.content(key, data)
+    const utteranceContent = this.content(key, options?.data)
+    const from = options?.from || 'owner'
 
     return {
       methodName: 'send_utterance',
       methodParams: {
+        from,
         utterance: utteranceContent
       }
     }
@@ -93,6 +102,10 @@ export abstract class Widget<T = unknown> {
 
     let content = widgetContents[key]
 
+    if (Array.isArray(content)) {
+      content = content[Math.floor(Math.random() * content.length)] as string
+    }
+
     if (data) {
       for (const key in data) {
         content = content.replaceAll(`%${key}%`, String(data[key]))

core/skills-endpoints.json

@@ -5,6 +5,17 @@
       "route": "/api/action/unknown/widget-playground/run",
       "params": []
     },
+    {
+      "method": "POST",
+      "route": "/api/action/news/github_trends/run",
+      "params": ["number", "daterange"],
+      "entitiesType": "builtIn"
+    },
+    {
+      "method": "GET",
+      "route": "/api/action/news/product_hunt_trends/run",
+      "params": []
+    },
     {
       "method": "POST",
       "route": "/api/action/games/akinator/choose_thematic",
@@ -57,6 +68,53 @@
       "route": "/api/action/games/rochambeau/rematch",
       "params": []
     },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/create_list",
+      "params": ["list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "GET",
+      "route": "/api/action/productivity/todo_list/view_lists",
+      "params": []
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/view_list",
+      "params": ["list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/rename_list",
+      "params": ["old_list", "new_list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/delete_list",
+      "params": ["list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/add_todos",
+      "params": ["todos", "list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/complete_todos",
+      "params": ["todos", "list"],
+      "entitiesType": "trim"
+    },
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/uncheck_todos",
+      "params": ["todos", "list"],
+      "entitiesType": "trim"
+    },
     {
       "method": "GET",
       "route": "/api/action/leon/age/run",
@@ -132,64 +190,6 @@
       "route": "/api/action/leon/thanks/run",
       "params": []
     },
-    {
-      "method": "POST",
-      "route": "/api/action/news/github_trends/run",
-      "params": ["number", "daterange"],
-      "entitiesType": "builtIn"
-    },
-    {
-      "method": "GET",
-      "route": "/api/action/news/product_hunt_trends/run",
-      "params": []
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/create_list",
-      "params": ["list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "GET",
-      "route": "/api/action/productivity/todo_list/view_lists",
-      "params": []
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/view_list",
-      "params": ["list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/rename_list",
-      "params": ["old_list", "new_list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/delete_list",
-      "params": ["list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/add_todos",
-      "params": ["todos", "list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/complete_todos",
-      "params": ["todos", "list"],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/action/productivity/todo_list/uncheck_todos",
-      "params": ["todos", "list"],
-      "entitiesType": "trim"
-    },
     {
       "method": "GET",
       "route": "/api/action/utilities/date_time/current_date_time",
@@ -236,6 +236,36 @@
       "route": "/api/action/utilities/speed_test/run",
       "params": []
     },
+    {
+      "method": "GET",
+      "route": "/api/action/utilities/timer/configure_set_timer",
+      "params": []
+    },
+    {
+      "method": "GET",
+      "route": "/api/action/utilities/timer/set_timer",
+      "params": []
+    },
+    {
+      "method": "GET",
+      "route": "/api/action/utilities/timer/cancel_timer",
+      "params": []
+    },
+    {
+      "method": "GET",
+      "route": "/api/action/utilities/timer/pause_timer",
+      "params": []
+    },
+    {
+      "method": "GET",
+      "route": "/api/action/utilities/timer/resume_timer",
+      "params": []
+    },
+    {
+      "method": "GET",
+      "route": "/api/action/utilities/timer/check_timer",
+      "params": []
+    },
     {
       "method": "GET",
       "route": "/api/action/utilities/translator-poc/setup",

server/src/core/nlp/nlu/ner.ts

@@ -1,6 +1,7 @@
 import type { ShortLanguageCode } from '@/types'
 import type {
   BuiltInEntityType,
+  NERDurationUnit,
   NEREntity,
   NERSpacyEntity,
   NLPUtterance,
@@ -39,6 +40,42 @@ export const MICROSOFT_BUILT_IN_ENTITIES = [
   'URL'
 ]
 
+function getDurationUnit(duration: string): NERDurationUnit | null {
+  const mapping = {
+    PT: {
+      S: 'seconds',
+      M: 'minutes',
+      H: 'hours'
+    },
+    P: {
+      D: 'days',
+      W: 'weeks',
+      M: 'months',
+      Y: 'years'
+    }
+  }
+
+  const prefix = duration.slice(0, 2)
+  const lastChar = duration.slice(-1)
+
+  if (prefix === 'PT') {
+    return (
+      (mapping.PT[lastChar as keyof typeof mapping.PT] as NERDurationUnit) ??
+      null
+    )
+  }
+  if (prefix.startsWith('P')) {
+    return (
+      (mapping.P[lastChar as keyof typeof mapping.P] as NERDurationUnit) ?? null
+    )
+  }
+
+  LogHelper.title('NER')
+  LogHelper.error(`Failed to get the duration unit: ${duration}`)
+
+  return null
+}
+
 export default class NER {
   private static instance: NER
   public spacyData: Map<
@@ -152,6 +189,15 @@ export default class NER {
             BUILT_IN_ENTITY_TYPES.includes(entity.entity as BuiltInEntityType)
           ) {
             entity.type = entity.entity as BuiltInEntityType
+
+            if (entity.type === 'duration' && entity.resolution.values[0]) {
+              entity.resolution.values[0] = {
+                ...entity.resolution.values[0],
+                unit: getDurationUnit(
+                  entity.resolution.values[0].timex
+                ) as NERDurationUnit
+              }
+            }
           }
 
           if (SPACY_ENTITY_TYPES.includes(entity.entity as SpacyEntityType)) {

server/src/core/nlp/types.ts

@@ -126,6 +126,15 @@ export const BUILT_IN_ENTITY_TYPES = [
   'temperature'
 ] as const
 
+export type NERDurationUnit =
+  | 'seconds'
+  | 'minutes'
+  | 'hours'
+  | 'days'
+  | 'weeks'
+  | 'months'
+  | 'years'
+
 export type BuiltInEntityType = (typeof BUILT_IN_ENTITY_TYPES)[number]
 
 export const CUSTOM_ENTITY_TYPES = ['regex', 'trim', 'enum', 'llm'] as const
@@ -294,8 +303,8 @@ export type BuiltInDurationEntity = BuiltInEntity<
     values: {
       timex: string
       type: string
-      Mod?: 'before' | 'after'
       value: string
+      unit: NERDurationUnit
     }[]
   }
 >

server/src/core/socket-server.ts

@@ -180,17 +180,20 @@ export default class SocketServer {
           })
 
           // Listen for widget events
-          this.socket?.on('widget-event', (event: WidgetDataEvent) => {
+          this.socket?.on('widget-event', async (event: WidgetDataEvent) => {
             LogHelper.title('Socket')
             LogHelper.info(`Widget event: ${JSON.stringify(event)}`)
 
             const { method } = event
 
             if (method.methodName === 'send_utterance') {
-              this.socket?.emit(
+              const utterance = method.methodParams['utterance']
-                'widget-send-utterance',
+
-                method.methodParams['utterance']
+              if (method.methodParams['from'] === 'leon') {
-              )
+                await BRAIN.talk(utterance as string, true)
+              } else {
+                this.socket?.emit('widget-send-utterance', utterance)
+              }
             } else if (method.methodName === 'run_skill_action') {
               this.socket?.emit('widget-run-skill-action', method.methodParams)
             }

server/src/schemas/skill-schemas.ts

@@ -286,7 +286,12 @@ export const skillConfigSchemaObject = Type.Strict(
       )
     ),
     answers: Type.Optional(Type.Record(Type.String(), Type.Array(answerTypes))),
-    widget_contents: Type.Optional(Type.Record(Type.String(), Type.String())),
+    widget_contents: Type.Optional(
+      Type.Record(
+        Type.String(),
+        Type.Union([Type.String(), Type.Array(Type.String())])
+      )
+    ),
     entities: Type.Optional(Type.Record(Type.String(), Type.String())),
     resolvers: Type.Optional(
       Type.Record(

skills/unknown/widget-playground/src/widgets/my-component.tsx

@@ -1,10 +0,0 @@
-import React from 'react'
-
-const MyComponent = () => {
-  const handler = () => {
-    console.log('clicked from handler')
-  }
-  return <button onClick={handler}>Hello from MyComponent</button>
-}
-
-export default MyComponent

skills/unknown/widget-playground/src/widgets/playground-test.ts

@@ -29,7 +29,9 @@ export class PlaygroundTestWidget extends Widget<Params> {
             children: provider,
             onClick: (): WidgetEventMethod => {
               return this.sendUtterance('choose_provider', {
-                provider
+                data: {
+                  provider
+                }
               })
             }
           })
@@ -37,9 +39,6 @@ export class PlaygroundTestWidget extends Widget<Params> {
       }
     )
 
-    // TODO: timer
-
-    // TODO
     return new Flexbox({
       gap: 'md',
       flexDirection: 'column',
@@ -101,7 +100,6 @@ export class PlaygroundTestWidget extends Widget<Params> {
             ...buttons
           ]
         })
-        // TODO: form input
       ]
     })
   }

skills/utilities/timer/config/en.json

@@ -0,0 +1,71 @@
+{
+  "$schema": "../../../../schemas/skill-schemas/skill-config.json",
+  "variables": {
+    "hours": "hours",
+    "minutes": "minutes",
+    "seconds": "seconds"
+  },
+  "actions": {
+    "configure_set_timer": {
+      "type": "dialog",
+      "utterance_samples": [
+        "[Set|Start|Create] a timer for @number [minutes|seconds]"
+      ],
+      "slots": [
+        {
+          "name": "duration",
+          "item": {
+            "type": "entity",
+            "name": "duration"
+          },
+          "questions": ["For how long time?"]
+        }
+      ],
+      "next_action": "set_timer"
+    },
+    "set_timer": {
+      "type": "logic"
+    },
+    "cancel_timer": {
+      "type": "logic",
+      "utterance_samples": [
+        "[Cancel|Stop|Turn off|Delete] the timer",
+        "Don't need the timer"
+      ]
+    },
+    "pause_timer": {
+      "type": "logic",
+      "utterance_samples": ["Pause the timer", "Put the timer on hold"]
+    },
+    "resume_timer": {
+      "type": "logic",
+      "utterance_samples": ["[Resume|Continue] the timer"]
+    },
+    "check_timer": {
+      "type": "logic",
+      "utterance_samples": [
+        "How much time left on the timer",
+        "Check the timer",
+        "What's the remaining time on the timer"
+      ]
+    }
+  },
+  "answers": {
+    "cannot_get_duration": [
+      "Sorry, I can't get the duration of the timer.",
+      "I can't get the duration of the timer. Sorry."
+    ],
+    "unit_not_supported": [
+      "Sorry, I can't set a timer for this unit. Use %hours%, %minutes% or %seconds% instead.",
+      "I can't set a timer for this duration. Use %hours%, %minutes% or %seconds% instead."
+    ]
+  },
+  "widget_contents": {
+    "second_unit": "second",
+    "seconds_unit": "seconds",
+    "minutes_unit": "minutes",
+    "minute_unit": "minute",
+    "total_time": "Total %value% %unit%",
+    "times_up": ["Time's up!", "The timer is up!", "The timer has ended!"]
+  }
+}

skills/utilities/timer/skill.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "../../../schemas/skill-schemas/skill.json",
+  "name": "Timer",
+  "bridge": "nodejs",
+  "version": "1.0.0",
+  "description": "Set timers to remind you of things",
+  "author": {
+    "name": "Louis Grenard",
+    "email": "louis@getleon.ai",
+    "url": "https://github.com/louistiti"
+  }
+}

skills/utilities/timer/src/actions/set_timer.ts

@@ -0,0 +1,36 @@
+import type { ActionFunction, BuiltInDurationEntity } from '@sdk/types'
+import { leon } from '@sdk/leon'
+
+import { TimerWidget } from '../widgets/timer'
+
+export const run: ActionFunction = async function (params) {
+  const supportedUnits = ['hours', 'minutes', 'seconds']
+  const durations = (
+    params.slots['duration']?.resolution as BuiltInDurationEntity['resolution']
+  ).values
+  const [duration] = durations
+
+  if (!duration) {
+    return leon.answer({ key: 'cannot_get_duration' })
+  }
+
+  const { unit } = duration
+  if (!supportedUnits.includes(unit)) {
+    return leon.answer({ key: 'unit_not_supported' })
+  }
+
+  const { value: durationValue } = duration
+  const seconds = Number(durationValue)
+  const timerWidget = new TimerWidget({
+    params: {
+      seconds
+    }
+  })
+
+  // TODO: return a speech without new utterance
+  /*await leon.answer({
+    widget: timerWidget,
+    speech: 'I set a timer for ... ...'
+  })*/
+  await leon.answer({ widget: timerWidget })
+}

skills/utilities/timer/src/settings.sample.json

@@ -0,0 +1 @@
+{}

skills/utilities/timer/src/widgets/components/timer.ts

@@ -0,0 +1,14 @@
+import { WidgetComponent } from '@sdk/widget-component'
+
+interface TimerProps {
+  initialTime: number
+  interval: number
+  totalTimeContent: string
+  onEnd?: () => void
+}
+
+export class Timer extends WidgetComponent<TimerProps> {
+  constructor(props: TimerProps) {
+    super(props)
+  }
+}

skills/utilities/timer/src/widgets/timer.ts

@@ -0,0 +1,56 @@
+import { Widget, type WidgetEventMethod, type WidgetOptions } from '@sdk/widget'
+import { type WidgetComponent } from '@sdk/widget-component'
+
+import { Timer } from './components/timer'
+
+interface Params {
+  seconds: number
+}
+
+export class TimerWidget extends Widget<Params> {
+  constructor(options: WidgetOptions<Params>) {
+    super(options)
+  }
+
+  /**
+   * TODO
+   * 1. Save timer + timer id in memory
+   * 2. On rendering, set widget id to timer id
+   * 3. When load feed, need to fetch all timers (onFetch?) as per their timer id. Need a built-in API here
+   * 4. onEnd (or onChange and check if done?), then trigger next action or utterance
+   */
+
+  public render(): WidgetComponent {
+    const { seconds } = this.params
+    const secondUnitContent = this.content('second_unit')
+    const secondsUnitContent = this.content('seconds_unit')
+    const minuteUnitContent = this.content('minute_unit')
+    const minutesUnitContent = this.content('minutes_unit')
+    let totalTimeContent = ''
+
+    if (seconds >= 60) {
+      const minutes = seconds / 60
+
+      totalTimeContent = this.content('total_time', {
+        value: minutes % 1 === 0 ? minutes : minutes.toFixed(2),
+        unit: minutes > 1 ? minutesUnitContent : minuteUnitContent
+      })
+    } else {
+      totalTimeContent = this.content('total_time', {
+        value: seconds,
+        unit: seconds > 1 ? secondsUnitContent : secondUnitContent
+      })
+    }
+
+    return new Timer({
+      initialTime: seconds,
+      interval: 1_000,
+      totalTimeContent,
+      onEnd: (): WidgetEventMethod => {
+        return this.sendUtterance('times_up', {
+          from: 'leon'
+        })
+      }
+    })
+  }
+}

tsconfig.json

@@ -6,6 +6,7 @@
     "baseUrl": ".",
     "moduleResolution": "Node",
     "module": "CommonJS",
+    "jsx": "react",
     "paths": {
       "@@/*": ["./*"],
       "@/*": ["./server/src/*"],