mirror of
https://github.com/leon-ai/leon.git
synced 2024-10-04 05:07:12 +03:00
Merge branch 'feat/widgets' into develop
This commit is contained in:
commit
d417384204
@ -138,7 +138,9 @@ table {
|
||||
--blue-color: #1c75db;
|
||||
--pink-color: #ed297a;
|
||||
|
||||
--a-loader-size-md: 20px !important;
|
||||
.settingup {
|
||||
--a-loader-size-md: 20px !important;
|
||||
}
|
||||
}
|
||||
|
||||
a {
|
||||
@ -263,7 +265,7 @@ textarea::-webkit-scrollbar-thumb {
|
||||
|
||||
#top-container {
|
||||
position: absolute;
|
||||
top: 6%;
|
||||
top: 4%;
|
||||
color: var(--grey-color);
|
||||
display: flex;
|
||||
width: 100%;
|
||||
@ -286,7 +288,7 @@ textarea::-webkit-scrollbar-thumb {
|
||||
#feed {
|
||||
position: absolute;
|
||||
width: 100%;
|
||||
top: 10%;
|
||||
top: 8%;
|
||||
height: 50%;
|
||||
overflow-y: auto;
|
||||
border: 2px solid var(--grey-color);
|
||||
@ -431,7 +433,7 @@ textarea::-webkit-scrollbar-thumb {
|
||||
#input-container {
|
||||
position: absolute;
|
||||
width: 100%;
|
||||
bottom: 22%;
|
||||
bottom: 18%;
|
||||
}
|
||||
|
||||
#mic-container {
|
||||
|
@ -5,6 +5,7 @@ interface TimerProps {
|
||||
initialTime: number
|
||||
interval: number
|
||||
totalTimeContent: string
|
||||
initialProgress?: number
|
||||
onEnd?: () => void
|
||||
}
|
||||
|
||||
@ -20,16 +21,17 @@ function formatTime(seconds: number): string {
|
||||
|
||||
export function Timer({
|
||||
initialTime,
|
||||
initialProgress,
|
||||
interval,
|
||||
totalTimeContent,
|
||||
onEnd
|
||||
}: TimerProps) {
|
||||
const [progress, setProgress] = useState(0)
|
||||
const [progress, setProgress] = useState(initialProgress || 0)
|
||||
const [timeLeft, setTimeLeft] = useState(initialTime)
|
||||
|
||||
useEffect(() => {
|
||||
setTimeLeft(initialTime)
|
||||
setProgress(0)
|
||||
setProgress(progress)
|
||||
}, [initialTime])
|
||||
|
||||
useEffect(() => {
|
||||
|
@ -1,10 +1,17 @@
|
||||
import { createElement } from 'react'
|
||||
import { createRoot } from 'react-dom/client'
|
||||
import axios from 'axios'
|
||||
import { WidgetWrapper, Flexbox, Loader } from '@leon-ai/aurora'
|
||||
|
||||
import renderAuroraComponent from './render-aurora-component'
|
||||
|
||||
const WIDGETS_TO_FETCH = []
|
||||
const WIDGETS_FETCH_CACHE = new Map()
|
||||
|
||||
export default class Chatbot {
|
||||
constructor(socket) {
|
||||
constructor(socket, serverURL) {
|
||||
this.socket = socket
|
||||
this.serverURL = serverURL
|
||||
this.et = new EventTarget()
|
||||
this.feed = document.querySelector('#feed')
|
||||
this.typing = document.querySelector('#is-typing')
|
||||
@ -18,11 +25,17 @@ export default class Chatbot {
|
||||
this.scrollDown()
|
||||
|
||||
this.et.addEventListener('to-leon', (event) => {
|
||||
this.createBubble('me', event.detail)
|
||||
this.createBubble({
|
||||
who: 'me',
|
||||
string: event.detail
|
||||
})
|
||||
})
|
||||
|
||||
this.et.addEventListener('me-received', (event) => {
|
||||
this.createBubble('leon', event.detail)
|
||||
this.createBubble({
|
||||
who: 'leon',
|
||||
string: event.detail
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@ -65,7 +78,7 @@ export default class Chatbot {
|
||||
}
|
||||
|
||||
loadFeed() {
|
||||
return new Promise((resolve) => {
|
||||
return new Promise(async (resolve) => {
|
||||
if (this.parsedBubbles === null || this.parsedBubbles.length === 0) {
|
||||
this.noBubbleMessage.classList.remove('hide')
|
||||
localStorage.setItem('bubbles', JSON.stringify([]))
|
||||
@ -75,7 +88,12 @@ export default class Chatbot {
|
||||
for (let i = 0; i < this.parsedBubbles.length; i += 1) {
|
||||
const bubble = this.parsedBubbles[i]
|
||||
|
||||
this.createBubble(bubble.who, bubble.string, false)
|
||||
this.createBubble({
|
||||
who: bubble.who,
|
||||
string: bubble.string,
|
||||
save: false,
|
||||
isCreatingFromLoadingFeed: true
|
||||
})
|
||||
|
||||
if (i + 1 === this.parsedBubbles.length) {
|
||||
setTimeout(() => {
|
||||
@ -83,20 +101,70 @@ export default class Chatbot {
|
||||
}, 100)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Browse widgets that need to be fetched.
|
||||
* Reverse widgets to fetch the last widgets first.
|
||||
* Replace the loading content with the fetched widget
|
||||
*/
|
||||
const widgetContainers = WIDGETS_TO_FETCH.reverse()
|
||||
for (let i = 0; i < widgetContainers.length; i += 1) {
|
||||
const widgetContainer = widgetContainers[i]
|
||||
const hasWidgetBeenFetched = WIDGETS_FETCH_CACHE.has(
|
||||
widgetContainer.widgetId
|
||||
)
|
||||
|
||||
if (hasWidgetBeenFetched) {
|
||||
const fetchedWidget = WIDGETS_FETCH_CACHE.get(
|
||||
widgetContainer.widgetId
|
||||
)
|
||||
widgetContainer.reactRootNode.render(fetchedWidget.reactNode)
|
||||
|
||||
setTimeout(() => {
|
||||
this.scrollDown()
|
||||
}, 100)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
const data = await axios.get(
|
||||
`${this.serverURL}/api/v1/fetch-widget?skill_action=${widgetContainer.onFetchAction}&widget_id=${widgetContainer.widgetId}`
|
||||
)
|
||||
const fetchedWidget = data.data.widget
|
||||
const reactNode = renderAuroraComponent(
|
||||
this.socket,
|
||||
fetchedWidget.componentTree,
|
||||
fetchedWidget.supportedEvents
|
||||
)
|
||||
|
||||
widgetContainer.reactRootNode.render(reactNode)
|
||||
WIDGETS_FETCH_CACHE.set(widgetContainer.widgetId, {
|
||||
...fetchedWidget,
|
||||
reactNode
|
||||
})
|
||||
this.scrollDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
createBubble(who, string, save = true, bubbleId) {
|
||||
createBubble(params) {
|
||||
const {
|
||||
who,
|
||||
string,
|
||||
save = true,
|
||||
bubbleId,
|
||||
isCreatingFromLoadingFeed = false
|
||||
} = params
|
||||
const container = document.createElement('div')
|
||||
const bubble = document.createElement('p')
|
||||
|
||||
container.className = `bubble-container ${who}`
|
||||
bubble.className = 'bubble'
|
||||
|
||||
string = this.formatMessage(string)
|
||||
const formattedString = this.formatMessage(string)
|
||||
|
||||
bubble.innerHTML = string
|
||||
bubble.innerHTML = formattedString
|
||||
|
||||
if (bubbleId) {
|
||||
container.classList.add(bubbleId)
|
||||
@ -104,22 +172,55 @@ export default class Chatbot {
|
||||
|
||||
this.feed.appendChild(container).appendChild(bubble)
|
||||
|
||||
let widgetTree = null
|
||||
let widgetComponentTree = null
|
||||
let widgetSupportedEvents = null
|
||||
|
||||
/**
|
||||
* Widget rendering
|
||||
*/
|
||||
if (string.startsWith && string.startsWith('{"tree":{"component')) {
|
||||
const parsedWidget = JSON.parse(string)
|
||||
const root = createRoot(container)
|
||||
if (
|
||||
formattedString.includes &&
|
||||
formattedString.includes('"component":"WidgetWrapper"')
|
||||
) {
|
||||
const parsedWidget = JSON.parse(formattedString)
|
||||
container.setAttribute('data-widget-id', parsedWidget.id)
|
||||
|
||||
widgetTree = parsedWidget.tree
|
||||
/**
|
||||
* On widget fetching, render the loader
|
||||
*/
|
||||
if (isCreatingFromLoadingFeed && parsedWidget.onFetchAction) {
|
||||
const root = createRoot(container)
|
||||
|
||||
root.render(
|
||||
createElement(WidgetWrapper, {
|
||||
children: createElement(Flexbox, {
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
children: createElement(Loader)
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
WIDGETS_TO_FETCH.push({
|
||||
reactRootNode: root,
|
||||
widgetId: parsedWidget.id,
|
||||
onFetchAction: parsedWidget.onFetchAction
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
widgetComponentTree = parsedWidget.componentTree
|
||||
widgetSupportedEvents = parsedWidget.supportedEvents
|
||||
|
||||
/**
|
||||
* On widget creation
|
||||
*/
|
||||
const root = createRoot(container)
|
||||
|
||||
const reactNode = renderAuroraComponent(
|
||||
this.socket,
|
||||
widgetTree,
|
||||
widgetComponentTree,
|
||||
widgetSupportedEvents
|
||||
)
|
||||
|
||||
@ -127,7 +228,7 @@ export default class Chatbot {
|
||||
}
|
||||
|
||||
if (save) {
|
||||
this.saveBubble(who, string)
|
||||
this.saveBubble(who, formattedString)
|
||||
}
|
||||
|
||||
return container
|
||||
|
@ -14,7 +14,7 @@ export default class Client {
|
||||
this.socket = io(this.serverUrl)
|
||||
this.history = localStorage.getItem('history')
|
||||
this.parsedHistory = []
|
||||
this.chatbot = new Chatbot(this.socket)
|
||||
this.chatbot = new Chatbot(this.socket, this.serverUrl)
|
||||
this.voiceEnergy = new VoiceEnergy(this)
|
||||
this._recorder = {}
|
||||
this._suggestions = []
|
||||
@ -178,7 +178,10 @@ export default class Client {
|
||||
})
|
||||
|
||||
this.socket.on('widget', (data) => {
|
||||
this.chatbot.createBubble('leon', data)
|
||||
this.chatbot.createBubble({
|
||||
who: 'leon',
|
||||
string: data
|
||||
})
|
||||
})
|
||||
|
||||
this.socket.on('widget-send-utterance', (utterance) => {
|
||||
@ -203,12 +206,12 @@ export default class Client {
|
||||
let bubbleContainerElement = null
|
||||
|
||||
if (!isSameGeneration) {
|
||||
bubbleContainerElement = this.chatbot.createBubble(
|
||||
'leon',
|
||||
data.token,
|
||||
false,
|
||||
newGenerationId
|
||||
)
|
||||
bubbleContainerElement = this.chatbot.createBubble({
|
||||
who: 'leon',
|
||||
string: data.token,
|
||||
save: false,
|
||||
bubbleId: newGenerationId
|
||||
})
|
||||
} else {
|
||||
bubbleContainerElement = document.querySelector(
|
||||
`.${previousGenerationId}`
|
||||
|
@ -24,18 +24,16 @@ export default function renderAuroraComponent(
|
||||
}
|
||||
|
||||
// Check if the browsed component has a supported event and bind it
|
||||
if (
|
||||
reactComponent &&
|
||||
component.events[0] &&
|
||||
supportedEvents.includes(component.events[0].type)
|
||||
) {
|
||||
const eventType = component.events[0].type
|
||||
if (reactComponent) {
|
||||
component.events.forEach((event) => {
|
||||
if (supportedEvents.includes(event.type)) {
|
||||
component.props[event.type] = (data) => {
|
||||
const { method } = event
|
||||
|
||||
component.props[eventType] = (data) => {
|
||||
const { method } = component.events[0]
|
||||
|
||||
socket.emit('widget-event', { method, data })
|
||||
}
|
||||
socket.emit('widget-event', { method, data })
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// When children is a component, then wrap it in an array to render properly
|
||||
|
@ -117,7 +117,11 @@ class Leon {
|
||||
|
||||
if (answerInput.widget) {
|
||||
answerObject.output.widget = {
|
||||
tree: new WidgetWrapper({
|
||||
actionName: `${INTENT_OBJECT.domain}:${INTENT_OBJECT.skill}:${INTENT_OBJECT.action}`,
|
||||
widget: answerInput.widget.widget,
|
||||
id: answerInput.widget.id,
|
||||
onFetchAction: answerInput.widget.onFetchAction,
|
||||
componentTree: new WidgetWrapper({
|
||||
...answerInput.widget.wrapperProps,
|
||||
children: [answerInput.widget.render()]
|
||||
}),
|
||||
|
13
bridges/nodejs/src/sdk/toolbox.ts
Normal file
13
bridges/nodejs/src/sdk/toolbox.ts
Normal file
@ -0,0 +1,13 @@
|
||||
import { INTENT_OBJECT } from '@bridge/constants'
|
||||
|
||||
/**
|
||||
* Get the widget ID if any
|
||||
* @example getWidgetId() // 'timerwidget-5q1xlzeh
|
||||
*/
|
||||
export function getWidgetId(): string | null {
|
||||
return (
|
||||
INTENT_OBJECT.current_entities.find(
|
||||
(entity) => entity.entity === 'widgetid'
|
||||
)?.sourceText ?? null
|
||||
)
|
||||
}
|
@ -9,8 +9,7 @@ export const SUPPORTED_WIDGET_EVENTS = [
|
||||
'onSubmit',
|
||||
'onChange',
|
||||
'onStart',
|
||||
'onEnd',
|
||||
'onFetch'
|
||||
'onEnd'
|
||||
] as const
|
||||
|
||||
function generateId(): string {
|
||||
|
@ -1,10 +1,15 @@
|
||||
import { type WidgetWrapperProps } from '@leon-ai/aurora'
|
||||
|
||||
import { SKILL_CONFIG } from '@bridge/constants'
|
||||
import { INTENT_OBJECT, SKILL_CONFIG } from '@bridge/constants'
|
||||
import { WidgetComponent } from '@sdk/widget-component'
|
||||
|
||||
type UtteranceSender = 'leon' | 'owner'
|
||||
|
||||
interface FetchWidgetDataWidgetEventMethodParams {
|
||||
actionName: string
|
||||
widgetId: string
|
||||
dataToSet: string[]
|
||||
}
|
||||
interface SendUtteranceWidgetEventMethodParams {
|
||||
from: UtteranceSender
|
||||
utterance: string
|
||||
@ -23,13 +28,19 @@ export interface WidgetEventMethod {
|
||||
methodParams:
|
||||
| SendUtteranceWidgetEventMethodParams
|
||||
| RunSkillActionWidgetEventMethodParams
|
||||
| FetchWidgetDataWidgetEventMethodParams
|
||||
}
|
||||
export interface WidgetOptions<T = unknown> {
|
||||
wrapperProps?: Omit<WidgetWrapperProps, 'children'>
|
||||
onFetchAction?: string
|
||||
params: T
|
||||
}
|
||||
|
||||
export abstract class Widget<T = unknown> {
|
||||
public actionName: string
|
||||
public id: string
|
||||
public widget: string
|
||||
public onFetchAction: string | null = null
|
||||
public wrapperProps: WidgetOptions<T>['wrapperProps']
|
||||
public params: WidgetOptions<T>['params']
|
||||
|
||||
@ -37,6 +48,14 @@ export abstract class Widget<T = unknown> {
|
||||
if (options?.wrapperProps) {
|
||||
this.wrapperProps = options.wrapperProps
|
||||
}
|
||||
if (options?.onFetchAction) {
|
||||
this.onFetchAction = `${INTENT_OBJECT.domain}:${INTENT_OBJECT.skill}:${options.onFetchAction}`
|
||||
}
|
||||
this.actionName = `${INTENT_OBJECT.domain}:${INTENT_OBJECT.skill}:${INTENT_OBJECT.action}`
|
||||
this.widget = this.constructor.name
|
||||
this.id = `${this.widget.toLowerCase()}-${Math.random()
|
||||
.toString(36)
|
||||
.substring(2, 10)}`
|
||||
this.params = options.params
|
||||
}
|
||||
|
||||
|
@ -5,17 +5,6 @@
|
||||
"route": "/api/action/unknown/widget-playground/run",
|
||||
"params": []
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/news/github_trends/run",
|
||||
"params": ["number", "daterange"],
|
||||
"entitiesType": "builtIn"
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"route": "/api/action/news/product_hunt_trends/run",
|
||||
"params": []
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/games/akinator/choose_thematic",
|
||||
@ -70,51 +59,15 @@
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/create_list",
|
||||
"params": ["list"],
|
||||
"entitiesType": "trim"
|
||||
"route": "/api/action/news/github_trends/run",
|
||||
"params": ["number", "daterange"],
|
||||
"entitiesType": "builtIn"
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"route": "/api/action/productivity/todo_list/view_lists",
|
||||
"route": "/api/action/news/product_hunt_trends/run",
|
||||
"params": []
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/view_list",
|
||||
"params": ["list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/rename_list",
|
||||
"params": ["old_list", "new_list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/delete_list",
|
||||
"params": ["list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/add_todos",
|
||||
"params": ["todos", "list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/complete_todos",
|
||||
"params": ["todos", "list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/uncheck_todos",
|
||||
"params": ["todos", "list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"route": "/api/action/leon/age/run",
|
||||
@ -190,6 +143,53 @@
|
||||
"route": "/api/action/leon/thanks/run",
|
||||
"params": []
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/create_list",
|
||||
"params": ["list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"route": "/api/action/productivity/todo_list/view_lists",
|
||||
"params": []
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/view_list",
|
||||
"params": ["list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/rename_list",
|
||||
"params": ["old_list", "new_list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/delete_list",
|
||||
"params": ["list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/add_todos",
|
||||
"params": ["todos", "list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/complete_todos",
|
||||
"params": ["todos", "list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "POST",
|
||||
"route": "/api/action/productivity/todo_list/uncheck_todos",
|
||||
"params": ["todos", "list"],
|
||||
"entitiesType": "trim"
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"route": "/api/action/utilities/date_time/current_date_time",
|
||||
@ -236,11 +236,6 @@
|
||||
"route": "/api/action/utilities/speed_test/run",
|
||||
"params": []
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"route": "/api/action/utilities/timer/configure_set_timer",
|
||||
"params": []
|
||||
},
|
||||
{
|
||||
"method": "GET",
|
||||
"route": "/api/action/utilities/timer/set_timer",
|
||||
|
@ -64,7 +64,7 @@ export default class Brain {
|
||||
private skillProcess: ChildProcessWithoutNullStreams | undefined = undefined
|
||||
private domainFriendlyName = ''
|
||||
private skillFriendlyName = ''
|
||||
private skillOutput = ''
|
||||
public skillOutput = ''
|
||||
public isMuted = false // Close Leon mouth if true; e.g. over HTTP
|
||||
|
||||
constructor() {
|
||||
@ -430,7 +430,7 @@ export default class Brain {
|
||||
LogHelper.title(`${this.skillFriendlyName} skill (on data)`)
|
||||
LogHelper.info(data.toString())
|
||||
|
||||
if (skillAnswer.output.widget) {
|
||||
if (skillAnswer.output.widget && !this.isMuted) {
|
||||
try {
|
||||
SOCKET_SERVER.socket?.emit(
|
||||
'widget',
|
||||
|
@ -39,7 +39,7 @@ export interface SkillResult {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
options: Record<string, any>
|
||||
widget?: {
|
||||
tree: WidgetWrapper
|
||||
componentTree: WidgetWrapper
|
||||
supportedEvents: typeof SUPPORTED_WIDGET_EVENTS
|
||||
}
|
||||
}
|
||||
@ -93,8 +93,12 @@ export interface SkillAnswerOutput extends IntentObject {
|
||||
answer: SkillAnswerConfigSchema
|
||||
core?: SkillAnswerCoreData
|
||||
widget?: {
|
||||
tree: WidgetWrapper
|
||||
actionName: string
|
||||
widget: string
|
||||
id: string
|
||||
componentTree: WidgetWrapper
|
||||
supportedEvents: typeof SUPPORTED_WIDGET_EVENTS
|
||||
onFetchAction: string | null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
126
server/src/core/http-server/api/fetch-widget/get.ts
Normal file
126
server/src/core/http-server/api/fetch-widget/get.ts
Normal file
@ -0,0 +1,126 @@
|
||||
import type { FastifyPluginAsync } from 'fastify'
|
||||
|
||||
import type { APIOptions } from '@/core/http-server/http-server'
|
||||
import { BRAIN } from '@/core'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
import { DEFAULT_NLU_RESULT } from '@/core/nlp/nlu/nlu'
|
||||
import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
|
||||
|
||||
export const fetchWidget: FastifyPluginAsync<APIOptions> = async (
|
||||
fastify,
|
||||
options
|
||||
) => {
|
||||
fastify.route({
|
||||
method: 'GET',
|
||||
url: `/api/${options.apiVersion}/fetch-widget`,
|
||||
handler: async (_request, reply) => {
|
||||
let message
|
||||
|
||||
try {
|
||||
const queryParams = _request.query as Record<string, string>
|
||||
const { skill_action: skillAction, widget_id: widgetId } = queryParams
|
||||
|
||||
if (!skillAction || !widgetId) {
|
||||
reply.statusCode = 400
|
||||
message = 'skill_action and widget_id are missing.'
|
||||
LogHelper.title('GET /fetch-widget')
|
||||
LogHelper.warning(message)
|
||||
return reply.send({
|
||||
success: false,
|
||||
status: reply.statusCode,
|
||||
code: 'missing_params',
|
||||
message,
|
||||
widget: null
|
||||
})
|
||||
}
|
||||
|
||||
const [domain, skill, action] = skillAction.split(':')
|
||||
|
||||
if (!domain || !skill || !action) {
|
||||
message = 'skill_action is not well formatted.'
|
||||
LogHelper.title('GET /fetch-widget')
|
||||
LogHelper.warning(message)
|
||||
return reply.send({
|
||||
success: false,
|
||||
status: reply.statusCode,
|
||||
code: 'skill_action_not_valid',
|
||||
message,
|
||||
widget: null
|
||||
})
|
||||
}
|
||||
|
||||
// Do not return any speech and new widget
|
||||
BRAIN.isMuted = true
|
||||
await BRAIN.execute({
|
||||
...DEFAULT_NLU_RESULT,
|
||||
currentEntities: [
|
||||
{
|
||||
start: 0,
|
||||
end: widgetId.length - 1,
|
||||
len: widgetId.length,
|
||||
levenshtein: 0,
|
||||
accuracy: 1,
|
||||
entity: 'widgetid',
|
||||
type: 'enum',
|
||||
option: widgetId,
|
||||
sourceText: widgetId,
|
||||
utteranceText: widgetId,
|
||||
resolution: {
|
||||
value: widgetId
|
||||
}
|
||||
}
|
||||
],
|
||||
skillConfigPath: SkillDomainHelper.getSkillConfigPath(
|
||||
domain,
|
||||
skill,
|
||||
BRAIN.lang
|
||||
),
|
||||
classification: {
|
||||
domain,
|
||||
skill,
|
||||
action,
|
||||
confidence: 1
|
||||
}
|
||||
})
|
||||
|
||||
const parsedOutput = JSON.parse(BRAIN.skillOutput)
|
||||
|
||||
if (parsedOutput.output.widget) {
|
||||
message = 'Widget fetched successfully.'
|
||||
LogHelper.title('GET /fetch-widget')
|
||||
LogHelper.success(message)
|
||||
return reply.send({
|
||||
success: true,
|
||||
status: 200,
|
||||
code: 'widget_fetched',
|
||||
message,
|
||||
widget: parsedOutput.output.widget
|
||||
})
|
||||
}
|
||||
|
||||
message = 'Widget not fetched.'
|
||||
LogHelper.title('GET /fetch-widget')
|
||||
LogHelper.success(message)
|
||||
return reply.send({
|
||||
success: true,
|
||||
status: 200,
|
||||
code: 'widget_not_fetched',
|
||||
message,
|
||||
widget: null
|
||||
})
|
||||
} catch (e) {
|
||||
LogHelper.title('HTTP Server')
|
||||
LogHelper.error(`Failed to fetch widget component tree: ${e}`)
|
||||
|
||||
reply.statusCode = 500
|
||||
return reply.send({
|
||||
success: false,
|
||||
status: reply.statusCode,
|
||||
code: 'fetch_widget_error',
|
||||
message: 'Failed to fetch widget component tree.',
|
||||
widget: null
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
12
server/src/core/http-server/api/fetch-widget/index.ts
Normal file
12
server/src/core/http-server/api/fetch-widget/index.ts
Normal file
@ -0,0 +1,12 @@
|
||||
import type { FastifyPluginAsync } from 'fastify'
|
||||
|
||||
import { fetchWidget } from '@/core/http-server/api/fetch-widget/get'
|
||||
import type { APIOptions } from '@/core/http-server/http-server'
|
||||
|
||||
export const fetchWidgetPlugin: FastifyPluginAsync<APIOptions> = async (
|
||||
fastify,
|
||||
options
|
||||
) => {
|
||||
// Fetch widget component tree
|
||||
await fastify.register(fetchWidget, options)
|
||||
}
|
@ -16,6 +16,7 @@ import { corsMidd } from '@/core/http-server/plugins/cors'
|
||||
import { otherMidd } from '@/core/http-server/plugins/other'
|
||||
import { infoPlugin } from '@/core/http-server/api/info'
|
||||
import { llmInferencePlugin } from '@/core/http-server/api/llm-inference'
|
||||
import { fetchWidgetPlugin } from '@/core/http-server/api/fetch-widget'
|
||||
import { keyMidd } from '@/core/http-server/plugins/key'
|
||||
import { utterancePlugin } from '@/core/http-server/api/utterance'
|
||||
import { LLM_MANAGER, PERSONA } from '@/core'
|
||||
@ -59,13 +60,9 @@ export default class HTTPServer {
|
||||
LogHelper.title('Initialization')
|
||||
LogHelper.info(`Environment: ${LEON_NODE_ENV}`)
|
||||
LogHelper.info(`Version: ${LEON_VERSION}`)
|
||||
|
||||
LogHelper.info(`Time zone: ${DateHelper.getTimeZone()}`)
|
||||
|
||||
LogHelper.info(`LLM provider: ${LLM_PROVIDER}`)
|
||||
|
||||
LogHelper.info(`Mood: ${PERSONA.mood.type}`)
|
||||
|
||||
LogHelper.info(`GPU: ${(await SystemHelper.getGPUDeviceNames())[0]}`)
|
||||
LogHelper.info(
|
||||
`Graphics compute API: ${await SystemHelper.getGraphicsComputeAPI()}`
|
||||
@ -101,8 +98,8 @@ export default class HTTPServer {
|
||||
reply.sendFile('index.html')
|
||||
})
|
||||
|
||||
this.fastify.register(fetchWidgetPlugin, { apiVersion: API_VERSION })
|
||||
this.fastify.register(infoPlugin, { apiVersion: API_VERSION })
|
||||
|
||||
this.fastify.register(llmInferencePlugin, { apiVersion: API_VERSION })
|
||||
|
||||
if (HAS_OVER_HTTP) {
|
||||
|
@ -22,13 +22,10 @@ export class ActionLoop {
|
||||
): Promise<Partial<BrainProcessResult> | null> {
|
||||
const { domain, intent } = NLU.conversation.activeContext
|
||||
const [skillName, actionName] = intent.split('.') as [string, string]
|
||||
const skillConfigPath = join(
|
||||
process.cwd(),
|
||||
'skills',
|
||||
const skillConfigPath = SkillDomainHelper.getSkillConfigPath(
|
||||
domain,
|
||||
skillName,
|
||||
'config',
|
||||
BRAIN.lang + '.json'
|
||||
BRAIN.lang
|
||||
)
|
||||
const newNLUResult = {
|
||||
...DEFAULT_NLU_RESULT, // Reset entities, slots, etc.
|
||||
|
@ -1,4 +1,3 @@
|
||||
import { join } from 'node:path'
|
||||
import { spawn } from 'node:child_process'
|
||||
|
||||
import kill from 'tree-kill'
|
||||
@ -81,13 +80,10 @@ export default class NLU {
|
||||
|
||||
const skillConfigPath = newNLUResult.skillConfigPath
|
||||
? newNLUResult.skillConfigPath
|
||||
: join(
|
||||
process.cwd(),
|
||||
'skills',
|
||||
: SkillDomainHelper.getSkillConfigPath(
|
||||
newNLUResult.classification.domain,
|
||||
newNLUResult.classification.skill,
|
||||
'config',
|
||||
BRAIN.lang + '.json'
|
||||
BRAIN.lang
|
||||
)
|
||||
const { actions } = await SkillDomainHelper.getSkillConfig(
|
||||
skillConfigPath,
|
||||
@ -433,13 +429,10 @@ export default class NLU {
|
||||
)}`
|
||||
)
|
||||
|
||||
const skillConfigPath = join(
|
||||
process.cwd(),
|
||||
'skills',
|
||||
const skillConfigPath = SkillDomainHelper.getSkillConfigPath(
|
||||
this._nluResult.classification.domain,
|
||||
this._nluResult.classification.skill,
|
||||
'config',
|
||||
BRAIN.lang + '.json'
|
||||
BRAIN.lang
|
||||
)
|
||||
this._nluResult.skillConfigPath = skillConfigPath
|
||||
|
||||
|
@ -1,5 +1,3 @@
|
||||
import { join } from 'node:path'
|
||||
|
||||
import type { NLPUtterance } from '@/core/nlp/types'
|
||||
import type { BrainProcessResult } from '@/core/brain/types'
|
||||
import { BRAIN, MODEL_LOADER, NER, NLU, SOCKET_SERVER } from '@/core'
|
||||
@ -61,13 +59,10 @@ export class SlotFilling {
|
||||
|
||||
const { domain, intent } = NLU.conversation.activeContext
|
||||
const [skillName, actionName] = intent.split('.') as [string, string]
|
||||
const skillConfigPath = join(
|
||||
process.cwd(),
|
||||
'skills',
|
||||
const skillConfigPath = SkillDomainHelper.getSkillConfigPath(
|
||||
domain,
|
||||
skillName,
|
||||
'config',
|
||||
BRAIN.lang + '.json'
|
||||
BRAIN.lang
|
||||
)
|
||||
|
||||
await NLU.setNLUResult({
|
||||
|
@ -140,6 +140,20 @@ export class SkillDomainHelper {
|
||||
return path.join(SKILLS_PATH, domain, skill)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get skill config path
|
||||
* @param domain Domain where the skill belongs
|
||||
* @param skill Skill to get config path from
|
||||
* @param lang Language short code
|
||||
*/
|
||||
public static getSkillConfigPath(
|
||||
domain: SkillDomain['name'],
|
||||
skill: SkillSchema['name'],
|
||||
lang: ShortLanguageCode
|
||||
): string {
|
||||
return path.join(SKILLS_PATH, domain, skill, 'config', `${lang}.json`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get skill config
|
||||
* @param configFilePath Path of the skill config file
|
||||
|
@ -6,25 +6,11 @@
|
||||
"seconds": "seconds"
|
||||
},
|
||||
"actions": {
|
||||
"configure_set_timer": {
|
||||
"type": "dialog",
|
||||
"set_timer": {
|
||||
"type": "logic",
|
||||
"utterance_samples": [
|
||||
"[Set|Start|Create] a timer for @number [minutes|seconds]"
|
||||
],
|
||||
"slots": [
|
||||
{
|
||||
"name": "duration",
|
||||
"item": {
|
||||
"type": "entity",
|
||||
"name": "duration"
|
||||
},
|
||||
"questions": ["For how long time?"]
|
||||
}
|
||||
],
|
||||
"next_action": "set_timer"
|
||||
},
|
||||
"set_timer": {
|
||||
"type": "logic"
|
||||
]
|
||||
},
|
||||
"cancel_timer": {
|
||||
"type": "logic",
|
||||
@ -52,13 +38,14 @@
|
||||
},
|
||||
"answers": {
|
||||
"cannot_get_duration": [
|
||||
"Sorry, I can't get the duration of the timer.",
|
||||
"I can't get the duration of the timer. Sorry."
|
||||
"You should provide a duration for the timer.",
|
||||
"You didn't provide a duration for the timer."
|
||||
],
|
||||
"unit_not_supported": [
|
||||
"Sorry, I can't set a timer for this unit. Use %hours%, %minutes% or %seconds% instead.",
|
||||
"I can't set a timer for this duration. Use %hours%, %minutes% or %seconds% instead."
|
||||
]
|
||||
],
|
||||
"no_timer_set": ["No timer is set.", "There is no timer set."]
|
||||
},
|
||||
"widget_contents": {
|
||||
"second_unit": "second",
|
||||
|
34
skills/utilities/timer/src/actions/check_timer.ts
Normal file
34
skills/utilities/timer/src/actions/check_timer.ts
Normal file
@ -0,0 +1,34 @@
|
||||
import type { ActionFunction } from '@sdk/types'
|
||||
import { leon } from '@sdk/leon'
|
||||
import { getWidgetId } from '@sdk/toolbox'
|
||||
|
||||
import { TimerWidget } from '../widgets/timer-widget'
|
||||
import { getTimerMemoryByWidgetId, getNewestTimerMemory } from '../lib/memory'
|
||||
|
||||
export const run: ActionFunction = async function () {
|
||||
const widgetId = getWidgetId()
|
||||
const timerMemory = widgetId
|
||||
? await getTimerMemoryByWidgetId(widgetId)
|
||||
: await getNewestTimerMemory()
|
||||
|
||||
if (!timerMemory) {
|
||||
return await leon.answer({ key: 'no_timer_set' })
|
||||
}
|
||||
|
||||
const { interval, finishedAt, duration } = timerMemory
|
||||
const remainingTime = finishedAt - Math.floor(Date.now() / 1_000)
|
||||
const initialProgress = 100 - (remainingTime / duration) * 100
|
||||
|
||||
const timerWidget = new TimerWidget({
|
||||
params: {
|
||||
id: widgetId ?? timerMemory.widgetId,
|
||||
seconds: remainingTime <= 0 ? 0 : remainingTime,
|
||||
initialProgress,
|
||||
initialDuration: duration,
|
||||
interval
|
||||
},
|
||||
onFetchAction: 'check_timer'
|
||||
})
|
||||
|
||||
await leon.answer({ widget: timerWidget })
|
||||
}
|
@ -1,14 +1,15 @@
|
||||
import type { ActionFunction, BuiltInDurationEntity } from '@sdk/types'
|
||||
import { leon } from '@sdk/leon'
|
||||
|
||||
import { TimerWidget } from '../widgets/timer'
|
||||
import { TimerWidget } from '../widgets/timer-widget'
|
||||
import { createTimerMemory } from '../lib/memory'
|
||||
|
||||
export const run: ActionFunction = async function (params) {
|
||||
const supportedUnits = ['hours', 'minutes', 'seconds']
|
||||
const durations = (
|
||||
params.slots['duration']?.resolution as BuiltInDurationEntity['resolution']
|
||||
const [duration] = (
|
||||
params.current_entities.find((entity) => entity.type === 'duration')
|
||||
?.resolution as BuiltInDurationEntity['resolution']
|
||||
).values
|
||||
const [duration] = durations
|
||||
|
||||
if (!duration) {
|
||||
return leon.answer({ key: 'cannot_get_duration' })
|
||||
@ -21,12 +22,18 @@ export const run: ActionFunction = async function (params) {
|
||||
|
||||
const { value: durationValue } = duration
|
||||
const seconds = Number(durationValue)
|
||||
const interval = 1_000
|
||||
const timerWidget = new TimerWidget({
|
||||
params: {
|
||||
seconds
|
||||
}
|
||||
seconds,
|
||||
initialProgress: 0,
|
||||
interval
|
||||
},
|
||||
onFetchAction: 'check_timer'
|
||||
})
|
||||
|
||||
await createTimerMemory(timerWidget.id, seconds, interval)
|
||||
|
||||
// TODO: return a speech without new utterance
|
||||
/*await leon.answer({
|
||||
widget: timerWidget,
|
||||
|
51
skills/utilities/timer/src/lib/memory.ts
Normal file
51
skills/utilities/timer/src/lib/memory.ts
Normal file
@ -0,0 +1,51 @@
|
||||
import { Memory } from '@sdk/memory'
|
||||
|
||||
export interface TimerMemory {
|
||||
widgetId: string
|
||||
duration: number
|
||||
interval: number
|
||||
createdAt: number
|
||||
finishedAt: number
|
||||
}
|
||||
|
||||
const TIMERS_MEMORY = new Memory<TimerMemory[]>({
|
||||
name: 'timers',
|
||||
defaultMemory: []
|
||||
})
|
||||
|
||||
export async function createTimerMemory(
|
||||
widgetId: string,
|
||||
duration: number,
|
||||
interval: number
|
||||
): Promise<TimerMemory> {
|
||||
const createdAt = Math.floor(Date.now() / 1_000)
|
||||
const newTimerMemory: TimerMemory = {
|
||||
duration,
|
||||
widgetId,
|
||||
interval,
|
||||
createdAt,
|
||||
finishedAt: createdAt + duration
|
||||
}
|
||||
|
||||
const timersMemory = await TIMERS_MEMORY.read()
|
||||
await TIMERS_MEMORY.write([...timersMemory, newTimerMemory])
|
||||
|
||||
return newTimerMemory
|
||||
}
|
||||
|
||||
export async function getTimerMemoryByWidgetId(
|
||||
widgetId: string
|
||||
): Promise<TimerMemory | null> {
|
||||
const timersMemory = await TIMERS_MEMORY.read()
|
||||
|
||||
return (
|
||||
timersMemory.find((timerMemory) => timerMemory.widgetId === widgetId) ||
|
||||
null
|
||||
)
|
||||
}
|
||||
|
||||
export async function getNewestTimerMemory(): Promise<TimerMemory | null> {
|
||||
const timersMemory = await TIMERS_MEMORY.read()
|
||||
|
||||
return timersMemory[timersMemory.length - 1] || null
|
||||
}
|
@ -1,10 +1,12 @@
|
||||
import type { WidgetEventMethod } from '@sdk/widget'
|
||||
import { WidgetComponent } from '@sdk/widget-component'
|
||||
|
||||
interface TimerProps {
|
||||
initialTime: number
|
||||
initialProgress: number
|
||||
interval: number
|
||||
totalTimeContent: string
|
||||
onEnd?: () => void
|
||||
onEnd?: () => WidgetEventMethod
|
||||
}
|
||||
|
||||
export class Timer extends WidgetComponent<TimerProps> {
|
||||
|
@ -1,35 +1,36 @@
|
||||
import type { WidgetComponent } from '@sdk/widget-component'
|
||||
import { Widget, type WidgetEventMethod, type WidgetOptions } from '@sdk/widget'
|
||||
import { type WidgetComponent } from '@sdk/widget-component'
|
||||
|
||||
import { Timer } from './components/timer'
|
||||
|
||||
interface Params {
|
||||
seconds: number
|
||||
interval: number
|
||||
initialProgress: number
|
||||
initialDuration?: number
|
||||
id?: string
|
||||
}
|
||||
|
||||
export class TimerWidget extends Widget<Params> {
|
||||
constructor(options: WidgetOptions<Params>) {
|
||||
super(options)
|
||||
|
||||
if (options.params.id) {
|
||||
this.id = options.params.id
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO
|
||||
* 1. Save timer + timer id in memory
|
||||
* 2. On rendering, set widget id to timer id
|
||||
* 3. When load feed, need to fetch all timers (onFetch?) as per their timer id. Need a built-in API here
|
||||
* 4. onEnd (or onChange and check if done?), then trigger next action or utterance
|
||||
*/
|
||||
|
||||
public render(): WidgetComponent {
|
||||
const { seconds } = this.params
|
||||
const { seconds, interval, initialDuration, initialProgress } = this.params
|
||||
const secondUnitContent = this.content('second_unit')
|
||||
const secondsUnitContent = this.content('seconds_unit')
|
||||
const minuteUnitContent = this.content('minute_unit')
|
||||
const minutesUnitContent = this.content('minutes_unit')
|
||||
const totalTime = initialDuration || seconds
|
||||
let totalTimeContent = ''
|
||||
|
||||
if (seconds >= 60) {
|
||||
const minutes = seconds / 60
|
||||
if (totalTime >= 60) {
|
||||
const minutes = totalTime / 60
|
||||
|
||||
totalTimeContent = this.content('total_time', {
|
||||
value: minutes % 1 === 0 ? minutes : minutes.toFixed(2),
|
||||
@ -37,14 +38,15 @@ export class TimerWidget extends Widget<Params> {
|
||||
})
|
||||
} else {
|
||||
totalTimeContent = this.content('total_time', {
|
||||
value: seconds,
|
||||
unit: seconds > 1 ? secondsUnitContent : secondUnitContent
|
||||
value: totalTime,
|
||||
unit: totalTime > 1 ? secondsUnitContent : secondUnitContent
|
||||
})
|
||||
}
|
||||
|
||||
return new Timer({
|
||||
initialTime: seconds,
|
||||
interval: 1_000,
|
||||
initialProgress,
|
||||
interval,
|
||||
totalTimeContent,
|
||||
onEnd: (): WidgetEventMethod => {
|
||||
return this.sendUtterance('times_up', {
|
Loading…
Reference in New Issue
Block a user