From b775ec4e9dcc841cc6a7b66b15f778d4644b8194 Mon Sep 17 00:00:00 2001 From: louistiti Date: Thu, 17 Feb 2022 20:10:42 +0800 Subject: [PATCH] refactor: continue from modules to skills transition --- .env.sample | 2 +- app/src/js/client.js | 2 +- scripts/clean-test-dbs.js | 35 +++++---- scripts/setup/setup-skills-config.js | 4 +- .../src/core/http-server/api/downloads/get.js | 44 +++++------ .../core/http-server/api/downloads/index.js | 2 +- server/src/core/http-server/server.js | 26 +++---- server/src/core/nlu.js | 2 +- server/src/intent-object.sample.json | 4 +- skills/productivity/todo_list/nlu/en.json | 12 ++- .../todo_list/src/actions/create_list.py | 2 +- .../todo_list/src/actions/view_list.py | 73 +++++++++++++++++++ 12 files changed, 146 insertions(+), 62 deletions(-) create mode 100644 skills/productivity/todo_list/src/actions/view_list.py diff --git a/.env.sample b/.env.sample index 9260be62..5c6f198a 100644 --- a/.env.sample +++ b/.env.sample @@ -41,7 +41,7 @@ PIPENV_PIPFILE=bridges/python/Pipfile PIPENV_VENV_IN_PROJECT=true # Fix https://click.palletsprojects.com/en/7.x/python3/#python-3-surrogate-handling -# If Leon replies you something like "Sorry, it seems I have a problem with the ... module" but +# If Leon replies you something like "Sorry, it seems I have a problem with the ... skill" but # still gives you the right answer, then: ## 1. Run `locale -a` ## 2. Pick a locale diff --git a/app/src/js/client.js b/app/src/js/client.js index cd57d206..aa969025 100644 --- a/app/src/js/client.js +++ b/app/src/js/client.js @@ -96,7 +96,7 @@ export default class Client { }) this.socket.on('download', (data) => { - window.location = `${this.serverUrl}/api/v1/downloads?domain=${data.package}&skill=${data.module}` + window.location = `${this.serverUrl}/api/v1/downloads?domain=${data.domain}&skill=${data.skill}` }) if (this.history !== null) { diff --git a/scripts/clean-test-dbs.js b/scripts/clean-test-dbs.js index 61c96785..634ded87 100644 --- a/scripts/clean-test-dbs.js +++ b/scripts/clean-test-dbs.js @@ -2,6 +2,7 @@ import fs from 'fs' import { join } from 'path' import log from '@/helpers/log' +import domain from '@/helpers/domain' /** * This script delete test DB files if they exist @@ -9,23 +10,29 @@ import log from '@/helpers/log' export default () => new Promise(async (resolve, reject) => { log.info('Cleaning test DB files...') - const packagesFolder = join(__dirname, '../packages') - const packages = fs.readdirSync(packagesFolder) - .filter((entity) => fs.statSync(join(packagesFolder, entity)).isDirectory()) + const [domainKeys, domains] = await Promise.all([domain.list(), domain.getDomainsObj()]) - for (let i = 0; i < packages.length; i += 1) { - try { - const dbFolder = join(packagesFolder, packages[i], 'data/db') - const dbTestFiles = fs.readdirSync(dbFolder).filter((entity) => entity.indexOf('.spec.json') !== -1) + for (let i = 0; i < domainKeys.length; i += 1) { + const currentDomain = domains[domainKeys[i]] + const skillKeys = Object.keys(currentDomain.skills) - if (dbTestFiles.length > 0) { - log.info(`Deleting ${dbTestFiles[0]}...`) - fs.unlinkSync(join(dbFolder, dbTestFiles[0])) - log.success(`${dbTestFiles[0]} deleted`) + for (let j = 0; j < skillKeys.length; j += 1) { + const currentSkill = currentDomain.skills[skillKeys[j]] + + try { + // TODO: handle case where the memory folder contain multiple DB nodes + const dbFolder = join(currentSkill.path, 'memory') + const dbTestFiles = fs.readdirSync(dbFolder).filter((entity) => entity.indexOf('.spec.json') !== -1) + + if (dbTestFiles.length > 0) { + log.info(`Deleting ${dbTestFiles[0]}...`) + fs.unlinkSync(join(dbFolder, dbTestFiles[0])) + log.success(`${dbTestFiles[0]} deleted`) + } + } catch (e) { + log.error(`Failed to clean: "${skillKeys[j]}" test DB file`) + reject(e) } - } catch (e) { - log.error(`Failed to clean: ${packages[i]} test DB file`) - reject(e) } } diff --git a/scripts/setup/setup-skills-config.js b/scripts/setup/setup-skills-config.js index 525adf2c..7ea81052 100644 --- a/scripts/setup/setup-skills-config.js +++ b/scripts/setup/setup-skills-config.js @@ -6,7 +6,7 @@ import log from '@/helpers/log' import domain from '@/helpers/domain' /** - * Setup packages configuration + * Setup skills configuration */ export default () => new Promise(async (resolve, reject) => { log.info('Setting up skills configuration...') @@ -46,7 +46,7 @@ export default () => new Promise(async (resolve, reject) => { } try { - // Add new module configuration in the config.json file + // Add new skill configuration in the config.json file commandSync(`json -I -f ${configFile} -e 'this.configurations.${configSampleKeys[j]}=${JSON.stringify(configKey[configSampleKeys[j]])}'`, { shell: true }) log.success(`"${configSampleKeys[j]}" configuration key added to ${configFile}`) } catch (e) { diff --git a/server/src/core/http-server/api/downloads/get.js b/server/src/core/http-server/api/downloads/get.js index 89765f51..b5d95006 100644 --- a/server/src/core/http-server/api/downloads/get.js +++ b/server/src/core/http-server/api/downloads/get.js @@ -1,4 +1,5 @@ import fs from 'fs' +import path from 'path' import archiver from 'archiver' import log from '@/helpers/log' @@ -9,7 +10,7 @@ const getDownloads = async (fastify, options) => { log.title('GET /downloads') const clean = (dir, files) => { - log.info('Cleaning module download directory...') + log.info('Cleaning skill download directory...') for (let i = 0; i < files.length; i += 1) { fs.unlinkSync(`${dir}/${files[i]}`) } @@ -18,29 +19,28 @@ const getDownloads = async (fastify, options) => { } let message = '' - if (request.query.package && request.query.module) { - const packageDir = `${__dirname}/../../../../packages/${request.query.package}` - const dlPackageDir = `${__dirname}/../../../../downloads/${request.query.package}` - const module = `${packageDir}/${request.query.module}.py` + if (request.query.domain && request.query.skill) { + const dlDomainDir = path.join(process.cwd(), 'downloads', request.query.domain) + const skill = path.join(dlDomainDir, `${request.query.skill}.py`) log.info( `Checking existence of the ${string.ucfirst( - request.query.module - )} module...` + request.query.skill + )} skill...` ) - if (fs.existsSync(module)) { - log.success(`${string.ucfirst(request.query.module)} module exists`) - const downloadsDir = `${dlPackageDir}/${request.query.module}` + if (fs.existsSync(skill)) { + log.success(`${string.ucfirst(request.query.skill)} skill exists`) + const downloadsDir = `${dlDomainDir}/${request.query.skill}` log.info('Reading downloads directory...') fs.readdir(downloadsDir, (err, files) => { if (err && err.code === 'ENOENT') { - message = 'There is no content to download for this module.' + message = 'There is no content to download for this skill.' log.error(message) reply.code(404).send({ success: false, status: 404, - code: 'module_dir_not_found', + code: 'skill_dir_not_found', message }) } else { @@ -54,22 +54,22 @@ const getDownloads = async (fastify, options) => { clean(downloadsDir, files) } else { log.info('Deleting previous archives...') - const zipSlug = `leon-${request.query.package}-${request.query.module}` - const pkgFiles = fs.readdirSync(dlPackageDir) + const zipSlug = `leon-${request.query.domain}-${request.query.skill}` + const domainsFiles = fs.readdirSync(dlDomainDir) - for (let i = 0; i < pkgFiles.length; i += 1) { + for (let i = 0; i < domainsFiles.length; i += 1) { if ( - pkgFiles[i].indexOf('.zip') !== -1 - && pkgFiles[i].indexOf(zipSlug) !== -1 + domainsFiles[i].indexOf('.zip') !== -1 + && domainsFiles[i].indexOf(zipSlug) !== -1 ) { - fs.unlinkSync(`${dlPackageDir}/${pkgFiles[i]}`) - log.success(`${pkgFiles[i]} archive deleted`) + fs.unlinkSync(`${dlDomainDir}/${domainsFiles[i]}`) + log.success(`${domainsFiles[i]} archive deleted`) } } log.info('Preparing new archive...') const zipName = `${zipSlug}-${Date.now()}.zip` - const zipFile = `${dlPackageDir}/${zipName}` + const zipFile = `${dlDomainDir}/${zipName}` const output = fs.createWriteStream(zipFile) const archive = archiver('zip', { zlib: { level: 9 } }) @@ -102,12 +102,12 @@ const getDownloads = async (fastify, options) => { } }) } else { - message = 'This module does not exist.' + message = 'This skill does not exist.' log.error(message) reply.code(404).send({ success: false, status: 404, - code: 'module_not_found', + code: 'skill_not_found', message }) } diff --git a/server/src/core/http-server/api/downloads/index.js b/server/src/core/http-server/api/downloads/index.js index 6e2ddde9..7c389c77 100644 --- a/server/src/core/http-server/api/downloads/index.js +++ b/server/src/core/http-server/api/downloads/index.js @@ -1,7 +1,7 @@ import getDownloads from '@/core/http-server/api/downloads/get' const downloadsPlugin = async (fastify, options) => { - // Get downloads to download module content + // Get downloads to download skill content fastify.register(getDownloads, options) } diff --git a/server/src/core/http-server/server.js b/server/src/core/http-server/server.js index 41429220..a84e8b1b 100644 --- a/server/src/core/http-server/server.js +++ b/server/src/core/http-server/server.js @@ -26,18 +26,18 @@ server.fastify = Fastify() server.httpServer = { } /** - * Generate packages routes + * Generate skills routes */ /* istanbul ignore next */ -server.generatePackagesRoutes = (instance) => { - // Dynamically expose Leon modules over HTTP +server.generateSkillsRoutes = (instance) => { + // Dynamically expose Leon skills over HTTP endpoints.forEach((endpoint) => { instance.route({ method: endpoint.method, url: endpoint.route, async handler (request, reply) { const timeout = endpoint.timeout || 60000 - const [, , , pkg, module, action] = endpoint.route.split('/') + const [, , , domain, skill, action] = endpoint.route.split('/') const handleRoute = async () => { const { params } = endpoint const entities = [] @@ -74,15 +74,15 @@ server.generatePackagesRoutes = (instance) => { utterance: '', entities, classification: { - package: pkg, - module, + domain, + skill, action, confidence: 1 } } const responseData = { - package: pkg, - module, + domain, + skill, action, speeches: [] } @@ -111,8 +111,8 @@ server.generatePackagesRoutes = (instance) => { setTimeout(() => { reply.statusCode = 408 reply.send({ - package: pkg, - module, + domain, + skill, action, message: 'The action has timed out', timeout, @@ -213,7 +213,7 @@ server.bootstrap = async () => { // Render the web app server.fastify.register(fastifyStatic, { - root: join(__dirname, '../../../../app/dist'), + root: join(process.cwd(), 'app/dist'), prefix: '/' }) server.fastify.get('/', (request, reply) => { @@ -246,7 +246,7 @@ server.bootstrap = async () => { } }) - server.generatePackagesRoutes(instance) + server.generateSkillsRoutes(instance) next() }) @@ -280,7 +280,7 @@ server.init = async () => { brain = new Brain() nlu = new Nlu(brain) - // Train modules utterance samples + // Load NLP model try { await nlu.loadModel(join(__dirname, '../../data/leon-model.nlp')) } catch (e) { diff --git a/server/src/core/nlu.js b/server/src/core/nlu.js index 05386ea4..76ff481a 100644 --- a/server/src/core/nlu.js +++ b/server/src/core/nlu.js @@ -191,7 +191,7 @@ class Nlu { /** * Pickup and compare the right fallback - * according to the wished module + * according to the wished skill action */ static fallback (obj, fallbacks) { const words = obj.utterance.toLowerCase().split(' ') diff --git a/server/src/intent-object.sample.json b/server/src/intent-object.sample.json index 91e8e99a..8217badf 100644 --- a/server/src/intent-object.sample.json +++ b/server/src/intent-object.sample.json @@ -1,7 +1,7 @@ { "lang": "en", - "package": "checker", - "module": "isitdown", + "domain": "checker", + "skill": "isitdown", "action": "run", "utterance": "Check if github.com, mozilla.org and twitter.com are up", "entities": [ diff --git a/skills/productivity/todo_list/nlu/en.json b/skills/productivity/todo_list/nlu/en.json index 9a8b7662..82ce7c42 100644 --- a/skills/productivity/todo_list/nlu/en.json +++ b/skills/productivity/todo_list/nlu/en.json @@ -17,10 +17,14 @@ } ] } - ], - "answers": [ - "Alright, I've created the \"%list%\" list.", - "Done, I created your \"%list%\" list." + ] + }, + "view_lists": { + "utterance_samples": [ + "Show all the lists", + "Show all my lists", + "What are the lists?", + "What are my lists?" ] } }, diff --git a/skills/productivity/todo_list/src/actions/create_list.py b/skills/productivity/todo_list/src/actions/create_list.py index baba8002..f612e15c 100644 --- a/skills/productivity/todo_list/src/actions/create_list.py +++ b/skills/productivity/todo_list/src/actions/create_list.py @@ -9,7 +9,7 @@ from ..lib.db import db_create_list # Skill database db = utils.db()['db'] -# Lists of the module table +# Todo lists table db_lists = db.table('todo_lists') # Query diff --git a/skills/productivity/todo_list/src/actions/view_list.py b/skills/productivity/todo_list/src/actions/view_list.py new file mode 100644 index 00000000..ee612ab2 --- /dev/null +++ b/skills/productivity/todo_list/src/actions/view_list.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- + +from time import time + +import utils +from ..lib.db import db_create_list + +# Skill database +db = utils.db()['db'] + +# Todo lists table +db_lists = db.table('todo_lists') + +# Query +Query = utils.db()['query']() + +def view_list(string, entities): + """View a to-do list""" + + # List name + list_name = '' + + # Find entities + for item in entities: + if item['entity'] == 'list': + list_name = item['sourceText'].lower() + + # Verify if the list exists + if db_lists.count(Query.name == list_name) == 0: + return utils.output('end', 'list_does_not_exist', utils.translate('list_does_not_exist', { 'list': list_name })) + + # Grab todos of the list + todos = db_todos.search(Query.list == list_name) + + if len(todos) == 0: + return utils.output('end', 'empty_list', utils.translate('empty_list', { 'list': list_name })) + + unchecked_todos = db_todos.search((Query.list == list_name) & (Query.is_completed == False)) + completed_todos = db_todos.search((Query.list == list_name) & (Query.is_completed == True)) + + result_unchecked_todos = '' + result_completed_todos = '' + + if len(unchecked_todos) == 0: + utils.output('inter', 'no_unchecked_todo', utils.translate('no_unchecked_todo', { 'list': list_name })) + else: + for todo in unchecked_todos: + result_unchecked_todos += utils.translate('list_todo_element', { + 'todo': todo['name'] + }) + + utils.output('inter', 'unchecked_todos_listed', utils.translate('unchecked_todos_listed', { + 'list': list_name, + 'result': result_unchecked_todos + } + ) + ) + + if len(completed_todos) == 0: + return utils.output('end', 'no_completed_todo', utils.translate('no_completed_todo', { 'list': list_name })) + + for todo in completed_todos: + result_completed_todos += utils.translate('list_completed_todo_element', { + 'todo': todo['name'] + }) + + return utils.output('end', 'completed_todos_listed', utils.translate('completed_todos_listed', { + 'list': list_name, + 'result': result_completed_todos + } + ) + )