From cad072c3a631dc4d714658f35809dcd9663f0c10 Mon Sep 17 00:00:00 2001 From: louistiti Date: Wed, 16 Feb 2022 22:28:05 +0800 Subject: [PATCH] refactor: skills over HTTP --- core/pkgs-endpoints.json | 146 ------------------ core/skills-endpoints.json | 12 ++ package.json | 8 +- ...points.js => generate-skills-endpoints.js} | 72 +++++---- .../generate/run-generate-pkgs-endpoints.js | 14 -- .../generate/run-generate-skills-endpoints.js | 14 ++ scripts/setup/setup-python-packages.js | 2 +- server/src/core/http-server/server.js | 2 +- 8 files changed, 75 insertions(+), 195 deletions(-) delete mode 100644 core/pkgs-endpoints.json create mode 100644 core/skills-endpoints.json rename scripts/generate/{generate-pkgs-endpoints.js => generate-skills-endpoints.js} (54%) delete mode 100644 scripts/generate/run-generate-pkgs-endpoints.js create mode 100644 scripts/generate/run-generate-skills-endpoints.js diff --git a/core/pkgs-endpoints.json b/core/pkgs-endpoints.json deleted file mode 100644 index 82353092..00000000 --- a/core/pkgs-endpoints.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "endpoints": [ - { - "method": "POST", - "route": "/api/p/calendar/todolist/create_list", - "params": [ - "list" - ], - "entitiesType": "trim" - }, - { - "method": "GET", - "route": "/api/p/calendar/todolist/view_lists", - "params": [] - }, - { - "method": "POST", - "route": "/api/p/calendar/todolist/view_list", - "params": [ - "list" - ], - "entitiesType": "trim" - }, - { - "method": "POST", - "route": "/api/p/calendar/todolist/rename_list", - "params": [ - "old_list", - "new_list" - ], - "entitiesType": "trim" - }, - { - "method": "POST", - "route": "/api/p/calendar/todolist/delete_list", - "params": [ - "list" - ], - "entitiesType": "trim" - }, - { - "method": "POST", - "route": "/api/p/calendar/todolist/add_todos", - "params": [ - "todos", - "list" - ], - "entitiesType": "trim" - }, - { - "method": "POST", - "route": "/api/p/calendar/todolist/complete_todos", - "params": [ - "todos", - "list" - ], - "entitiesType": "trim" - }, - { - "method": "POST", - "route": "/api/p/calendar/todolist/uncheck_todos", - "params": [ - "todos", - "list" - ], - "entitiesType": "trim" - }, - { - "method": "POST", - "route": "/api/p/checker/isitdown/run", - "params": [ - "url" - ], - "entitiesType": "builtIn" - }, - { - "method": "GET", - "route": "/api/p/checker/haveibeenpwned/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/whoami/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/joke/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/greeting/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/welcome/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/meaningoflife/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/randomnumber/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/bye/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/leon/partnerassistant/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/network/speedtest/run", - "params": [] - }, - { - "method": "POST", - "route": "/api/p/trend/github/run", - "params": [ - "number", - "daterange" - ], - "entitiesType": "builtIn" - }, - { - "method": "GET", - "route": "/api/p/trend/producthunt/run", - "params": [] - }, - { - "method": "GET", - "route": "/api/p/videodownloader/youtube/run", - "params": [] - } - ] -} \ No newline at end of file diff --git a/core/skills-endpoints.json b/core/skills-endpoints.json new file mode 100644 index 00000000..b12694f0 --- /dev/null +++ b/core/skills-endpoints.json @@ -0,0 +1,12 @@ +{ + "endpoints": [ + { + "method": "POST", + "route": "/api/action/productivity/todo_list/create_list", + "params": [ + "list" + ], + "entitiesType": "trim" + } + ] +} \ No newline at end of file diff --git a/package.json b/package.json index 3fd754cb..6832c073 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "test:e2e:modules": "babel-node scripts/run-clean-test-dbs.js && npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && babel-node scripts/run-clean-test-dbs.js && npm run train", "test:e2e:nlp-modules": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train", "test:json": "jest --silent --projects test/json/json.jest.json", - "test:over-http": "npm run generate:pkgs-endpoints && npm run train && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing LEON_HOST=http://localhost LEON_PORT=1338 LEON_HTTP_API_KEY=72aeb5ba324580963114481144385d7179c106fc jest --forceExit --silent --verbose --notify=false --bail --collectCoverage=false test/e2e/over-http.spec.js", + "test:over-http": "npm run generate:skills-endpoints && npm run train && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing LEON_HOST=http://localhost LEON_PORT=1338 LEON_HTTP_API_KEY=72aeb5ba324580963114481144385d7179c106fc jest --forceExit --silent --verbose --notify=false --bail --collectCoverage=false test/e2e/over-http.spec.js", "test:module": "babel-node scripts/test-module.js", "setup:offline": "babel-node scripts/setup-offline/setup-offline.js", "setup:offline-stt": "babel-node scripts/setup-offline/run-setup-stt.js", @@ -37,15 +37,15 @@ "preinstall": "node scripts/setup/preinstall.js", "postinstall": "babel-node scripts/setup/setup.js", "dev:app": "vite --config app/vite.config.js", - "dev:server": "npm run train && npm run generate:pkgs-endpoints && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node", + "dev:server": "npm run train && npm run generate:skills-endpoints && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node", "wake": "cross-env LEON_HOST=http://localhost LEON_PORT=1337 node hotword/index.js", "delete-dist:server": "shx rm -rf ./server/dist", "prepare": "husky install", - "generate:pkgs-endpoints": "babel-node scripts/generate/run-generate-pkgs-endpoints.js", + "generate:skills-endpoints": "babel-node scripts/generate/run-generate-skills-endpoints.js", "generate:http-api-key": "babel-node scripts/generate/run-generate-http-api-key.js", "build": "npm run build:app && npm run build:server", "build:app": "cross-env LEON_NODE_ENV=production babel-node scripts/app/run-build-app.js", - "build:server": "npm run delete-dist:server && npm run train && npm run generate:pkgs-endpoints && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp", + "build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp", "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js", "train": "babel-node scripts/run-train.js", "prepare-release": "babel-node scripts/release/prepare-release.js", diff --git a/scripts/generate/generate-pkgs-endpoints.js b/scripts/generate/generate-skills-endpoints.js similarity index 54% rename from scripts/generate/generate-pkgs-endpoints.js rename to scripts/generate/generate-skills-endpoints.js index 99813fdf..5248d706 100644 --- a/scripts/generate/generate-pkgs-endpoints.js +++ b/scripts/generate/generate-skills-endpoints.js @@ -5,44 +5,55 @@ import path from 'path' import log from '@/helpers/log' import { langs } from '@@/core/langs.json' +import domain from '@/helpers/domain' dotenv.config() /** - * Generate packages endpoints script - * Parse and convert packages configuration into a JSON file understandable by Fastify - * to dynamically generate endpoints so packages can be accessible over HTTP + * Generate skills endpoints script + * Parse and convert skills NLU config into a JSON file understandable by Fastify + * to dynamically generate endpoints so skills can be accessible over HTTP */ export default () => new Promise(async (resolve, reject) => { const supportedMethods = ['DELETE', 'GET', 'HEAD', 'PATCH', 'POST', 'PUT', 'OPTIONS'] - const packagesDir = 'packages' - const outputFile = '/core/pkgs-endpoints.json' + const outputFile = '/core/skills-endpoints.json' const outputFilePath = path.join(__dirname, `../..${outputFile}`) const lang = langs[process.env.LEON_HTTP_API_LANG].short try { - const packages = fs.readdirSync(packagesDir) - .filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory()) + const [domainKeys, domains] = await Promise.all([domain.list(), domain.getDomainsObj()]) const finalObj = { endpoints: [] } let isFileNeedToBeGenerated = true - let pkgObj = { } + let loopIsBroken = false // Check if a new routing generation is necessary if (fs.existsSync(outputFilePath)) { const mtimeEndpoints = fs.statSync(outputFilePath).mtime.getTime() - for (let i = 0; i < packages.length; i += 1) { - const pkg = packages[i] - const fileInfo = fs.statSync(`${packagesDir}/${pkg}/data/expressions/${lang}.json`) - const mtime = fileInfo.mtime.getTime() + for (let i = 0; i < domainKeys.length; i += 1) { + const currentDomain = domains[domainKeys[i]] + const skillKeys = Object.keys(currentDomain.skills) - if (mtime > mtimeEndpoints) { + // Browse skills + for (let j = 0; j < skillKeys.length; j += 1) { + const skillFriendlyName = skillKeys[j] + const currentSkill = currentDomain.skills[skillFriendlyName] + const fileInfo = fs.statSync(path.join(currentSkill.path, 'nlu', `${lang}.json`)) + const mtime = fileInfo.mtime.getTime() + + if (mtime > mtimeEndpoints) { + loopIsBroken = true + break + } + } + + if (loopIsBroken) { break } - if ((i + 1) === packages.length) { + if ((i + 1) === domainKeys.length) { log.success(`${outputFile} is already up-to-date`) isFileNeedToBeGenerated = false } @@ -51,37 +62,40 @@ export default () => new Promise(async (resolve, reject) => { // Force if a language is given if (isFileNeedToBeGenerated) { - log.info('Parsing packages configuration...') + log.info('Parsing skills NLU configuration...') - for (let i = 0; i < packages.length; i += 1) { - const pkg = packages[i] + for (let i = 0; i < domainKeys.length; i += 1) { + const currentDomain = domains[domainKeys[i]] + const skillKeys = Object.keys(currentDomain.skills) - pkgObj = JSON.parse(fs.readFileSync(`${packagesDir}/${pkg}/data/expressions/${lang}.json`, 'utf8')) + // Browse skills + for (let j = 0; j < skillKeys.length; j += 1) { + const skillFriendlyName = skillKeys[j] + const currentSkill = currentDomain.skills[skillFriendlyName] - const modules = Object.keys(pkgObj) - for (let j = 0; j < modules.length; j += 1) { - const module = modules[j] - const actions = Object.keys(pkgObj[module]) + const nluFilePath = path.join(currentSkill.path, 'nlu', `${lang}.json`) + const { actions } = JSON.parse(fs.readFileSync(nluFilePath, 'utf8')) + const actionsKeys = Object.keys(actions) - for (let k = 0; k < actions.length; k += 1) { - const action = actions[k] - const actionObj = pkgObj[module][action] + for (let k = 0; k < actionsKeys.length; k += 1) { + const action = actionsKeys[k] + const actionObj = actions[action] const { entities, http_api } = actionObj // eslint-disable-line camelcase - let finalMethod = entities || http_api?.entities ? 'POST' : 'GET' + let finalMethod = (entities || http_api?.entities) ? 'POST' : 'GET' - // Only generate this route if it is not disabled from the package config + // Only generate this route if it is not disabled from the skill config if (!http_api?.disabled || (http_api?.disabled && http_api?.disabled === false)) { if (http_api?.method) { finalMethod = http_api.method.toUpperCase() } if (!supportedMethods.includes(finalMethod)) { - reject(`The "${finalMethod}" HTTP method of the ${pkg}/${module}/${action} action is not supported`) + reject(`The "${finalMethod}" HTTP method of the ${currentDomain.name}/${currentSkill.name}/${action} action is not supported`) } const endpoint = { method: finalMethod.toUpperCase(), - route: `/api/p/${pkg}/${module}/${action}`, + route: `/api/action/${currentDomain.name}/${currentSkill.name}/${action}`, params: [] } diff --git a/scripts/generate/run-generate-pkgs-endpoints.js b/scripts/generate/run-generate-pkgs-endpoints.js deleted file mode 100644 index 0aa2ccca..00000000 --- a/scripts/generate/run-generate-pkgs-endpoints.js +++ /dev/null @@ -1,14 +0,0 @@ -import log from '@/helpers/log' - -import generatePkgsEndpoints from './generate-pkgs-endpoints' - -/** - * Execute the generating packages endpoints script - */ -(async () => { - try { - await generatePkgsEndpoints() - } catch (e) { - log.error(`Failed to generate packages endpoints: ${e}`) - } -})() diff --git a/scripts/generate/run-generate-skills-endpoints.js b/scripts/generate/run-generate-skills-endpoints.js new file mode 100644 index 00000000..b4c35ddb --- /dev/null +++ b/scripts/generate/run-generate-skills-endpoints.js @@ -0,0 +1,14 @@ +import log from '@/helpers/log' + +import generateSkillsEndpoints from './generate-skills-endpoints' + +/** + * Execute the generating skills endpoints script + */ +(async () => { + try { + await generateSkillsEndpoints() + } catch (e) { + log.error(`Failed to generate skills endpoints: ${e}`) + } +})() diff --git a/scripts/setup/setup-python-packages.js b/scripts/setup/setup-python-packages.js index eeaf68a5..46b38ef7 100644 --- a/scripts/setup/setup-python-packages.js +++ b/scripts/setup/setup-python-packages.js @@ -5,7 +5,7 @@ import path from 'path' import log from '@/helpers/log' /** - * Download and setup Leon's packages Python dependencies + * Download and setup Leon's Python packages dependencies */ export default () => new Promise(async (resolve, reject) => { log.info('Checking Python env...') diff --git a/server/src/core/http-server/server.js b/server/src/core/http-server/server.js index d289639a..41429220 100644 --- a/server/src/core/http-server/server.js +++ b/server/src/core/http-server/server.js @@ -4,7 +4,7 @@ import socketio from 'socket.io' import { join } from 'path' import { version } from '@@/package.json' -import { endpoints } from '@@/core/pkgs-endpoints.json' +import { endpoints } from '@@/core/skills-endpoints.json' import Nlu from '@/core/nlu' import Brain from '@/core/brain' import Asr from '@/core/asr'