refactor: skills over HTTP

2024-12-24 01:04:28 +03:00 · 2022-02-16 22:28:05 +08:00 · 2022-02-16 22:28:05 +08:00 · cad072c3a6
commit cad072c3a6
parent 963e4de8df
8 changed files with 75 additions and 195 deletions
--- a/core/pkgs-endpoints.json
+++ b/core/pkgs-endpoints.json
@ -1,146 +0,0 @@
-{
-  "endpoints": [
-    {
-      "method": "POST",
-      "route": "/api/p/calendar/todolist/create_list",
-      "params": [
-        "list"
-      ],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/calendar/todolist/view_lists",
-      "params": []
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/calendar/todolist/view_list",
-      "params": [
-        "list"
-      ],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/calendar/todolist/rename_list",
-      "params": [
-        "old_list",
-        "new_list"
-      ],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/calendar/todolist/delete_list",
-      "params": [
-        "list"
-      ],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/calendar/todolist/add_todos",
-      "params": [
-        "todos",
-        "list"
-      ],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/calendar/todolist/complete_todos",
-      "params": [
-        "todos",
-        "list"
-      ],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/calendar/todolist/uncheck_todos",
-      "params": [
-        "todos",
-        "list"
-      ],
-      "entitiesType": "trim"
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/checker/isitdown/run",
-      "params": [
-        "url"
-      ],
-      "entitiesType": "builtIn"
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/checker/haveibeenpwned/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/whoami/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/joke/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/greeting/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/welcome/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/meaningoflife/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/randomnumber/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/bye/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/leon/partnerassistant/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/network/speedtest/run",
-      "params": []
-    },
-    {
-      "method": "POST",
-      "route": "/api/p/trend/github/run",
-      "params": [
-        "number",
-        "daterange"
-      ],
-      "entitiesType": "builtIn"
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/trend/producthunt/run",
-      "params": []
-    },
-    {
-      "method": "GET",
-      "route": "/api/p/videodownloader/youtube/run",
-      "params": []
-    }
-  ]
-}
--- a/core/skills-endpoints.json
+++ b/core/skills-endpoints.json
@ -0,0 +1,12 @@
+{
+  "endpoints": [
+    {
+      "method": "POST",
+      "route": "/api/action/productivity/todo_list/create_list",
+      "params": [
+        "list"
+      ],
+      "entitiesType": "trim"
+    }
+  ]
+}
--- a/package.json
+++ b/package.json
@ -28,7 +28,7 @@
    "test:e2e:modules": "babel-node scripts/run-clean-test-dbs.js && npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && babel-node scripts/run-clean-test-dbs.js && npm run train",
    "test:e2e:nlp-modules": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train",
    "test:json": "jest --silent --projects test/json/json.jest.json",
-    "test:over-http": "npm run generate:pkgs-endpoints && npm run train && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing LEON_HOST=http://localhost LEON_PORT=1338 LEON_HTTP_API_KEY=72aeb5ba324580963114481144385d7179c106fc jest --forceExit --silent --verbose --notify=false --bail --collectCoverage=false test/e2e/over-http.spec.js",
+    "test:over-http": "npm run generate:skills-endpoints && npm run train && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing LEON_HOST=http://localhost LEON_PORT=1338 LEON_HTTP_API_KEY=72aeb5ba324580963114481144385d7179c106fc jest --forceExit --silent --verbose --notify=false --bail --collectCoverage=false test/e2e/over-http.spec.js",
    "test:module": "babel-node scripts/test-module.js",
    "setup:offline": "babel-node scripts/setup-offline/setup-offline.js",
    "setup:offline-stt": "babel-node scripts/setup-offline/run-setup-stt.js",
@ -37,15 +37,15 @@
    "preinstall": "node scripts/setup/preinstall.js",
    "postinstall": "babel-node scripts/setup/setup.js",
    "dev:app": "vite --config app/vite.config.js",
-    "dev:server": "npm run train && npm run generate:pkgs-endpoints && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node",
+    "dev:server": "npm run train && npm run generate:skills-endpoints && cross-env LEON_NODE_ENV=development nodemon --watch server ./server/src/index.js --ignore server/src/tmp/ --exec babel-node",
    "wake": "cross-env LEON_HOST=http://localhost LEON_PORT=1337 node hotword/index.js",
    "delete-dist:server": "shx rm -rf ./server/dist",
    "prepare": "husky install",
-    "generate:pkgs-endpoints": "babel-node scripts/generate/run-generate-pkgs-endpoints.js",
+    "generate:skills-endpoints": "babel-node scripts/generate/run-generate-skills-endpoints.js",
    "generate:http-api-key": "babel-node scripts/generate/run-generate-http-api-key.js",
    "build": "npm run build:app && npm run build:server",
    "build:app": "cross-env LEON_NODE_ENV=production babel-node scripts/app/run-build-app.js",
-    "build:server": "npm run delete-dist:server && npm run train && npm run generate:pkgs-endpoints && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp",
+    "build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp",
    "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
    "train": "babel-node scripts/run-train.js",
    "prepare-release": "babel-node scripts/release/prepare-release.js",
--- a/scripts/generate/generate-skills-endpoints.js
+++ b/scripts/generate/generate-skills-endpoints.js
@ -5,44 +5,55 @@ import path from 'path'
 import log from '@/helpers/log'

 import { langs } from '@@/core/langs.json'
+import domain from '@/helpers/domain'

 dotenv.config()

 /**
- * Generate packages endpoints script
- * Parse and convert packages configuration into a JSON file understandable by Fastify
- * to dynamically generate endpoints so packages can be accessible over HTTP
+ * Generate skills endpoints script
+ * Parse and convert skills NLU config into a JSON file understandable by Fastify
+ * to dynamically generate endpoints so skills can be accessible over HTTP
 */
 export default () => new Promise(async (resolve, reject) => {
  const supportedMethods = ['DELETE', 'GET', 'HEAD', 'PATCH', 'POST', 'PUT', 'OPTIONS']
-  const packagesDir = 'packages'
-  const outputFile = '/core/pkgs-endpoints.json'
+  const outputFile = '/core/skills-endpoints.json'
  const outputFilePath = path.join(__dirname, `../..${outputFile}`)
  const lang = langs[process.env.LEON_HTTP_API_LANG].short

  try {
-    const packages = fs.readdirSync(packagesDir)
-      .filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory())
+    const [domainKeys, domains] = await Promise.all([domain.list(), domain.getDomainsObj()])
    const finalObj = {
      endpoints: []
    }
    let isFileNeedToBeGenerated = true
-    let pkgObj = { }
+    let loopIsBroken = false

    // Check if a new routing generation is necessary
    if (fs.existsSync(outputFilePath)) {
      const mtimeEndpoints = fs.statSync(outputFilePath).mtime.getTime()

-      for (let i = 0; i < packages.length; i += 1) {
-        const pkg = packages[i]
-        const fileInfo = fs.statSync(`${packagesDir}/${pkg}/data/expressions/${lang}.json`)
-        const mtime = fileInfo.mtime.getTime()
+      for (let i = 0; i < domainKeys.length; i += 1) {
+        const currentDomain = domains[domainKeys[i]]
+        const skillKeys = Object.keys(currentDomain.skills)

-        if (mtime > mtimeEndpoints) {
+        // Browse skills
+        for (let j = 0; j < skillKeys.length; j += 1) {
+          const skillFriendlyName = skillKeys[j]
+          const currentSkill = currentDomain.skills[skillFriendlyName]
+          const fileInfo = fs.statSync(path.join(currentSkill.path, 'nlu', `${lang}.json`))
+          const mtime = fileInfo.mtime.getTime()
+
+          if (mtime > mtimeEndpoints) {
+            loopIsBroken = true
+            break
+          }
+        }
+
+        if (loopIsBroken) {
          break
        }

-        if ((i + 1) === packages.length) {
+        if ((i + 1) === domainKeys.length) {
          log.success(`${outputFile} is already up-to-date`)
          isFileNeedToBeGenerated = false
        }
@ -51,37 +62,40 @@ export default () => new Promise(async (resolve, reject) => {

    // Force if a language is given
    if (isFileNeedToBeGenerated) {
-      log.info('Parsing packages configuration...')
+      log.info('Parsing skills NLU configuration...')

-      for (let i = 0; i < packages.length; i += 1) {
-        const pkg = packages[i]
+      for (let i = 0; i < domainKeys.length; i += 1) {
+        const currentDomain = domains[domainKeys[i]]
+        const skillKeys = Object.keys(currentDomain.skills)

-        pkgObj = JSON.parse(fs.readFileSync(`${packagesDir}/${pkg}/data/expressions/${lang}.json`, 'utf8'))
+        // Browse skills
+        for (let j = 0; j < skillKeys.length; j += 1) {
+          const skillFriendlyName = skillKeys[j]
+          const currentSkill = currentDomain.skills[skillFriendlyName]

-        const modules = Object.keys(pkgObj)
-        for (let j = 0; j < modules.length; j += 1) {
-          const module = modules[j]
-          const actions = Object.keys(pkgObj[module])
+          const nluFilePath = path.join(currentSkill.path, 'nlu', `${lang}.json`)
+          const { actions } = JSON.parse(fs.readFileSync(nluFilePath, 'utf8'))
+          const actionsKeys = Object.keys(actions)

-          for (let k = 0; k < actions.length; k += 1) {
-            const action = actions[k]
-            const actionObj = pkgObj[module][action]
+          for (let k = 0; k < actionsKeys.length; k += 1) {
+            const action = actionsKeys[k]
+            const actionObj = actions[action]
            const { entities, http_api } = actionObj // eslint-disable-line camelcase
-            let finalMethod = entities || http_api?.entities ? 'POST' : 'GET'
+            let finalMethod = (entities || http_api?.entities) ? 'POST' : 'GET'

-            // Only generate this route if it is not disabled from the package config
+            // Only generate this route if it is not disabled from the skill config
            if (!http_api?.disabled || (http_api?.disabled && http_api?.disabled === false)) {
              if (http_api?.method) {
                finalMethod = http_api.method.toUpperCase()
              }

              if (!supportedMethods.includes(finalMethod)) {
-                reject(`The "${finalMethod}" HTTP method of the ${pkg}/${module}/${action} action is not supported`)
+                reject(`The "${finalMethod}" HTTP method of the ${currentDomain.name}/${currentSkill.name}/${action} action is not supported`)
              }

              const endpoint = {
                method: finalMethod.toUpperCase(),
-                route: `/api/p/${pkg}/${module}/${action}`,
+                route: `/api/action/${currentDomain.name}/${currentSkill.name}/${action}`,
                params: []
              }

--- a/scripts/generate/run-generate-pkgs-endpoints.js
+++ b/scripts/generate/run-generate-pkgs-endpoints.js
@ -1,14 +0,0 @@
-import log from '@/helpers/log'
-
-import generatePkgsEndpoints from './generate-pkgs-endpoints'
-
-/**
- * Execute the generating packages endpoints script
- */
-(async () => {
-  try {
-    await generatePkgsEndpoints()
-  } catch (e) {
-    log.error(`Failed to generate packages endpoints: ${e}`)
-  }
-})()
--- a/scripts/generate/run-generate-skills-endpoints.js
+++ b/scripts/generate/run-generate-skills-endpoints.js
@ -0,0 +1,14 @@
+import log from '@/helpers/log'
+
+import generateSkillsEndpoints from './generate-skills-endpoints'
+
+/**
+ * Execute the generating skills endpoints script
+ */
+(async () => {
+  try {
+    await generateSkillsEndpoints()
+  } catch (e) {
+    log.error(`Failed to generate skills endpoints: ${e}`)
+  }
+})()
--- a/scripts/setup/setup-python-packages.js
+++ b/scripts/setup/setup-python-packages.js
@ -5,7 +5,7 @@ import path from 'path'
 import log from '@/helpers/log'

 /**
- * Download and setup Leon's packages Python dependencies
+ * Download and setup Leon's Python packages dependencies
 */
 export default () => new Promise(async (resolve, reject) => {
  log.info('Checking Python env...')
--- a/server/src/core/http-server/server.js
+++ b/server/src/core/http-server/server.js
@ -4,7 +4,7 @@ import socketio from 'socket.io'
 import { join } from 'path'

 import { version } from '@@/package.json'
-import { endpoints } from '@@/core/pkgs-endpoints.json'
+import { endpoints } from '@@/core/skills-endpoints.json'
 import Nlu from '@/core/nlu'
 import Brain from '@/core/brain'
 import Asr from '@/core/asr'