From 2e66be25ef0d53b9293453e7f203e0b2794a05e4 Mon Sep 17 00:00:00 2001
From: louistiti <louis.grenard@gmail.com>
Date: Sun, 26 Jun 2022 12:01:43 +0800
Subject: [PATCH] refactor: train global entities at a higher level

---
 .../color.json}                               |  0
 .../handsign.json}                            |  0
 .../level.json}                               |  0
 .../color.json}                               |  0
 .../level.json}                               |  0
 scripts/train.js                              | 52 +++++++++----------
 skills/games/rochambeau/nlu/en.json           |  2 +-
 skills/leon/color/nlu/en.json                 |  2 +-
 8 files changed, 27 insertions(+), 29 deletions(-)
 rename core/data/en/{entities/colors.json => global-entities/color.json} (100%)
 rename core/data/en/{entities/handsigns.json => global-entities/handsign.json} (100%)
 rename core/data/en/{entities/levels.json => global-entities/level.json} (100%)
 rename core/data/fr/{entities/colors.json => global-entities/color.json} (100%)
 rename core/data/fr/{entities/levels.json => global-entities/level.json} (100%)

diff --git a/core/data/en/entities/colors.json b/core/data/en/global-entities/color.json
similarity index 100%
rename from core/data/en/entities/colors.json
rename to core/data/en/global-entities/color.json
diff --git a/core/data/en/entities/handsigns.json b/core/data/en/global-entities/handsign.json
similarity index 100%
rename from core/data/en/entities/handsigns.json
rename to core/data/en/global-entities/handsign.json
diff --git a/core/data/en/entities/levels.json b/core/data/en/global-entities/level.json
similarity index 100%
rename from core/data/en/entities/levels.json
rename to core/data/en/global-entities/level.json
diff --git a/core/data/fr/entities/colors.json b/core/data/fr/global-entities/color.json
similarity index 100%
rename from core/data/fr/entities/colors.json
rename to core/data/fr/global-entities/color.json
diff --git a/core/data/fr/entities/levels.json b/core/data/fr/global-entities/level.json
similarity index 100%
rename from core/data/fr/entities/levels.json
rename to core/data/fr/global-entities/level.json
diff --git a/scripts/train.js b/scripts/train.js
index 495931dd..eb53fbac 100644
--- a/scripts/train.js
+++ b/scripts/train.js
@@ -46,11 +46,36 @@ export default () => new Promise(async (resolve, reject) => {
 
     for (let h = 0; h < shortLangs.length; h += 1) {
       const lang = shortLangs[h]
+      const globalEntitiesPath = path.join(process.cwd(), 'core/data', lang, 'global-entities')
+      const globalEntityFiles = fs.readdirSync(globalEntitiesPath)
       const resolversPath = path.join(process.cwd(), 'core/data', lang, 'resolvers')
       const resolverFiles = fs.readdirSync(resolversPath)
+      const newEntitiesObj = { }
 
       nlp.addLanguage(lang)
 
+      // Add global entities annotations (@...)
+      for (let i = 0; i < globalEntityFiles.length; i += 1) {
+        const globalEntityFileName = globalEntityFiles[i]
+        const [entityName] = globalEntityFileName.split('.')
+        const globalEntityPath = path.join(globalEntitiesPath, globalEntityFileName)
+        const { options } = JSON.parse(fs.readFileSync(globalEntityPath, 'utf8'))
+        const optionKeys = Object.keys(options)
+        const optionsObj = { }
+
+        optionKeys.forEach((optionKey) => {
+          const { synonyms } = options[optionKey]
+
+          optionsObj[optionKey] = synonyms
+        })
+
+        newEntitiesObj[entityName] = { options: optionsObj }
+      }
+
+      console.log('newEntitiesObj1', newEntitiesObj)
+
+      nlp.addEntities(newEntitiesObj, lang)
+
       // Train resolvers
       for (let i = 0; i < resolverFiles.length; i += 1) {
         const resolverFileName = resolverFiles[i]
@@ -92,7 +117,6 @@ export default () => new Promise(async (resolve, reject) => {
           if (fs.existsSync(nluFilePath)) {
             const {
               actions,
-              entities,
               variables
             } = await json.loadNluData(nluFilePath, lang) // eslint-disable-line no-await-in-loop
             const actionsKeys = Object.keys(actions)
@@ -157,32 +181,6 @@ export default () => new Promise(async (resolve, reject) => {
                   nlp.addAnswer(lang, `${skillName}.${actionName}`, answers[l])
                 }
               }
-
-              /**
-               * TODO: load common entities not per action but globally?
-               * TODO: as these entities are exposed to all actions
-               */
-              // Add entities annotations (@...)
-              if (entities) {
-                const newEntitiesObj = { }
-                const entityKeys = Object.keys(entities)
-
-                for (let l = 0; l < entityKeys.length; l += 1) {
-                  const entity = entities[entityKeys[l]]
-                  const optionKeys = Object.keys(entity.options)
-                  const options = { }
-
-                  for (let m = 0; m < optionKeys.length; m += 1) {
-                    const option = entity.options[optionKeys[m]]
-
-                    options[optionKeys[m]] = option.synonyms
-                  }
-
-                  newEntitiesObj[entityKeys[l]] = { options }
-                }
-
-                nlp.addEntities(newEntitiesObj, lang)
-              }
             }
           }
         }
diff --git a/skills/games/rochambeau/nlu/en.json b/skills/games/rochambeau/nlu/en.json
index 2c2a2fbf..5d81d8df 100644
--- a/skills/games/rochambeau/nlu/en.json
+++ b/skills/games/rochambeau/nlu/en.json
@@ -45,7 +45,7 @@
     }
   },
   "entities": {
-    "handsign": "entities/handsigns.json"
+    "handsign": "global-entities/handsign.json"
   },
   "answers": {
     "ready": [
diff --git a/skills/leon/color/nlu/en.json b/skills/leon/color/nlu/en.json
index a01f290b..9fcb27be 100644
--- a/skills/leon/color/nlu/en.json
+++ b/skills/leon/color/nlu/en.json
@@ -73,6 +73,6 @@
     }
   },
   "entities": {
-    "color": "entities/colors.json"
+    "color": "global-entities/color.json"
   }
 }