From ca3f5f42da26eb634e10b56e9b84bd45b5543024 Mon Sep 17 00:00:00 2001 From: louistiti Date: Fri, 11 Feb 2022 21:34:39 +0800 Subject: [PATCH] feat(server): (WIP) NLU refactoring --- package-lock.json | 856 +++++++++++++++++++++++++++++++++++++++-- package.json | 1 + scripts/train.js | 61 +-- server/src/core/nlu.js | 9 +- 4 files changed, 871 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index 233ed5bb..29462085 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@nlpjs/basic": "^4.22.0", "@nlpjs/builtin-microsoft": "^4.22.0", "@nlpjs/core-loader": "^4.22.0", + "@nlpjs/lang-all": "^4.22.12", "@nlpjs/ner": "^4.22.0", "@nlpjs/nlp": "^4.22.0", "archiver": "^5.3.0", @@ -5055,9 +5056,9 @@ } }, "node_modules/@nlpjs/core": { - "version": "4.22.0", - "resolved": "https://registry.npmjs.org/@nlpjs/core/-/core-4.22.0.tgz", - "integrity": "sha512-L7Qb5kP+/ZJ7Wyjoli1nk+SQvvMY0BuuLfYLNvJ2vKL3tMmrSdYBJcXhgmOHEROU6jf4LFgY8VwBpFTsRJV97g==" + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/core/-/core-4.22.7.tgz", + "integrity": "sha512-E8IaPSn4BG4UUrdKxTbJK1Qa4kqmou743IUFv1ZHWCZRWAlSYlq/QTApYwTsuWgA//XV49o2wBfyDMXT1Hoxdg==" }, "node_modules/@nlpjs/core-loader": { "version": "4.22.0", @@ -5077,23 +5078,391 @@ "esprima": "^4.0.1" } }, - "node_modules/@nlpjs/lang-en": { - "version": "4.22.0", - "resolved": "https://registry.npmjs.org/@nlpjs/lang-en/-/lang-en-4.22.0.tgz", - "integrity": "sha512-dtwz/1V6HpKBIvZ3iTBhfQ7cjGAVGmc0g6n88HsBnDYV0subzCAnZcRrzN5r9WBlHgjOMLX77k6pTRCsmR67qw==", + "node_modules/@nlpjs/lang-all": { + "version": "4.22.12", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-all/-/lang-all-4.22.12.tgz", + "integrity": "sha512-Zz1ODC82I92p7EYkEYXFDYkIm52Z25bjbwO0RPTC02upHQ4pxkWx50WVI6d5sH1Ak61lyotuBWdOed+bJy8T0w==", "dependencies": { - "@nlpjs/core": "^4.22.0", - "@nlpjs/lang-en-min": "^4.22.0" + "@nlpjs/core": "^4.22.7", + "@nlpjs/lang-ar": "^4.22.7", + "@nlpjs/lang-bn": "^4.22.7", + "@nlpjs/lang-ca": "^4.22.7", + "@nlpjs/lang-cs": "^4.22.7", + "@nlpjs/lang-da": "^4.22.7", + "@nlpjs/lang-de": "^4.22.7", + "@nlpjs/lang-el": "^4.22.7", + "@nlpjs/lang-en": "^4.22.7", + "@nlpjs/lang-es": "^4.22.7", + "@nlpjs/lang-eu": "^4.22.7", + "@nlpjs/lang-fa": "^4.22.7", + "@nlpjs/lang-fi": "^4.22.7", + "@nlpjs/lang-fr": "^4.22.7", + "@nlpjs/lang-ga": "^4.22.7", + "@nlpjs/lang-gl": "^4.22.7", + "@nlpjs/lang-hi": "^4.22.7", + "@nlpjs/lang-hu": "^4.22.7", + "@nlpjs/lang-hy": "^4.22.7", + "@nlpjs/lang-id": "^4.22.7", + "@nlpjs/lang-it": "^4.22.7", + "@nlpjs/lang-ja": "^4.22.12", + "@nlpjs/lang-ko": "^4.22.7", + "@nlpjs/lang-lt": "^4.22.7", + "@nlpjs/lang-ms": "^4.22.7", + "@nlpjs/lang-ne": "^4.22.7", + "@nlpjs/lang-nl": "^4.22.7", + "@nlpjs/lang-no": "^4.22.7", + "@nlpjs/lang-pl": "^4.22.7", + "@nlpjs/lang-pt": "^4.22.7", + "@nlpjs/lang-ro": "^4.22.7", + "@nlpjs/lang-ru": "^4.22.7", + "@nlpjs/lang-sl": "^4.22.7", + "@nlpjs/lang-sr": "^4.22.7", + "@nlpjs/lang-sv": "^4.22.7", + "@nlpjs/lang-ta": "^4.22.7", + "@nlpjs/lang-th": "^4.22.7", + "@nlpjs/lang-tl": "^4.22.7", + "@nlpjs/lang-tr": "^4.22.7", + "@nlpjs/lang-uk": "^4.22.7", + "@nlpjs/lang-zh": "^4.22.7", + "@nlpjs/language": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ar": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ar/-/lang-ar-4.22.7.tgz", + "integrity": "sha512-qJpWRH0w/nIjEvpVLRUdbDX/MsZRUOTwdBWMy8iqAKU2Ft5aF6vvIa/4Ne4tJD7sv6N64a1g4MUlpz9eC9quIA==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-bn": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-bn/-/lang-bn-4.22.7.tgz", + "integrity": "sha512-ytyVE26PYVtIfumzo5g02vZtdyY8ekwDS58ilh6QNSWpnLlwOEQl4zR3wcrBymVWEmtk641E1rB12buZpA51Ew==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ca": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ca/-/lang-ca-4.22.7.tgz", + "integrity": "sha512-qkbJwPMk91emjSaf4bUlWSijLZIqu1on0tyic9UWftNYcEqiIUTp+q3a9oxyztJw27Oz/1y6Rv3r7595WpIFqQ==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-cs": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-cs/-/lang-cs-4.22.7.tgz", + "integrity": "sha512-pMfGMvDy+ZZSGU4IRyrWctfbYN+pXEiLg4WQkAKRxD1FAW/XuLvOtf3dRO9bYlK0QKM7lS2lE4bCoWELF8VKTw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-da": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-da/-/lang-da-4.22.7.tgz", + "integrity": "sha512-nMI1dKu8yGysAkqyS1OJu9NwRBIIRW6rmyk6LNDkE7gWgKLeH3IxA6v0Q0x3VY97H1AI2W2zO+h7jfpZvP+uLw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-de": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-de/-/lang-de-4.22.7.tgz", + "integrity": "sha512-jXPA+Ny45I/x6uNakAey/j3Yr8GuZp1H8M6/EwX554QLEiwsdxcRn7S/2DyEa/H+IiAEzQHLS9s6R/wIzXJdLQ==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-el": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-el/-/lang-el-4.22.7.tgz", + "integrity": "sha512-16qIVshmAcFbs9A9mrXFZu26qym5/GoTE+sf6auQt0RmyyfEdVUMWlta9vug9OTzhGdx1bm9qTIPIPLPlMJ08g==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-en": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-en/-/lang-en-4.22.7.tgz", + "integrity": "sha512-hyFDZZ6nTebzZHaZCFK+lhj4Bv85CQS+P0Qx/E7WiT5TRXE6g2PxBfqr6fjkAN/tG+YExZMRUOVbkcOhZhbOJg==", + "dependencies": { + "@nlpjs/core": "^4.22.7", + "@nlpjs/lang-en-min": "^4.22.7" } }, "node_modules/@nlpjs/lang-en-min": { - "version": "4.22.0", - "resolved": "https://registry.npmjs.org/@nlpjs/lang-en-min/-/lang-en-min-4.22.0.tgz", - "integrity": "sha512-kmOH5xTTmUcM1hEPJTmN0v44Jp9ZFbSkrS+w8kjV7rFhGhQgIvifT2pVcBuRS7Ho7RSwT2EMSkyR3EiMohjhDQ==", + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-en-min/-/lang-en-min-4.22.7.tgz", + "integrity": "sha512-IOdEiWqZ9alPr+k916/znvLliG/Y8pzxoc2lLvFDzXATIzrmUG2unlHQ9vEXta1rxzDTYOYk57XcFo16GITS3g==", "dependencies": { - "@nlpjs/core": "^4.22.0" + "@nlpjs/core": "^4.22.7" } }, + "node_modules/@nlpjs/lang-es": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-es/-/lang-es-4.22.7.tgz", + "integrity": "sha512-KEenYiU2iaVXlRH1IS9eQkGktRaN7xHRFHOL0CBHTm6Xw9E+SUyVTs1UnGQ6COf3CDcgrU/xZofmGQEpmxoixg==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-eu": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-eu/-/lang-eu-4.22.7.tgz", + "integrity": "sha512-75vBPeP/JVNMyUMYzTPyGDlDzDgKDjIwfSBE8FriSOdim/E8XdSpk/G71z3V9gtPFdTL6ys/NVvg21ZgJw4ddA==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-fa": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-fa/-/lang-fa-4.22.7.tgz", + "integrity": "sha512-d56RvIUCfYvKRxkmOwRFzpEjjJ/ETWWHE5oYgiXp2SdJap6BfF/3iWWtvjU2e/zcFWbxLzJcIHaTdvAWQemUlg==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-fi": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-fi/-/lang-fi-4.22.7.tgz", + "integrity": "sha512-LEhRlvnIcs6cyxm+o3NoANfkDlSLhxkdFyg87MIBMD95AUnD0Xz8UytRybpJiG021zw9tyRMGGcqMZKcFO5Cyg==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-fr": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-fr/-/lang-fr-4.22.7.tgz", + "integrity": "sha512-dJ8evvANYP5lt2dq/tz4OZwcoOqPLs6llrlxEEb3qTH1Zl0v80+TR82+eeriwbm/414+vvWAl9Xg2K8FtmnbAg==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ga": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ga/-/lang-ga-4.22.7.tgz", + "integrity": "sha512-yFSLmhuHVBadfuVjewpDyKCPHY2z0llansA49wD6dqbA+pGkd3La43mThcvJ7TFZrGs7lN8+MdTdeYX0Q8Dzcw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-gl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-gl/-/lang-gl-4.22.7.tgz", + "integrity": "sha512-30LR+/JZop3QDdHAq4jGNeJ5IMiqE0P66/ZbiZIApVQmwlSLkMb2Xta4MZXcSU1Yls/CZy1fGDRhZxOdUWXKvA==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-hi": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-hi/-/lang-hi-4.22.7.tgz", + "integrity": "sha512-NgOHNBLUKdyS1zcyV5HIImlQ7xC1F/72qCEJxW6TEjuhubfPTMGn+mxee1D9I/GvNRSf4YnMYtvzMc3RMPCubQ==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-hu": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-hu/-/lang-hu-4.22.7.tgz", + "integrity": "sha512-2vKRA0iIQL3tWw0ilw1ksx38VUfCX52ELOx7eDvbC86ZE9DfalEON4EaWYNQaOMSCHdy/fyzJdWQgprCAifepQ==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-hy": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-hy/-/lang-hy-4.22.7.tgz", + "integrity": "sha512-5bUWGG14wCuyyvqWd7PKEMR70XElCZK75GYGJGWQP8E1OwvCkHzwWrzUV/uzqa1Tp0Dy7mMuK+H55+taz1qLcQ==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-id": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-id/-/lang-id-4.22.7.tgz", + "integrity": "sha512-Sd3TTwA2Il4T5Ifzz8HlM+XD+6p9NUp8OHjXx834jf9QuWco/PQbMQlnlYtNvEyI5WzlvOIdqp9eyVggfiu47w==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-it": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-it/-/lang-it-4.22.7.tgz", + "integrity": "sha512-0EdrS+atEtH4NQv7Geko4tLU6aqhi7jwvHWjfi5kD4Q5onxkXyzEdlHdH+sQ2qE8sDvricbrtOnm2hSELxqbtw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ja": { + "version": "4.22.12", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ja/-/lang-ja-4.22.12.tgz", + "integrity": "sha512-3ByJ7re3hgj9fytijOd/EKhBHbreYqLIoAyQ23YiWTJDvLdCC/n45Ga/Uat1axqoJS3OThWJlXG3xgmdEkb9+A==", + "dependencies": { + "@nlpjs/core": "^4.22.7", + "kuromoji": "^0.1.2" + } + }, + "node_modules/@nlpjs/lang-ko": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ko/-/lang-ko-4.22.7.tgz", + "integrity": "sha512-q3mpvoKLLqfiyMVp6ZomwLfgvEOcSD+ZXGV7aQjZaWzzNzZ0ccHUFgCxxJ8/efuyJM6+2j0UJOa9bYy/GGAETQ==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-lt": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-lt/-/lang-lt-4.22.7.tgz", + "integrity": "sha512-F621Uq6DGFg3wVb3BhHlSTUUSYlE3k41qXBAQMM/XdU2mhr3mKnvKxfnnbWDUr5YSFHCV6RtJxh+BmKWae+TEw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ms": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ms/-/lang-ms-4.22.7.tgz", + "integrity": "sha512-W/KhiG4e8iqnf+rDl3Ekf7HT1g8ENhrt+RQ5e1cK3iOObjb5LPBUMxWxlH0Sw7opk68AxkN5cgGR1IP5JcjlvQ==", + "dependencies": { + "@nlpjs/core": "^4.22.7", + "@nlpjs/lang-id": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ne": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ne/-/lang-ne-4.22.7.tgz", + "integrity": "sha512-zRcF3qELwwxILFSQCHd7F8hl0Kr6TYuH3xgK/pVRS5VODNanntR7qL/fxSnk9a1vmLJZqe+e9XcfwqZbp7APMw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-nl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-nl/-/lang-nl-4.22.7.tgz", + "integrity": "sha512-4Y2rwFw0bj+SMzGe4HuD4dcl7KLxd6wUK4ONaCWCsb43IK4sNv5f8iN0QO9AojK9ZVV0Fm1H09ylkOcb8mnZtg==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-no": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-no/-/lang-no-4.22.7.tgz", + "integrity": "sha512-pQXcMDoWU6+MRjiXOH+NerLRtSzyVkv3ssFa2LKyVGlFf78wyHM1ktUcSPMZ299JMEUgEafU5NXFkms2ITkNsA==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-pl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-pl/-/lang-pl-4.22.7.tgz", + "integrity": "sha512-4qMzbppKPIdfZmz/Aor0zgxAyG/jYP/+E+kqo6LAkukNsSdgtqrU703XEoTb2xP/7P3L7FtrPlXyGJiRg+aP5g==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-pt": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-pt/-/lang-pt-4.22.7.tgz", + "integrity": "sha512-54YvjSYQdRXEm/lVlBgQrIEHKBqGUpb9EhE0m47qDXaVEKgqCAWC9Gzi0lbYglVn2LWYuS34eciZ2WkPyakOjw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ro": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ro/-/lang-ro-4.22.7.tgz", + "integrity": "sha512-MnFjNNjP5RcIf6Y+jlEoxpiT/WR7Y99INoenG9uInNO+5zcWC8R2cMH0ya4rKQjxuILK2PgN3UYgHmAOaMTe2Q==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ru": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ru/-/lang-ru-4.22.7.tgz", + "integrity": "sha512-zDZsRoJBWfy9KcR98kaKbd6Tzz3C8rN+PW7w80MjAGlXZ/E+s9G4lnIsu94ygOU22/r09eJRw0ssvxVX4dNdow==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-sl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-sl/-/lang-sl-4.22.7.tgz", + "integrity": "sha512-i/6qODPK1lkL+80vuwB4fikU/P2EKFjM4UtqbryNmsq9Xay/Moqnrr8V6JVIRhAb/yzYrUwlJ/MqhQCouh5BhA==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-sr": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-sr/-/lang-sr-4.22.7.tgz", + "integrity": "sha512-+qr0DRZlO7DABg7Lh/kIN4X05ZPab5/HAZnGIJqfig5QCmCCjly5MLd5QhzTmHcwBB3ZNq+NhcC4ue8HvsWu8w==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-sv": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-sv/-/lang-sv-4.22.7.tgz", + "integrity": "sha512-UUo+QKOHwUgSd6jVdjcWXuGqE78lHx2UwhOaIocgz1a5j2Gx+o6b/KKdUaf4SFu1+V41cOycptBmXBVg4yZF7A==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-ta": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ta/-/lang-ta-4.22.7.tgz", + "integrity": "sha512-25D7dCFfE+O1LqPcHS0vJGOCnKe0PR1kKn5MqcBKy0TjzkalXifxyF5YspdCoPkGfyc3V7/2piCSIVWRZtJbpw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-th": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-th/-/lang-th-4.22.7.tgz", + "integrity": "sha512-kQFQn+urftoIR3wLwT3bNlFlpf9/Pa0aFL3aDz+ACBAzgZktvFYaB/NImU4xTywPooew44+iVmXiqOkDtzpPPg==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-tl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-tl/-/lang-tl-4.22.7.tgz", + "integrity": "sha512-dhsNRUTENQ1Apzh6m2p3Nk6OnAIfTFDunCPm+wjSV3Z9wxydstcq1ybJvKyqJKmiY1k408lAu9u7JilI+7b6Jw==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-tr": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-tr/-/lang-tr-4.22.7.tgz", + "integrity": "sha512-2EDgbIbOnTzcc0FbtkB/fVpNV+EEr93nl0BqFT6h5jgwNGiYSKN/kbL6Rbj40XqBv2yIELrI/97iqnlm6VT60Q==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-uk": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-uk/-/lang-uk-4.22.7.tgz", + "integrity": "sha512-+1KlvUa/ZjZx/PSNw7BWZobci1mZoy4q0FoQEt0ZbtW8fSJlqV5ShQCtnmvlYQCOi6gYuIl9AmlCMsQZ0JRoBA==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/lang-zh": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-zh/-/lang-zh-4.22.7.tgz", + "integrity": "sha512-aaYJurU5owbsUB4VlfCfGr6qS6HTqXUwvSaDxMnNi8uQmeLBIpnuXcEgMoVofq1amRqSuwA6Twez4KITBsO6lA==", + "dependencies": { + "@nlpjs/core": "^4.22.7" + } + }, + "node_modules/@nlpjs/language": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/language/-/language-4.22.7.tgz", + "integrity": "sha512-oz5PrPwsIharMGRKZbcIkDtOko5tvynK/s765Wu7fgRHaILXT0ZoqNR/kNeo9BQCsTLV0GvFJreQ/UFy1XGZew==" + }, "node_modules/@nlpjs/language-min": { "version": "4.21.1", "resolved": "https://registry.npmjs.org/@nlpjs/language-min/-/language-min-4.21.1.tgz", @@ -7758,6 +8127,11 @@ "node": ">=10" } }, + "node_modules/doublearray": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/doublearray/-/doublearray-0.0.2.tgz", + "integrity": "sha1-Yxhv6NNEEydtNiH2qg7F954ifvk=" + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -15192,6 +15566,24 @@ "node": ">=6" } }, + "node_modules/kuromoji": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/kuromoji/-/kuromoji-0.1.2.tgz", + "integrity": "sha512-V0dUf+C2LpcPEXhoHLMAop/bOht16Dyr+mDiIE39yX3vqau7p80De/koFqpiTcL1zzdZlc3xuHZ8u5gjYRfFaQ==", + "dependencies": { + "async": "^2.0.1", + "doublearray": "0.0.2", + "zlibjs": "^0.3.1" + } + }, + "node_modules/kuromoji/node_modules/async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "dependencies": { + "lodash": "^4.17.14" + } + }, "node_modules/latest-version": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", @@ -15310,8 +15702,7 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.camelcase": { "version": "4.3.0", @@ -19546,6 +19937,14 @@ "engines": { "node": ">= 6" } + }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha1-UBl+2yihxCymWcyLTmqd3W1ERVQ=", + "engines": { + "node": "*" + } } }, "dependencies": { @@ -23391,9 +23790,9 @@ } }, "@nlpjs/core": { - "version": "4.22.0", - "resolved": "https://registry.npmjs.org/@nlpjs/core/-/core-4.22.0.tgz", - "integrity": "sha512-L7Qb5kP+/ZJ7Wyjoli1nk+SQvvMY0BuuLfYLNvJ2vKL3tMmrSdYBJcXhgmOHEROU6jf4LFgY8VwBpFTsRJV97g==" + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/core/-/core-4.22.7.tgz", + "integrity": "sha512-E8IaPSn4BG4UUrdKxTbJK1Qa4kqmou743IUFv1ZHWCZRWAlSYlq/QTApYwTsuWgA//XV49o2wBfyDMXT1Hoxdg==" }, "@nlpjs/core-loader": { "version": "4.22.0", @@ -23413,23 +23812,391 @@ "esprima": "^4.0.1" } }, - "@nlpjs/lang-en": { - "version": "4.22.0", - "resolved": "https://registry.npmjs.org/@nlpjs/lang-en/-/lang-en-4.22.0.tgz", - "integrity": "sha512-dtwz/1V6HpKBIvZ3iTBhfQ7cjGAVGmc0g6n88HsBnDYV0subzCAnZcRrzN5r9WBlHgjOMLX77k6pTRCsmR67qw==", + "@nlpjs/lang-all": { + "version": "4.22.12", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-all/-/lang-all-4.22.12.tgz", + "integrity": "sha512-Zz1ODC82I92p7EYkEYXFDYkIm52Z25bjbwO0RPTC02upHQ4pxkWx50WVI6d5sH1Ak61lyotuBWdOed+bJy8T0w==", "requires": { - "@nlpjs/core": "^4.22.0", - "@nlpjs/lang-en-min": "^4.22.0" + "@nlpjs/core": "^4.22.7", + "@nlpjs/lang-ar": "^4.22.7", + "@nlpjs/lang-bn": "^4.22.7", + "@nlpjs/lang-ca": "^4.22.7", + "@nlpjs/lang-cs": "^4.22.7", + "@nlpjs/lang-da": "^4.22.7", + "@nlpjs/lang-de": "^4.22.7", + "@nlpjs/lang-el": "^4.22.7", + "@nlpjs/lang-en": "^4.22.7", + "@nlpjs/lang-es": "^4.22.7", + "@nlpjs/lang-eu": "^4.22.7", + "@nlpjs/lang-fa": "^4.22.7", + "@nlpjs/lang-fi": "^4.22.7", + "@nlpjs/lang-fr": "^4.22.7", + "@nlpjs/lang-ga": "^4.22.7", + "@nlpjs/lang-gl": "^4.22.7", + "@nlpjs/lang-hi": "^4.22.7", + "@nlpjs/lang-hu": "^4.22.7", + "@nlpjs/lang-hy": "^4.22.7", + "@nlpjs/lang-id": "^4.22.7", + "@nlpjs/lang-it": "^4.22.7", + "@nlpjs/lang-ja": "^4.22.12", + "@nlpjs/lang-ko": "^4.22.7", + "@nlpjs/lang-lt": "^4.22.7", + "@nlpjs/lang-ms": "^4.22.7", + "@nlpjs/lang-ne": "^4.22.7", + "@nlpjs/lang-nl": "^4.22.7", + "@nlpjs/lang-no": "^4.22.7", + "@nlpjs/lang-pl": "^4.22.7", + "@nlpjs/lang-pt": "^4.22.7", + "@nlpjs/lang-ro": "^4.22.7", + "@nlpjs/lang-ru": "^4.22.7", + "@nlpjs/lang-sl": "^4.22.7", + "@nlpjs/lang-sr": "^4.22.7", + "@nlpjs/lang-sv": "^4.22.7", + "@nlpjs/lang-ta": "^4.22.7", + "@nlpjs/lang-th": "^4.22.7", + "@nlpjs/lang-tl": "^4.22.7", + "@nlpjs/lang-tr": "^4.22.7", + "@nlpjs/lang-uk": "^4.22.7", + "@nlpjs/lang-zh": "^4.22.7", + "@nlpjs/language": "^4.22.7" + } + }, + "@nlpjs/lang-ar": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ar/-/lang-ar-4.22.7.tgz", + "integrity": "sha512-qJpWRH0w/nIjEvpVLRUdbDX/MsZRUOTwdBWMy8iqAKU2Ft5aF6vvIa/4Ne4tJD7sv6N64a1g4MUlpz9eC9quIA==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-bn": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-bn/-/lang-bn-4.22.7.tgz", + "integrity": "sha512-ytyVE26PYVtIfumzo5g02vZtdyY8ekwDS58ilh6QNSWpnLlwOEQl4zR3wcrBymVWEmtk641E1rB12buZpA51Ew==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-ca": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ca/-/lang-ca-4.22.7.tgz", + "integrity": "sha512-qkbJwPMk91emjSaf4bUlWSijLZIqu1on0tyic9UWftNYcEqiIUTp+q3a9oxyztJw27Oz/1y6Rv3r7595WpIFqQ==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-cs": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-cs/-/lang-cs-4.22.7.tgz", + "integrity": "sha512-pMfGMvDy+ZZSGU4IRyrWctfbYN+pXEiLg4WQkAKRxD1FAW/XuLvOtf3dRO9bYlK0QKM7lS2lE4bCoWELF8VKTw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-da": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-da/-/lang-da-4.22.7.tgz", + "integrity": "sha512-nMI1dKu8yGysAkqyS1OJu9NwRBIIRW6rmyk6LNDkE7gWgKLeH3IxA6v0Q0x3VY97H1AI2W2zO+h7jfpZvP+uLw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-de": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-de/-/lang-de-4.22.7.tgz", + "integrity": "sha512-jXPA+Ny45I/x6uNakAey/j3Yr8GuZp1H8M6/EwX554QLEiwsdxcRn7S/2DyEa/H+IiAEzQHLS9s6R/wIzXJdLQ==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-el": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-el/-/lang-el-4.22.7.tgz", + "integrity": "sha512-16qIVshmAcFbs9A9mrXFZu26qym5/GoTE+sf6auQt0RmyyfEdVUMWlta9vug9OTzhGdx1bm9qTIPIPLPlMJ08g==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-en": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-en/-/lang-en-4.22.7.tgz", + "integrity": "sha512-hyFDZZ6nTebzZHaZCFK+lhj4Bv85CQS+P0Qx/E7WiT5TRXE6g2PxBfqr6fjkAN/tG+YExZMRUOVbkcOhZhbOJg==", + "requires": { + "@nlpjs/core": "^4.22.7", + "@nlpjs/lang-en-min": "^4.22.7" } }, "@nlpjs/lang-en-min": { - "version": "4.22.0", - "resolved": "https://registry.npmjs.org/@nlpjs/lang-en-min/-/lang-en-min-4.22.0.tgz", - "integrity": "sha512-kmOH5xTTmUcM1hEPJTmN0v44Jp9ZFbSkrS+w8kjV7rFhGhQgIvifT2pVcBuRS7Ho7RSwT2EMSkyR3EiMohjhDQ==", + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-en-min/-/lang-en-min-4.22.7.tgz", + "integrity": "sha512-IOdEiWqZ9alPr+k916/znvLliG/Y8pzxoc2lLvFDzXATIzrmUG2unlHQ9vEXta1rxzDTYOYk57XcFo16GITS3g==", "requires": { - "@nlpjs/core": "^4.22.0" + "@nlpjs/core": "^4.22.7" } }, + "@nlpjs/lang-es": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-es/-/lang-es-4.22.7.tgz", + "integrity": "sha512-KEenYiU2iaVXlRH1IS9eQkGktRaN7xHRFHOL0CBHTm6Xw9E+SUyVTs1UnGQ6COf3CDcgrU/xZofmGQEpmxoixg==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-eu": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-eu/-/lang-eu-4.22.7.tgz", + "integrity": "sha512-75vBPeP/JVNMyUMYzTPyGDlDzDgKDjIwfSBE8FriSOdim/E8XdSpk/G71z3V9gtPFdTL6ys/NVvg21ZgJw4ddA==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-fa": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-fa/-/lang-fa-4.22.7.tgz", + "integrity": "sha512-d56RvIUCfYvKRxkmOwRFzpEjjJ/ETWWHE5oYgiXp2SdJap6BfF/3iWWtvjU2e/zcFWbxLzJcIHaTdvAWQemUlg==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-fi": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-fi/-/lang-fi-4.22.7.tgz", + "integrity": "sha512-LEhRlvnIcs6cyxm+o3NoANfkDlSLhxkdFyg87MIBMD95AUnD0Xz8UytRybpJiG021zw9tyRMGGcqMZKcFO5Cyg==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-fr": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-fr/-/lang-fr-4.22.7.tgz", + "integrity": "sha512-dJ8evvANYP5lt2dq/tz4OZwcoOqPLs6llrlxEEb3qTH1Zl0v80+TR82+eeriwbm/414+vvWAl9Xg2K8FtmnbAg==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-ga": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ga/-/lang-ga-4.22.7.tgz", + "integrity": "sha512-yFSLmhuHVBadfuVjewpDyKCPHY2z0llansA49wD6dqbA+pGkd3La43mThcvJ7TFZrGs7lN8+MdTdeYX0Q8Dzcw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-gl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-gl/-/lang-gl-4.22.7.tgz", + "integrity": "sha512-30LR+/JZop3QDdHAq4jGNeJ5IMiqE0P66/ZbiZIApVQmwlSLkMb2Xta4MZXcSU1Yls/CZy1fGDRhZxOdUWXKvA==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-hi": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-hi/-/lang-hi-4.22.7.tgz", + "integrity": "sha512-NgOHNBLUKdyS1zcyV5HIImlQ7xC1F/72qCEJxW6TEjuhubfPTMGn+mxee1D9I/GvNRSf4YnMYtvzMc3RMPCubQ==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-hu": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-hu/-/lang-hu-4.22.7.tgz", + "integrity": "sha512-2vKRA0iIQL3tWw0ilw1ksx38VUfCX52ELOx7eDvbC86ZE9DfalEON4EaWYNQaOMSCHdy/fyzJdWQgprCAifepQ==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-hy": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-hy/-/lang-hy-4.22.7.tgz", + "integrity": "sha512-5bUWGG14wCuyyvqWd7PKEMR70XElCZK75GYGJGWQP8E1OwvCkHzwWrzUV/uzqa1Tp0Dy7mMuK+H55+taz1qLcQ==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-id": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-id/-/lang-id-4.22.7.tgz", + "integrity": "sha512-Sd3TTwA2Il4T5Ifzz8HlM+XD+6p9NUp8OHjXx834jf9QuWco/PQbMQlnlYtNvEyI5WzlvOIdqp9eyVggfiu47w==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-it": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-it/-/lang-it-4.22.7.tgz", + "integrity": "sha512-0EdrS+atEtH4NQv7Geko4tLU6aqhi7jwvHWjfi5kD4Q5onxkXyzEdlHdH+sQ2qE8sDvricbrtOnm2hSELxqbtw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-ja": { + "version": "4.22.12", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ja/-/lang-ja-4.22.12.tgz", + "integrity": "sha512-3ByJ7re3hgj9fytijOd/EKhBHbreYqLIoAyQ23YiWTJDvLdCC/n45Ga/Uat1axqoJS3OThWJlXG3xgmdEkb9+A==", + "requires": { + "@nlpjs/core": "^4.22.7", + "kuromoji": "^0.1.2" + } + }, + "@nlpjs/lang-ko": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ko/-/lang-ko-4.22.7.tgz", + "integrity": "sha512-q3mpvoKLLqfiyMVp6ZomwLfgvEOcSD+ZXGV7aQjZaWzzNzZ0ccHUFgCxxJ8/efuyJM6+2j0UJOa9bYy/GGAETQ==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-lt": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-lt/-/lang-lt-4.22.7.tgz", + "integrity": "sha512-F621Uq6DGFg3wVb3BhHlSTUUSYlE3k41qXBAQMM/XdU2mhr3mKnvKxfnnbWDUr5YSFHCV6RtJxh+BmKWae+TEw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-ms": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ms/-/lang-ms-4.22.7.tgz", + "integrity": "sha512-W/KhiG4e8iqnf+rDl3Ekf7HT1g8ENhrt+RQ5e1cK3iOObjb5LPBUMxWxlH0Sw7opk68AxkN5cgGR1IP5JcjlvQ==", + "requires": { + "@nlpjs/core": "^4.22.7", + "@nlpjs/lang-id": "^4.22.7" + } + }, + "@nlpjs/lang-ne": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ne/-/lang-ne-4.22.7.tgz", + "integrity": "sha512-zRcF3qELwwxILFSQCHd7F8hl0Kr6TYuH3xgK/pVRS5VODNanntR7qL/fxSnk9a1vmLJZqe+e9XcfwqZbp7APMw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-nl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-nl/-/lang-nl-4.22.7.tgz", + "integrity": "sha512-4Y2rwFw0bj+SMzGe4HuD4dcl7KLxd6wUK4ONaCWCsb43IK4sNv5f8iN0QO9AojK9ZVV0Fm1H09ylkOcb8mnZtg==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-no": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-no/-/lang-no-4.22.7.tgz", + "integrity": "sha512-pQXcMDoWU6+MRjiXOH+NerLRtSzyVkv3ssFa2LKyVGlFf78wyHM1ktUcSPMZ299JMEUgEafU5NXFkms2ITkNsA==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-pl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-pl/-/lang-pl-4.22.7.tgz", + "integrity": "sha512-4qMzbppKPIdfZmz/Aor0zgxAyG/jYP/+E+kqo6LAkukNsSdgtqrU703XEoTb2xP/7P3L7FtrPlXyGJiRg+aP5g==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-pt": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-pt/-/lang-pt-4.22.7.tgz", + "integrity": "sha512-54YvjSYQdRXEm/lVlBgQrIEHKBqGUpb9EhE0m47qDXaVEKgqCAWC9Gzi0lbYglVn2LWYuS34eciZ2WkPyakOjw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-ro": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ro/-/lang-ro-4.22.7.tgz", + "integrity": "sha512-MnFjNNjP5RcIf6Y+jlEoxpiT/WR7Y99INoenG9uInNO+5zcWC8R2cMH0ya4rKQjxuILK2PgN3UYgHmAOaMTe2Q==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-ru": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ru/-/lang-ru-4.22.7.tgz", + "integrity": "sha512-zDZsRoJBWfy9KcR98kaKbd6Tzz3C8rN+PW7w80MjAGlXZ/E+s9G4lnIsu94ygOU22/r09eJRw0ssvxVX4dNdow==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-sl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-sl/-/lang-sl-4.22.7.tgz", + "integrity": "sha512-i/6qODPK1lkL+80vuwB4fikU/P2EKFjM4UtqbryNmsq9Xay/Moqnrr8V6JVIRhAb/yzYrUwlJ/MqhQCouh5BhA==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-sr": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-sr/-/lang-sr-4.22.7.tgz", + "integrity": "sha512-+qr0DRZlO7DABg7Lh/kIN4X05ZPab5/HAZnGIJqfig5QCmCCjly5MLd5QhzTmHcwBB3ZNq+NhcC4ue8HvsWu8w==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-sv": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-sv/-/lang-sv-4.22.7.tgz", + "integrity": "sha512-UUo+QKOHwUgSd6jVdjcWXuGqE78lHx2UwhOaIocgz1a5j2Gx+o6b/KKdUaf4SFu1+V41cOycptBmXBVg4yZF7A==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-ta": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-ta/-/lang-ta-4.22.7.tgz", + "integrity": "sha512-25D7dCFfE+O1LqPcHS0vJGOCnKe0PR1kKn5MqcBKy0TjzkalXifxyF5YspdCoPkGfyc3V7/2piCSIVWRZtJbpw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-th": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-th/-/lang-th-4.22.7.tgz", + "integrity": "sha512-kQFQn+urftoIR3wLwT3bNlFlpf9/Pa0aFL3aDz+ACBAzgZktvFYaB/NImU4xTywPooew44+iVmXiqOkDtzpPPg==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-tl": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-tl/-/lang-tl-4.22.7.tgz", + "integrity": "sha512-dhsNRUTENQ1Apzh6m2p3Nk6OnAIfTFDunCPm+wjSV3Z9wxydstcq1ybJvKyqJKmiY1k408lAu9u7JilI+7b6Jw==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-tr": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-tr/-/lang-tr-4.22.7.tgz", + "integrity": "sha512-2EDgbIbOnTzcc0FbtkB/fVpNV+EEr93nl0BqFT6h5jgwNGiYSKN/kbL6Rbj40XqBv2yIELrI/97iqnlm6VT60Q==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-uk": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-uk/-/lang-uk-4.22.7.tgz", + "integrity": "sha512-+1KlvUa/ZjZx/PSNw7BWZobci1mZoy4q0FoQEt0ZbtW8fSJlqV5ShQCtnmvlYQCOi6gYuIl9AmlCMsQZ0JRoBA==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/lang-zh": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/lang-zh/-/lang-zh-4.22.7.tgz", + "integrity": "sha512-aaYJurU5owbsUB4VlfCfGr6qS6HTqXUwvSaDxMnNi8uQmeLBIpnuXcEgMoVofq1amRqSuwA6Twez4KITBsO6lA==", + "requires": { + "@nlpjs/core": "^4.22.7" + } + }, + "@nlpjs/language": { + "version": "4.22.7", + "resolved": "https://registry.npmjs.org/@nlpjs/language/-/language-4.22.7.tgz", + "integrity": "sha512-oz5PrPwsIharMGRKZbcIkDtOko5tvynK/s765Wu7fgRHaILXT0ZoqNR/kNeo9BQCsTLV0GvFJreQ/UFy1XGZew==" + }, "@nlpjs/language-min": { "version": "4.21.1", "resolved": "https://registry.npmjs.org/@nlpjs/language-min/-/language-min-4.21.1.tgz", @@ -25523,6 +26290,11 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==" }, + "doublearray": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/doublearray/-/doublearray-0.0.2.tgz", + "integrity": "sha1-Yxhv6NNEEydtNiH2qg7F954ifvk=" + }, "duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -31098,6 +31870,26 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, + "kuromoji": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/kuromoji/-/kuromoji-0.1.2.tgz", + "integrity": "sha512-V0dUf+C2LpcPEXhoHLMAop/bOht16Dyr+mDiIE39yX3vqau7p80De/koFqpiTcL1zzdZlc3xuHZ8u5gjYRfFaQ==", + "requires": { + "async": "^2.0.1", + "doublearray": "0.0.2", + "zlibjs": "^0.3.1" + }, + "dependencies": { + "async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "requires": { + "lodash": "^4.17.14" + } + } + } + }, "latest-version": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", @@ -31193,8 +31985,7 @@ "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.camelcase": { "version": "4.3.0", @@ -34480,6 +35271,11 @@ } } } + }, + "zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha1-UBl+2yihxCymWcyLTmqd3W1ERVQ=" } } } diff --git a/package.json b/package.json index 7b13977b..85cd9a4a 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@nlpjs/basic": "^4.22.0", "@nlpjs/builtin-microsoft": "^4.22.0", "@nlpjs/core-loader": "^4.22.0", + "@nlpjs/lang-all": "^4.22.12", "@nlpjs/ner": "^4.22.0", "@nlpjs/nlp": "^4.22.0", "archiver": "^5.3.0", diff --git a/scripts/train.js b/scripts/train.js index 0ae26fdb..2be08829 100644 --- a/scripts/train.js +++ b/scripts/train.js @@ -6,7 +6,7 @@ import path from 'path' import log from '@/helpers/log' import string from '@/helpers/string' -import { langs } from '@@/core/langs.json' +// import { langs } from '@@/core/langs.json' dotenv.config() @@ -16,50 +16,61 @@ dotenv.config() * npm run train [en or fr] */ export default () => new Promise(async (resolve, reject) => { - const { argv } = process + // const { argv } = process const packagesDir = 'packages' const modelFileName = 'server/src/data/leon-model.nlp' - const lang = argv[2] + /* const lang = argv[2] ? argv[2].toLowerCase() - : langs[process.env.LEON_LANG].short.toLowerCase().substr(0, 2) + : langs[process.env.LEON_LANG].short.toLowerCase().substr(0, 2) */ try { - const dock = await dockStart({ use: ['Basic'] }) + const dock = await dockStart({ use: ['Basic', 'LangEn', 'LangFr'] }) const nlp = dock.get('nlp') + const nluManager = dock.get('nlu-manager') + + nluManager.settings.log = false + nluManager.settings.trainByDomain = true + nluManager.settings.spellCheck = true + nlp.settings.calculateSentiment = true nlp.settings.modelFileName = modelFileName nlp.settings.threshold = 0.8 - nlp.addLanguage(lang) + // TODO: grab from core/langs.json + const langs = ['en', 'fr'] - const packages = fs.readdirSync(packagesDir) - .filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory()) - let utteranceSamplesObj = { } + langs.forEach(async (lang) => { + nlp.addLanguage(lang) - for (let i = 0; i < packages.length; i += 1) { - log.info(`Training "${string.ucfirst(packages[i])}" package modules utterance samples...`) + const packages = fs.readdirSync(packagesDir) + .filter((entity) => fs.statSync(path.join(packagesDir, entity)).isDirectory()) + let utteranceSamplesObj = { } - utteranceSamplesObj = JSON.parse(fs.readFileSync(`${packagesDir}/${packages[i]}/data/expressions/${lang}.json`, 'utf8')) + for (let i = 0; i < packages.length; i += 1) { + log.info(`[${lang}] Training "${string.ucfirst(packages[i])}" package modules utterance samples...`) - const modules = Object.keys(utteranceSamplesObj) - for (let j = 0; j < modules.length; j += 1) { - const module = modules[j] - const actions = Object.keys(utteranceSamplesObj[module]) + utteranceSamplesObj = JSON.parse(fs.readFileSync(`${packagesDir}/${packages[i]}/data/expressions/${lang}.json`, 'utf8')) - for (let k = 0; k < actions.length; k += 1) { - const action = actions[k] - const exprs = utteranceSamplesObj[module][action].utterance_samples + const modules = Object.keys(utteranceSamplesObj) + for (let j = 0; j < modules.length; j += 1) { + const module = modules[j] + const actions = Object.keys(utteranceSamplesObj[module]) - nlp.assignDomain(lang, `${module}.${action}`, packages[i]) + for (let k = 0; k < actions.length; k += 1) { + const action = actions[k] + const exprs = utteranceSamplesObj[module][action].utterance_samples - for (let l = 0; l < exprs.length; l += 1) { - nlp.addDocument(lang, exprs[l], `${module}.${action}`) + nlp.assignDomain(lang, `${module}.${action}`, packages[i]) + + for (let l = 0; l < exprs.length; l += 1) { + nlp.addDocument(lang, exprs[l], `${module}.${action}`) + } } - } - log.success(`"${string.ucfirst(module)}" module utterance samples trained`) + log.success(`[${lang}] "${string.ucfirst(module)}" module utterance samples trained`) + } } - } + }) try { await nlp.train() diff --git a/server/src/core/nlu.js b/server/src/core/nlu.js index 0ad32719..a18e7d16 100644 --- a/server/src/core/nlu.js +++ b/server/src/core/nlu.js @@ -79,7 +79,14 @@ class Nlu { } const lang = langs[process.env.LEON_LANG].short - const result = await this.nlp.process(lang, utterance) + const guessedLang = await this.nlp.guessLanguage(utterance) + + console.log('guessedLang', guessedLang) + console.log('sentiment', await this.nlp.getSentiment(utterance)) + + const result = await this.nlp.process(utterance) + + console.log('result', result) const { domain, intent, score