feat: Python bridge binary build with cx_Freeze

.dockerignore

@@ -1,2 +1,2 @@
 node_modules/
-bridges/python/.venv/*
+bridges/python/src/.venv/*

.gitignore

@@ -2,7 +2,6 @@ __pycache__/
 .idea/
 .vscode/
 **/dist/
-!bridges/python/dist/
 **/build/
 **/node_modules/
 test/coverage/

bridges/python/src/Pipfile

@@ -7,7 +7,7 @@ name = "pypi"
 python_version = "3.9.10"
 
 [packages]
-requests = "==2.21.0"
+requests = "==2.28.1"
 pytube = "==9.5.0"
 tinydb = "==4.7.0"
 beautifulsoup4 = "==4.7.1"
@@ -19,4 +19,4 @@ python-dotenv = "==0.19.2"
 geonamescache = "==1.3.0"
 
 [dev-packages]
-pyinstaller = "==5.4.1"
+cx-freeze = "==6.10"

bridges/python/src/python-bridge.spec

@@ -1,50 +0,0 @@
-# -*- mode: python ; coding: utf-8 -*-
-
-
-block_cipher = None
-
-
-a = Analysis(
-    ['main.py'],
-    pathex=[],
-    binaries=[],
-    datas=[],
-    hiddenimports=[],
-    hookspath=[],
-    hooksconfig={},
-    runtime_hooks=[],
-    excludes=[],
-    win_no_prefer_redirects=False,
-    win_private_assemblies=False,
-    cipher=block_cipher,
-    noarchive=False,
-)
-pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
-
-exe = EXE(
-    pyz,
-    a.scripts,
-    [],
-    exclude_binaries=True,
-    name='python-bridge',
-    debug=False,
-    bootloader_ignore_signals=False,
-    strip=False,
-    upx=True,
-    console=True,
-    disable_windowed_traceback=False,
-    argv_emulation=False,
-    target_arch=None,
-    codesign_identity=None,
-    entitlements_file=None,
-)
-coll = COLLECT(
-    exe,
-    a.binaries,
-    a.zipfiles,
-    a.datas,
-    strip=False,
-    upx=True,
-    upx_exclude=[],
-    name='python-bridge',
-)

bridges/python/src/setup.py

@@ -0,0 +1,23 @@
+import requests.certs
+from cx_Freeze import setup, Executable
+
+options = {
+    'build_exe': {
+    	# Add common dependencies for skills
+        'includes': ['bs4', 'pytube']
+	},
+	'compiler': {}
+}
+
+executables = [
+	Executable(
+		script='main.py',
+		target_name='leon-python-bridge'
+    )
+]
+
+setup(
+    name='leon-python-bridge',
+    executables=executables,
+    options=options
+)

bridges/python/src/utils.py

@@ -2,20 +2,20 @@
 # -*- coding:utf-8 -*-
 
 from json import loads, dumps
-from os import path, environ
 from pathlib import Path
 from random import choice
-from sys import argv, stdout
 from vars import useragent
 from tinydb import TinyDB, Query, table, operations
 from time import sleep
+import sys
+import os
 import sqlite3
 import requests
 import re
 
-dirname = path.dirname(path.realpath(__file__))
+dirname = os.path.dirname(os.path.realpath(__file__))
 
-intent_object_path = argv[1]
+intent_object_path = sys.argv[1]
 codes = []
 
 intent_obj_file = open(intent_object_path, 'r', encoding = 'utf8')
@@ -37,7 +37,7 @@ def translate(key, dict = { }):
 	output = ''
 	variables = { }
 
-	file = open(path.join(dirname, '../../../../skills', intent_obj['domain'], intent_obj['skill'], 'config', intent_obj['lang'] + '.json'), 'r', encoding = 'utf8')
+	file = open(os.path.join(os.getcwd(), 'skills', intent_obj['domain'], intent_obj['skill'], 'config', intent_obj['lang'] + '.json'), 'r', encoding = 'utf8')
 	obj = loads(file.read())
 	file.close()
 
@@ -92,7 +92,7 @@ def output(type, content = '', core = { }):
 	}))
 
 	if (type == 'inter'):
-		stdout.flush()
+		sys.stdout.flush()
 
 def http(method, url, headers = None):
 	"""Send HTTP request with the Leon user agent"""
@@ -108,7 +108,7 @@ def http(method, url, headers = None):
 def config(key):
 	"""Get a skill configuration value"""
 
-	file = open(path.join(dirname, '../../../../skills', intent_obj['domain'], intent_obj['skill'], 'src/config.json'), 'r', encoding = 'utf8')
+	file = open(os.path.join(os.getcwd(), 'skills', intent_obj['domain'], intent_obj['skill'], 'src/config.json'), 'r', encoding = 'utf8')
 	obj = loads(file.read())
 	file.close()
 
@@ -117,8 +117,9 @@ def config(key):
 def create_dl_dir():
 	"""Create the downloads folder of a current skill"""
 
-	dl_dir = path.dirname(path.realpath(__file__)) + '/../../../../downloads/'
-	skill_dl_dir = path.join(dl_dir, intent_obj['domain'], intent_obj['skill'])
+	dl_dir = os.path.join(os.getcwd(), 'downloads')
+	# dl_dir = os.path.dirname(os.path.realpath(__file__)) + '/../../../../downloads/'
+	skill_dl_dir = os.path.join(dl_dir, intent_obj['domain'], intent_obj['skill'])
 
 	Path(skill_dl_dir).mkdir(parents = True, exist_ok = True)
 
@@ -129,8 +130,8 @@ def db(db_type = 'tinydb'):
 	for a specific skill"""
 
 	if db_type == 'tinydb':
-		ext = '.json' if environ.get('LEON_NODE_ENV') != 'testing' else '.spec.json'
-		db = TinyDB(path.join(dirname, '../../../../skills', intent_obj['domain'], intent_obj['skill'], 'memory/db' + ext))
+		ext = '.json' if os.environ.get('LEON_NODE_ENV') != 'testing' else '.spec.json'
+		db = TinyDB(os.path.join(os.getcwd(), 'skills', intent_obj['domain'], intent_obj['skill'], 'memory/db' + ext))
 		return {
 			'db': db,
 			'query': Query,
@@ -142,6 +143,6 @@ def get_table(slug):
 	"""Get a table from a specific skill"""
 
 	domain, skill, table = slug.split('.')
-	ext = '.json' if environ.get('LEON_NODE_ENV') != 'testing' else '.spec.json'
-	db = TinyDB(path.join(dirname, '../../../../skills', domain, skill, 'memory/db' + ext))
+	ext = '.json' if os.environ.get('LEON_NODE_ENV') != 'testing' else '.spec.json'
+	db = TinyDB(os.path.join(os.getcwd(), 'skills', domain, skill, 'memory/db' + ext))
 	return db.table(table)

bridges/python/src/vars.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 # -*- coding:utf-8 -*-
 
-from os import path
+import sys
+import os
 from json import loads
 
-packagejsonfile = open(path.dirname(path.realpath(__file__)) + '/../../../../package.json', 'r', encoding = 'utf8')
+packagejsonfile = open(os.path.join(os.getcwd(), 'package.json'), 'r', encoding = 'utf8')
 packagejson = loads(packagejsonfile.read())
 packagejsonfile.close()
 

package.json

@@ -23,12 +23,12 @@
   "scripts": {
     "lint": "ts-node scripts/lint.js",
     "test": "npm run test:json && npm run test:over-http && npm run test:unit && npm run test:e2e",
-    "test:unit": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --projects test/unit/unit.jest.json && npm run train",
+    "test:unit": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --projects test/unit/unit.jest.json && npm run train",
     "test:e2e": "npm run test:e2e:nlp-modules && npm run test:e2e:modules",
-    "test:e2e:modules": "ts-node scripts/run-clean-test-dbs.js && npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && ts-node scripts/run-clean-test-dbs.js && npm run train",
-    "test:e2e:nlp-modules": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train",
+    "test:e2e:modules": "ts-node scripts/run-clean-test-dbs.js && npm run train en && cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --projects test/e2e/modules/e2e.modules.jest.json && ts-node scripts/run-clean-test-dbs.js && npm run train",
+    "test:e2e:nlp-modules": "npm run train en && cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile LEON_NODE_ENV=testing jest --forceExit --silent --verbose --setupTestFrameworkScriptFile=./test/paths.setup.js test/e2e/nlp-modules.spec.js && npm run train",
     "test:json": "jest --silent --projects test/json/json.jest.json",
-    "test:over-http": "npm run generate:skills-endpoints && npm run train && cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing LEON_HOST=http://localhost LEON_PORT=1338 LEON_HTTP_API_KEY=72aeb5ba324580963114481144385d7179c106fc jest --forceExit --silent --verbose --notify=false --bail --collectCoverage=false test/e2e/over-http.spec.js",
+    "test:over-http": "npm run generate:skills-endpoints && npm run train && cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile LEON_NODE_ENV=testing LEON_HOST=http://localhost LEON_PORT=1338 LEON_HTTP_API_KEY=72aeb5ba324580963114481144385d7179c106fc jest --forceExit --silent --verbose --notify=false --bail --collectCoverage=false test/e2e/over-http.spec.js",
     "test:module": "ts-node scripts/test-module.js",
     "setup:offline": "ts-node scripts/setup-offline/setup-offline.js",
     "setup:offline-stt": "ts-node scripts/setup-offline/run-setup-stt.js",
@@ -40,14 +40,14 @@
     "dev:server": "npm run train && npm run generate:skills-endpoints && cross-env LEON_NODE_ENV=development tsc-watch --noClear --onSuccess \"nodemon\"",
     "wake": "cross-env LEON_HOST=http://localhost LEON_PORT=1337 node hotword/index.js",
     "delete-dist:server": "shx rm -rf ./server/dist",
-    "clean:python-deps": "shx rm -rf ./bridges/python/.venv && npm run postinstall",
+    "clean:python-deps": "shx rm -rf ./bridges/python/src/.venv && npm run postinstall",
     "prepare": "husky install",
     "generate:skills-endpoints": "ts-node scripts/generate/run-generate-skills-endpoints.js",
     "generate:http-api-key": "ts-node scripts/generate/run-generate-http-api-key.js",
     "build": "npm run build:app && npm run build:server",
     "build:app": "cross-env LEON_NODE_ENV=production ts-node scripts/app/run-build-app.js",
     "build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && tsc && resolve-tspaths && shx rm -rf server/dist/core server/dist/package.json && shx mv -f server/dist/server/src/* server/dist && shx rm -rf server/dist/server && shx mkdir -p server/dist/tmp",
-    "start:tcp-server": "cross-env PIPENV_PIPFILE=bridges/python/Pipfile pipenv run python bridges/python/src/tcp_server/main.py",
+    "start:tcp-server": "cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile pipenv run python bridges/python/src/tcp_server/main.py",
     "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
     "train": "ts-node scripts/train/run-train.js",
     "prepare-release": "ts-node scripts/release/prepare-release.js",

scripts/check.js

@@ -171,7 +171,7 @@ export default () =>
       try {
         LogHelper.time('Skill execution time')
         const p = await command(
-          './bridges/python/dist/python-bridge/python-bridge scripts/assets/intent-object.json',
+          './bridges/python/dist/python-bridge/leon-python-bridge scripts/assets/intent-object.json',
           { shell: true }
         )
         LogHelper.timeEnd('Skill execution time')

scripts/setup/setup-python-packages.js

@@ -13,8 +13,8 @@ export default () =>
     LogHelper.info('Checking Python env...')
 
     // Check if the Pipfile exists
-    if (fs.existsSync('bridges/python/Pipfile')) {
-      LogHelper.success('bridges/python/Pipfile found')
+    if (fs.existsSync('bridges/python/src/Pipfile')) {
+      LogHelper.success('bridges/python/src/Pipfile found')
 
       try {
         // Check if Pipenv is installed
@@ -39,8 +39,11 @@ export default () =>
       }
 
       try {
-        const dotVenvPath = path.join(process.cwd(), 'bridges/python/.venv')
-        const pipfilePath = path.join(process.cwd(), 'bridges/python/Pipfile')
+        const dotVenvPath = path.join(process.cwd(), 'bridges/python/src/.venv')
+        const pipfilePath = path.join(
+          process.cwd(),
+          'bridges/python/src/Pipfile'
+        )
         const pipfileMtime = fs.statSync(pipfilePath).mtime
         const isDotVenvExist = fs.existsSync(dotVenvPath)
         const installPythonPackages = async () => {
@@ -52,10 +55,10 @@ export default () =>
 
           // Installing Python packages
           LogHelper.info(
-            'Installing Python packages from bridges/python/Pipfile...'
+            'Installing Python packages from bridges/python/src/Pipfile...'
           )
 
-          await command('pipenv install --site-packages', { shell: true })
+          await command('pipenv install --site-packages --dev', { shell: true })
           LogHelper.success('Python packages installed')
 
           LogHelper.info('Installing spaCy models...')
@@ -79,7 +82,7 @@ export default () =>
         } else {
           const dotProjectPath = path.join(
             process.cwd(),
-            'bridges/python/.venv/.project'
+            'bridges/python/src/.venv/.project'
           )
           if (fs.existsSync(dotProjectPath)) {
             const dotProjectMtime = fs.statSync(dotProjectPath).mtime
@@ -102,7 +105,7 @@ export default () =>
       }
     } else {
       LogHelper.error(
-        'bridges/python/Pipfile does not exist. Try to pull the project (git pull)'
+        'bridges/python/src/Pipfile does not exist. Try to pull the project (git pull)'
       )
       reject()
     }

scripts/setup/setup.js

@@ -17,7 +17,7 @@ import setupPythonPackages from './setup-python-packages'
 ;(async () => {
   try {
     // Required env vars to setup
-    process.env.PIPENV_PIPFILE = 'bridges/python/Pipfile'
+    process.env.PIPENV_PIPFILE = 'bridges/python/src/Pipfile'
     process.env.PIPENV_VENV_IN_PROJECT = 'true'
 
     await setupDotenv()

scripts/test-module.js

@@ -18,7 +18,7 @@ import { LoaderHelper } from '@/helpers/loader-helper'
     LoaderHelper.start()
     await command('npm run train en', { shell: true })
     const cmd = await command(
-      `cross-env PIPENV_PIPFILE=bridges/python/Pipfile LEON_NODE_ENV=testing jest --silent --config=./test/e2e/modules/e2e.modules.jest.json packages/${pkg}/test/${module}.spec.js && npm run train`,
+      `cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile LEON_NODE_ENV=testing jest --silent --config=./test/e2e/modules/e2e.modules.jest.json packages/${pkg}/test/${module}.spec.js && npm run train`,
       { shell: true }
     )
 

server/src/core/brain.js

@@ -199,8 +199,8 @@ class Brain {
              *
              * 1. Need to be at the root of the project
              * 2. Edit: server/src/intent-object.sample.json
-             * 3. Run: PIPENV_PIPFILE=bridges/python/Pipfile pipenv run
-             *    python bridges/python/main.py server/src/intent-object.sample.json
+             * 3. Run: PIPENV_PIPFILE=bridges/python/src/Pipfile pipenv run
+             *    python bridges/python/src/main.py server/src/intent-object.sample.json
              */
             const slots = {}
             if (obj.slots) {
@@ -225,7 +225,7 @@ class Brain {
             try {
               fs.writeFileSync(intentObjectPath, JSON.stringify(intentObj))
               this.process = spawn(
-                `./bridges/python/dist/python-bridge/python-bridge ${intentObjectPath}`,
+                `./bridges/python/dist/python-bridge/leon-python-bridge ${intentObjectPath}`,
                 { shell: true }
               )
             } catch (e) {

server/src/core/nlu.js

@@ -224,7 +224,7 @@ class Nlu {
     // Recreate a new TCP server process and reconnect the TCP client
     kill(global.tcpServerProcess.pid, () => {
       global.tcpServerProcess = spawn(
-        `pipenv run python bridges/python/tcp_server/main.py ${locale}`,
+        `./bridges/python/dist/tcp-server/leon-tcp-server ${locale}`,
         { shell: true }
       )
 

server/src/index.ts

@@ -13,7 +13,7 @@ import server from '@/core/http-server/server'
   process.title = 'leon'
 
   global.tcpServerProcess = spawn(
-    `pipenv run python bridges/python/tcp_server/main.py ${LangHelper.getShortCode(
+    `./bridges/python/dist/tcp-server/leon-tcp-server ${LangHelper.getShortCode(
       LEON_LANG
     )}`,
     {