feat: pack TCP server and optimize setup scripts

package.json

@@ -34,8 +34,8 @@
     "setup:offline-stt": "ts-node scripts/setup-offline/run-setup-stt.js",
     "setup:offline-tts": "ts-node scripts/setup-offline/run-setup-tts.js",
     "setup:offline-hotword": "ts-node scripts/setup-offline/run-setup-hotword.js",
-    "setup:python-bridge": "ts-node scripts/python-bridge/setup-python-bridge.js",
-    "setup:tcp-server": "ts-node scripts/tcp-server/setup-tcp-server.js",
+    "setup:python-bridge": "ts-node scripts/setup/setup-python-packages.js python-bridge",
+    "setup:tcp-server": "ts-node scripts/setup/setup-python-packages.js tcp-server",
     "preinstall": "node scripts/setup/preinstall.js",
     "postinstall": "ts-node scripts/setup/setup.js",
     "dev:app": "vite --config app/vite.config.js",
@@ -49,7 +49,7 @@
     "build": "npm run build:app && npm run build:server",
     "build:app": "cross-env LEON_NODE_ENV=production ts-node scripts/app/run-build-app.js",
     "build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && tsc && resolve-tspaths && shx rm -rf server/dist/core server/dist/package.json && shx mv -f server/dist/server/src/* server/dist && shx rm -rf server/dist/server && shx mkdir -p server/dist/tmp",
-    "build:python-bridge": "ts-node scripts/python-bridge/build-python-bridge.js",
+    "build:python-bridge": "ts-node scripts/build-python-bridge.js",
     "start:tcp-server": "cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile pipenv run python tcp_server/src/main.py",
     "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
     "train": "ts-node scripts/train/run-train.js",

scripts/build-tcp-server.js

@@ -0,0 +1,102 @@
+import path from 'node:path'
+import fs from 'node:fs'
+
+import { command } from 'execa'
+import archiver from 'archiver'
+
+import { LogHelper } from '@/helpers/log-helper'
+import { LoaderHelper } from '@/helpers/loader-helper'
+import { OSHelper, OSTypes } from '@/helpers/os-helper'
+
+/**
+ * Build TCP server
+ * 1. Get the correct OS platform and CPU architecture
+ * 2. If Linux, install the required dependencies
+ * 3. Install spaCy models
+ * 4. Build the Python bridge
+ * 5. Pack the distribution entities into a ZIP file
+ */
+
+const PIPFILE_PATH = 'bridges/python/src/Pipfile'
+const SETUP_FILE_PATH = 'bridges/python/src/setup.py'
+const DIST_PATH = 'bridges/python/dist'
+
+const BINARIES_FOLDER_NAME = OSHelper.getBinariesFolderName()
+const BUILD_PATH = path.join(DIST_PATH, BINARIES_FOLDER_NAME)
+
+const ARCHIVE_NAME = `leon-python-bridge-${BINARIES_FOLDER_NAME}.zip`
+const ARCHIVE_PATH = path.join(DIST_PATH, ARCHIVE_NAME)
+
+;(async () => {
+  LoaderHelper.start()
+
+  process.env.PIPENV_PIPFILE = PIPFILE_PATH
+  process.env.PIPENV_VENV_IN_PROJECT = true
+
+  const { type } = OSHelper.getInformation()
+
+  /**
+   * Install requirements
+   */
+  try {
+    if (type === OSTypes.Linux) {
+      LogHelper.info('Checking whether the "patchelf" utility can be found...')
+
+      await command('patchelf --version', { shell: true })
+
+      LogHelper.success('The "patchelf" utility has been found')
+    }
+  } catch (e) {
+    const installPatchelfCommand = 'sudo apt install patchelf'
+
+    LogHelper.error(
+      `The "patchelf" utility is not installed. Please run the following command: "${installPatchelfCommand}" or install it via a packages manager supported by your Linux distribution such as DNF, YUM, etc. Then try again`
+    )
+    process.exit(1)
+  }
+
+  /**
+   * Build TCP server
+   */
+  try {
+    LogHelper.info('Building the TCP server...')
+
+    await command(
+      `pipenv run python ${SETUP_FILE_PATH} build --build-exe ${BUILD_PATH}`,
+      {
+        shell: true
+      }
+    )
+
+    LogHelper.success('Python bridge built')
+  } catch (e) {
+    LogHelper.error(`Failed to build the Python bridge: ${e}`)
+  }
+
+  /**
+   * Pack distribution entities into a ZIP archive
+   */
+  LogHelper.info(`Packing to ${ARCHIVE_PATH}...`)
+
+  const output = fs.createWriteStream(ARCHIVE_PATH)
+  const archive = archiver('zip')
+
+  output.on('close', () => {
+    LogHelper.success(`Python bridge packed to ${ARCHIVE_PATH}`)
+    LogHelper.success(
+      `Python bridge archive size: ${(archive.pointer() / 1_000_000).toFixed(
+        1
+      )} MB`
+    )
+    process.exit(0)
+  })
+
+  archive.on('error', (err) => {
+    LogHelper.error(`Failed to pack the Python bridge: ${err}`)
+  })
+
+  archive.pipe(output)
+  archive.directory(BUILD_PATH, BINARIES_FOLDER_NAME)
+
+  await archive.finalize()
+})()

scripts/python-bridge/setup-python-bridge.js

@@ -1,37 +0,0 @@
-import { command } from 'execa'
-
-import { LogHelper } from '@/helpers/log-helper'
-import { LoaderHelper } from '@/helpers/loader-helper'
-
-/**
- * Set up Python bridge development environment
- * 1. Install Python packages from the Pipfile
- */
-
-const PIPFILE_PATH = 'bridges/python/src/Pipfile'
-
-;(async () => {
-  LoaderHelper.start()
-  LogHelper.info('Setting up Python bridge development environment...')
-  LogHelper.info(`Installing Python packages from ${PIPFILE_PATH}.lock...`)
-
-  try {
-    process.env.PIPENV_PIPFILE = PIPFILE_PATH
-    process.env.PIPENV_VENV_IN_PROJECT = true
-    // As per: https://github.com/marcelotduarte/cx_Freeze/issues/1548
-    process.env.PIP_NO_BINARY = 'cx_Freeze'
-
-    await command(`pipenv install --site-packages`, {
-      shell: true
-    })
-
-    LogHelper.success('Python packages installed')
-    LogHelper.success('Python bridge development environment ready')
-  } catch (e) {
-    LogHelper.error(
-      `Failed to set up the Python bridge development environment: ${e}`
-    )
-  } finally {
-    LoaderHelper.stop()
-  }
-})()

scripts/setup/setup-python-packages.js

@@ -4,109 +4,225 @@ import path from 'node:path'
 import { command } from 'execa'
 
 import { LogHelper } from '@/helpers/log-helper'
+import { LoaderHelper } from '@/helpers/loader-helper'
 
 /**
- * Download and setup Leon's Python packages dependencies
+ * Set up development environment according to the given setup target
+ * 1. Verify Python environment
+ * 2. Verify if the targeted development environment is up-to-date
+ * 3. If up-to-date, exit
+ * 4. If not up-to-date, delete the outdated development environment and install the new one
+ * 5. Install spaCy models if the targeted development environment is the TCP server
  */
-export default () =>
-  new Promise(async (resolve, reject) => {
-    LogHelper.info('Checking Python env...')
 
-    // Check if the Pipfile exists
-    if (fs.existsSync('bridges/python/src/Pipfile')) {
-      LogHelper.success('bridges/python/src/Pipfile found')
+const SETUP_TARGETS = new Map()
+// Find new spaCy models:  https://github.com/explosion/spacy-models/releases
+const SPACY_MODELS = ['en_core_web_trf-3.4.0', 'fr_core_news_md-3.4.0']
+const PYTHON_BRIDGE_SRC_PATH = 'bridges/python/src'
+const TCP_SERVER_SRC_PATH = 'tcp_server/src'
 
-      try {
-        // Check if Pipenv is installed
-        const pipenvVersionChild = await command('pipenv --version', {
-          shell: true
-        })
-        let pipenvVersion = pipenvVersionChild.stdout
+SETUP_TARGETS.set('python-bridge', {
+  name: 'Python bridge',
+  pipfilePath: path.join(PYTHON_BRIDGE_SRC_PATH, 'Pipfile'),
+  dotVenvPath: path.join(PYTHON_BRIDGE_SRC_PATH, '.venv'),
+  dotProjectPath: path.join(PYTHON_BRIDGE_SRC_PATH, '.venv', '.project')
+})
+SETUP_TARGETS.set('tcp-server', {
+  name: 'TCP server',
+  pipfilePath: path.join(TCP_SERVER_SRC_PATH, 'Pipfile'),
+  dotVenvPath: path.join(TCP_SERVER_SRC_PATH, '.venv'),
+  dotProjectPath: path.join(TCP_SERVER_SRC_PATH, '.venv', '.project')
+})
+;(async () => {
+  LoaderHelper.start()
 
-        if (pipenvVersion.indexOf('version') !== -1) {
-          pipenvVersion = pipenvVersion.substr(
-            pipenvVersion.indexOf('version') + 'version '.length
-          )
-          pipenvVersion = `${pipenvVersion} version`
-        }
+  const { argv } = process
+  const givenSetupTarget = argv[2].toLowerCase()
 
-        LogHelper.success(`Pipenv ${pipenvVersion} found`)
-      } catch (e) {
-        LogHelper.error(
-          `${e}\nPlease install Pipenv: "pip install pipenv" or read the documentation https://docs.pipenv.org`
-        )
-        reject(e)
+  if (!SETUP_TARGETS.has(givenSetupTarget)) {
+    LogHelper.error(
+      `Invalid setup target: ${givenSetupTarget}. Valid targets are: ${Array.from(
+        SETUP_TARGETS.keys()
+      ).join(', ')}`
+    )
+    process.exit(1)
+  }
+
+  const {
+    name: setupTarget,
+    pipfilePath,
+    dotVenvPath,
+    dotProjectPath
+  } = SETUP_TARGETS.get(givenSetupTarget)
+
+  LogHelper.info('Checking Python environment...')
+
+  /**
+   * Verify Python environment
+   */
+
+  // Check if the Pipfile exists
+  if (fs.existsSync(pipfilePath)) {
+    LogHelper.success(`${pipfilePath} found`)
+
+    try {
+      // Check if Pipenv is installed
+      const pipenvVersionChild = await command('pipenv --version', {
+        shell: true
+      })
+      let pipenvVersion = String(pipenvVersionChild.stdout)
+
+      if (pipenvVersion.includes('version')) {
+        pipenvVersion = pipenvVersion.split('version')[1].trim()
+        pipenvVersion = `${pipenvVersion} version`
       }
 
-      try {
-        const dotVenvPath = path.join(process.cwd(), 'bridges/python/src/.venv')
-        const pipfilePath = path.join(
-          process.cwd(),
-          'bridges/python/src/Pipfile'
-        )
-        const pipfileMtime = fs.statSync(pipfilePath).mtime
-        const isDotVenvExist = fs.existsSync(dotVenvPath)
-        const installPythonPackages = async () => {
-          if (isDotVenvExist) {
-            LogHelper.info(`Deleting ${dotVenvPath}...`)
-            fs.rmSync(dotVenvPath, { recursive: true, force: true })
-            LogHelper.success(`${dotVenvPath} deleted`)
-          }
+      LogHelper.success(`Pipenv ${pipenvVersion} found`)
+    } catch (e) {
+      LogHelper.error(
+        `${e}\nPlease install Pipenv: "pip install pipenv" or read the documentation https://docs.pipenv.org`
+      )
+      process.exit(1)
+    }
+  }
 
-          // Installing Python packages
-          LogHelper.info(
-            'Installing Python packages from bridges/python/src/Pipfile...'
-          )
+  /**
+   * Install Python packages
+   */
 
-          await command('pipenv install --site-packages', { shell: true })
-          LogHelper.success('Python packages installed')
+  LogHelper.info(`Setting up ${setupTarget} development environment...`)
 
-          LogHelper.info('Installing spaCy models...')
-          // Find new spaCy models:  https://github.com/explosion/spacy-models/releases
-          await Promise.all([
-            command(
-              'pipenv run spacy download en_core_web_trf-3.4.0 --direct',
-              { shell: true }
-            ),
-            command(
-              'pipenv run spacy download fr_core_news_md-3.4.0 --direct',
-              { shell: true }
-            )
-          ])
+  const pipfileMtime = fs.statSync(pipfilePath).mtime
+  const hasDotVenv = fs.existsSync(dotVenvPath)
+  const installPythonPackages = async () => {
+    LogHelper.info(`Installing Python packages from ${pipfilePath}.lock...`)
 
-          LogHelper.success('spaCy models installed')
+    // Delete .venv directory to reset the development environment
+    if (hasDotVenv) {
+      LogHelper.info(`Deleting ${dotVenvPath}...`)
+      fs.rmSync(dotVenvPath, { recursive: true, force: true })
+      LogHelper.success(`${dotVenvPath} deleted`)
+    }
+
+    try {
+      await command(`pipenv install --site-packages`, {
+        shell: true,
+        stdio: 'inherit'
+      })
+
+      LogHelper.success('Python packages installed')
+    } catch (e) {
+      LogHelper.error(`Failed to install Python packages: ${e}`)
+      process.exit(1)
+    }
+
+    try {
+      if (givenSetupTarget === 'tcp-server') {
+        LogHelper.info('Installing spaCy models...')
+
+        // Install models one by one to avoid network throttling
+        for (const model of SPACY_MODELS) {
+          await command(`pipenv run spacy download ${model} --direct`, {
+            shell: true,
+            stdio: 'inherit'
+          })
         }
 
-        if (!isDotVenvExist) {
+        LogHelper.success('spaCy models installed')
+      }
+    } catch (e) {
+      LogHelper.error(`Failed to install spaCy models: ${e}`)
+      process.exit(1)
+    }
+  }
+
+  /**
+   * Verify if a fresh development environment installation is necessary
+   */
+
+  try {
+    // Required environment variables to set up
+    process.env.PIPENV_PIPFILE = pipfilePath
+    process.env.PIPENV_VENV_IN_PROJECT = true
+
+    if (givenSetupTarget === 'python-bridge') {
+      // As per: https://github.com/marcelotduarte/cx_Freeze/issues/1548
+      process.env.PIP_NO_BINARY = 'cx_Freeze'
+    }
+
+    if (!hasDotVenv) {
+      await installPythonPackages()
+    } else if (givenSetupTarget === 'tcp-server') {
+      /**
+       * When TCP server setup target, verify whether all spaCy models are installed
+       */
+      let hasAllSpacyModelsInstalled = false
+
+      if (givenSetupTarget === 'tcp-server') {
+        LogHelper.info('Checking whether all spaCy models are installed...')
+
+        const toGraphModelName = (model) => {
+          const [name, version] = model.split('-')
+
+          return `${name.replaceAll('_', '-')}==${version}`
+        }
+        let { stdout: graph } = await command('pipenv graph', { shell: true })
+        graph = graph.split('\n')
+
+        const models = new Map()
+        SPACY_MODELS.forEach((model) =>
+          models.set(toGraphModelName(model), { isInstalled: false })
+        )
+
+        SPACY_MODELS.forEach((model) => {
+          /**
+           * From the deps graph, models appear like this: fr-core-news-md==3.4.0
+           * So we need to comply to this format on the fly
+           */
+
+          model = toGraphModelName(model)
+
+          if (graph.includes(model)) {
+            models.set(model, { isInstalled: true })
+          }
+        })
+
+        for (const { isInstalled } of models.values()) {
+          if (!isInstalled) {
+            break
+          }
+
+          hasAllSpacyModelsInstalled = true
+        }
+
+        if (!hasAllSpacyModelsInstalled) {
+          LogHelper.info('Not all spaCy models are installed')
           await installPythonPackages()
         } else {
-          const dotProjectPath = path.join(
-            process.cwd(),
-            'bridges/python/src/.venv/.project'
-          )
-          if (fs.existsSync(dotProjectPath)) {
-            const dotProjectMtime = fs.statSync(dotProjectPath).mtime
-
-            // Check if Python deps tree has been modified since the initial setup
-            if (pipfileMtime > dotProjectMtime) {
-              await installPythonPackages()
-            } else {
-              LogHelper.success('Python packages are up-to-date')
-            }
-          } else {
-            await installPythonPackages()
-          }
+          LogHelper.success('All spaCy models are already installed')
         }
+      }
+    }
 
-        resolve()
-      } catch (e) {
-        LogHelper.error(`Failed to install the Python packages: ${e}`)
-        reject(e)
+    if (fs.existsSync(dotProjectPath)) {
+      const dotProjectMtime = fs.statSync(dotProjectPath).mtime
+
+      // Check if Python deps tree has been modified since the initial setup
+      if (pipfileMtime > dotProjectMtime) {
+        await installPythonPackages()
+      } else {
+        LogHelper.success('Python packages are up-to-date')
       }
     } else {
-      LogHelper.error(
-        'bridges/python/src/Pipfile does not exist. Try to pull the project (git pull)'
-      )
-      reject()
+      await installPythonPackages()
     }
-  })
+
+    LogHelper.success(`${setupTarget} development environment ready`)
+  } catch (e) {
+    LogHelper.error(
+      `Failed to set up the ${setupTarget} development environment: ${e}`
+    )
+  } finally {
+    LoaderHelper.stop()
+  }
+})()

scripts/setup/setup.js

@@ -7,7 +7,7 @@ import generateHttpApiKey from '../generate/generate-http-api-key'
 import setupDotenv from './setup-dotenv'
 import setupCore from './setup-core'
 import setupSkillsConfig from './setup-skills-config'
-import setupPythonPackages from './setup-python-packages'
+// import setupPythonPackages from './setup-python-packages'
 
 // Do not load ".env" file because it is not created yet
 
@@ -16,14 +16,12 @@ import setupPythonPackages from './setup-python-packages'
  */
 ;(async () => {
   try {
-    // Required env vars to setup
-    process.env.PIPENV_PIPFILE = 'bridges/python/src/Pipfile'
-    process.env.PIPENV_VENV_IN_PROJECT = 'true'
-
     await setupDotenv()
     LoaderHelper.start()
     await Promise.all([setupCore(), setupSkillsConfig()])
-    await setupPythonPackages()
+    // TODO: download Python bridge archive; unpack it; remove archive
+    // TODO: download TCP server archive; unpack it; remove archive
+    // await setupPythonPackages()
     LoaderHelper.stop()
     await generateHttpApiKey()
     LoaderHelper.start()

scripts/tcp-server/setup-tcp-server.js

@@ -1,35 +0,0 @@
-import { command } from 'execa'
-
-import { LogHelper } from '@/helpers/log-helper'
-import { LoaderHelper } from '@/helpers/loader-helper'
-
-/**
- * Set up TCP server development environment
- * 1. Install Python packages from the Pipfile
- */
-
-const PIPFILE_PATH = 'tcp_server/src/Pipfile'
-
-;(async () => {
-  LoaderHelper.start()
-  LogHelper.info('Setting up TCP server development environment...')
-  LogHelper.info(`Installing Python packages from ${PIPFILE_PATH}.lock...`)
-
-  try {
-    process.env.PIPENV_PIPFILE = PIPFILE_PATH
-    process.env.PIPENV_VENV_IN_PROJECT = true
-
-    await command(`pipenv install --site-packages`, {
-      shell: true
-    })
-
-    LogHelper.success('Python packages installed')
-    LogHelper.success('TCP server development environment ready')
-  } catch (e) {
-    LogHelper.error(
-      `Failed to set up the TCP server development environment: ${e}`
-    )
-  } finally {
-    LoaderHelper.stop()
-  }
-})()