mirror of
https://github.com/leon-ai/leon.git
synced 2024-08-16 21:50:33 +03:00
342 lines
11 KiB
JavaScript
342 lines
11 KiB
JavaScript
import fs from 'node:fs'
|
|
import path from 'node:path'
|
|
|
|
import { command } from 'execa'
|
|
|
|
import {
|
|
EN_SPACY_MODEL_NAME,
|
|
EN_SPACY_MODEL_VERSION,
|
|
FR_SPACY_MODEL_NAME,
|
|
FR_SPACY_MODEL_VERSION,
|
|
PYTHON_BRIDGE_SRC_PATH,
|
|
PYTHON_TCP_SERVER_SRC_PATH
|
|
} from '@/constants'
|
|
import { CPUArchitectures, OSTypes } from '@/types'
|
|
import { LogHelper } from '@/helpers/log-helper'
|
|
import { LoaderHelper } from '@/helpers/loader-helper'
|
|
import { SystemHelper } from '@/helpers/system-helper'
|
|
|
|
/**
|
|
* Set up development environment according to the given setup target
|
|
* 1. Verify Python environment
|
|
* 2. Verify if the targeted development environment is up-to-date
|
|
* 3. If up-to-date, exit
|
|
* 4. If not up-to-date, delete the outdated development environment and install the new one
|
|
* 5. Install spaCy models if the targeted development environment is the TCP server
|
|
*/
|
|
|
|
// Define mirror to download models installation file
|
|
function getModelInstallationFileUrl(model, mirror = undefined) {
|
|
const { name, version } = SPACY_MODELS.get(model)
|
|
const suffix = 'py3-none-any.whl'
|
|
let urlPrefix = 'https://github.com/explosion/spacy-models/releases/download'
|
|
|
|
if (mirror === 'cn') {
|
|
LogHelper.info(
|
|
'Using Chinese mirror to download model installation file...'
|
|
)
|
|
urlPrefix =
|
|
'https://download.fastgit.org/explosion/spacy-models/releases/download'
|
|
}
|
|
|
|
return `${urlPrefix}/${name}-${version}/${name}-${version}-${suffix}`
|
|
}
|
|
|
|
const SETUP_TARGETS = new Map()
|
|
const SPACY_MODELS = new Map()
|
|
|
|
SETUP_TARGETS.set('python-bridge', {
|
|
name: 'Python bridge',
|
|
pipfilePath: path.join(PYTHON_BRIDGE_SRC_PATH, 'Pipfile'),
|
|
dotVenvPath: path.join(PYTHON_BRIDGE_SRC_PATH, '.venv'),
|
|
dotProjectPath: path.join(PYTHON_BRIDGE_SRC_PATH, '.venv', '.project')
|
|
})
|
|
SETUP_TARGETS.set('tcp-server', {
|
|
name: 'TCP server',
|
|
pipfilePath: path.join(PYTHON_TCP_SERVER_SRC_PATH, 'Pipfile'),
|
|
dotVenvPath: path.join(PYTHON_TCP_SERVER_SRC_PATH, '.venv'),
|
|
dotProjectPath: path.join(PYTHON_TCP_SERVER_SRC_PATH, '.venv', '.project')
|
|
})
|
|
|
|
SPACY_MODELS.set('en', {
|
|
name: EN_SPACY_MODEL_NAME,
|
|
version: EN_SPACY_MODEL_VERSION
|
|
})
|
|
SPACY_MODELS.set('fr', {
|
|
name: FR_SPACY_MODEL_NAME,
|
|
version: FR_SPACY_MODEL_VERSION
|
|
})
|
|
;(async () => {
|
|
LoaderHelper.start()
|
|
|
|
const { argv } = process
|
|
const givenSetupTarget = argv[2].toLowerCase()
|
|
// cn
|
|
const givenMirror = argv[3]?.toLowerCase()
|
|
|
|
if (!SETUP_TARGETS.has(givenSetupTarget)) {
|
|
LogHelper.error(
|
|
`Invalid setup target: ${givenSetupTarget}. Valid targets are: ${Array.from(
|
|
SETUP_TARGETS.keys()
|
|
).join(', ')}`
|
|
)
|
|
process.exit(1)
|
|
}
|
|
|
|
const {
|
|
name: setupTarget,
|
|
pipfilePath,
|
|
dotVenvPath,
|
|
dotProjectPath
|
|
} = SETUP_TARGETS.get(givenSetupTarget)
|
|
|
|
LogHelper.info('Checking Python environment...')
|
|
|
|
/**
|
|
* Verify Python environment
|
|
*/
|
|
|
|
// Check if the Pipfile exists
|
|
if (fs.existsSync(pipfilePath)) {
|
|
LogHelper.success(`${pipfilePath} found`)
|
|
|
|
try {
|
|
// Check if Pipenv is installed
|
|
const pipenvVersionChild = await command('pipenv --version', {
|
|
shell: true
|
|
})
|
|
let pipenvVersion = String(pipenvVersionChild.stdout)
|
|
|
|
if (pipenvVersion.includes('version')) {
|
|
pipenvVersion = pipenvVersion.split('version')[1].trim()
|
|
pipenvVersion = `${pipenvVersion} version`
|
|
}
|
|
|
|
LogHelper.success(`Pipenv ${pipenvVersion} found`)
|
|
} catch (e) {
|
|
LogHelper.error(
|
|
`${e}\nPlease install Pipenv: "pip install pipenv" or read the documentation https://docs.pipenv.org`
|
|
)
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Install Python packages
|
|
*/
|
|
|
|
LogHelper.info(`Setting up ${setupTarget} development environment...`)
|
|
|
|
const pipfileMtime = fs.statSync(pipfilePath).mtime
|
|
const hasDotVenv = fs.existsSync(dotVenvPath)
|
|
const { type: osType, cpuArchitecture } = SystemHelper.getInformation()
|
|
/**
|
|
* Install PyTorch with CUDA support
|
|
* as it is required by the latest NVIDIA drivers for CUDA runtime APIs.
|
|
* PyTorch will automatically download nvidia-* packages and bundle them.
|
|
*
|
|
* It is important to specify the "--ignore-installed" flag to make sure the
|
|
* "~/.pyenv/versions/3.11.9/lib/python3.11/site-packages" is not used in case
|
|
* NVIDIA deps are already installed. Otherwise, it won't install it in our
|
|
* TCP server .venv as it is already installed (satisfied) in
|
|
* the path mentioned above
|
|
*
|
|
* @see https://github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix
|
|
* @see https://pytorch.org/get-started/locally/
|
|
* @see https://stackoverflow.com/a/76972265/1768162
|
|
* @see https://docs.nvidia.com/deeplearning/cudnn/latest/reference/support-matrix.html
|
|
*/
|
|
const installPytorch = async () => {
|
|
const logInfo =
|
|
osType === OSTypes.MacOS
|
|
? 'Installing PyTorch...'
|
|
: 'Installing PyTorch with CUDA support...'
|
|
LogHelper.info(logInfo)
|
|
|
|
try {
|
|
// There is no CUDA support on macOS
|
|
const commandToExecute =
|
|
osType === OSTypes.MacOS
|
|
? 'pipenv run pip install --ignore-installed --force-reinstall torch==2.3.1'
|
|
: 'pipenv run pip install --ignore-installed --force-reinstall torch==2.3.1 --index-url https://download.pytorch.org/whl/cu121'
|
|
|
|
await command(commandToExecute, {
|
|
shell: true,
|
|
stdio: 'inherit'
|
|
})
|
|
LogHelper.success('PyTorch with CUDA support installed')
|
|
} catch (e) {
|
|
LogHelper.error(`Failed to install PyTorch with CUDA support: ${e}`)
|
|
process.exit(1)
|
|
}
|
|
}
|
|
const installPythonPackages = async () => {
|
|
LogHelper.info(`Installing Python packages from ${pipfilePath}...`)
|
|
|
|
// Delete .venv directory to reset the development environment
|
|
if (hasDotVenv) {
|
|
LogHelper.info(`Deleting ${dotVenvPath}...`)
|
|
await fs.promises.rm(dotVenvPath, { recursive: true, force: true })
|
|
LogHelper.success(`${dotVenvPath} deleted`)
|
|
}
|
|
|
|
try {
|
|
await command('pipenv install --verbose --skip-lock', {
|
|
shell: true,
|
|
stdio: 'inherit'
|
|
})
|
|
|
|
if (
|
|
osType === OSTypes.MacOS &&
|
|
cpuArchitecture === CPUArchitectures.ARM64
|
|
) {
|
|
LogHelper.info('macOS ARM64 detected')
|
|
|
|
LogHelper.info(
|
|
'Installing Rust installer as it is needed for the "tokenizers" package for macOS ARM64 architecture...'
|
|
)
|
|
await command('curl https://sh.rustup.rs -sSf | sh -s -- -y', {
|
|
shell: true,
|
|
stdio: 'inherit'
|
|
})
|
|
LogHelper.success('Rust installer installed')
|
|
|
|
LogHelper.info('Reloading configuration from "$HOME/.cargo/env"...')
|
|
await command('source "$HOME/.cargo/env"', {
|
|
shell: true,
|
|
stdio: 'inherit'
|
|
})
|
|
LogHelper.success('Configuration reloaded')
|
|
|
|
LogHelper.info('Checking Rust compiler version...')
|
|
await command('rustc --version', {
|
|
shell: true,
|
|
stdio: 'inherit'
|
|
})
|
|
LogHelper.success('Rust compiler OK')
|
|
}
|
|
|
|
LogHelper.success('Python packages installed')
|
|
|
|
if (givenSetupTarget === 'tcp-server') {
|
|
await installPytorch()
|
|
}
|
|
} catch (e) {
|
|
if (hasDotVenv) {
|
|
await fs.promises.rm(dotVenvPath, { recursive: true, force: true })
|
|
LogHelper.info(`Error occurred, so "${dotVenvPath}" was deleted`)
|
|
}
|
|
|
|
LogHelper.error(`Failed to install Python packages: ${e}`)
|
|
|
|
if (osType === OSTypes.Linux || osType === OSTypes.MacOS) {
|
|
LogHelper.error(
|
|
'If the error is related to "PortAudio" not installed or found, you can install it by running: "sudo apt install portaudio19-dev" or "brew install portaudio". Then retry. PortAudio is required for the "pyaudio" package used to record audio'
|
|
)
|
|
}
|
|
|
|
if (osType === OSTypes.Windows) {
|
|
LogHelper.error(
|
|
'Please check the error above. It might be related to Microsoft C++ Build Tools. If it is, you can check here: "https://stackoverflow.com/a/64262038/1768162" then restart your machine and retry'
|
|
)
|
|
LogHelper.error(
|
|
'If it is related to some hash mismatch, you can try by installing Pipenv 2024.0.1: pip install pipenv==2024.0.1'
|
|
)
|
|
}
|
|
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Verify if a fresh development environment installation is necessary
|
|
*/
|
|
|
|
// Required environment variables to set up
|
|
process.env.PIPENV_PIPFILE = pipfilePath
|
|
process.env.PIPENV_VENV_IN_PROJECT = true
|
|
|
|
if (givenSetupTarget === 'python-bridge') {
|
|
// As per: https://github.com/marcelotduarte/cx_Freeze/issues/1548
|
|
process.env.PIP_NO_BINARY = 'cx_Freeze'
|
|
}
|
|
|
|
try {
|
|
if (!hasDotVenv) {
|
|
await installPythonPackages()
|
|
} else {
|
|
if (fs.existsSync(dotProjectPath)) {
|
|
const dotProjectMtime = (await fs.promises.stat(dotProjectPath)).mtime
|
|
|
|
// Check if Python deps tree has been modified since the initial setup
|
|
if (pipfileMtime > dotProjectMtime) {
|
|
LogHelper.info('The development environment is not up-to-date')
|
|
await installPythonPackages()
|
|
} else {
|
|
LogHelper.success('Python packages are up-to-date')
|
|
}
|
|
} else {
|
|
await installPythonPackages()
|
|
}
|
|
}
|
|
} catch (e) {
|
|
LogHelper.error(
|
|
`Failed to set up the ${setupTarget} development environment: ${e}`
|
|
)
|
|
} finally {
|
|
LoaderHelper.stop()
|
|
}
|
|
|
|
if (givenSetupTarget === 'tcp-server') {
|
|
const installSpacyModels = async () => {
|
|
try {
|
|
LogHelper.info('Installing spaCy models...')
|
|
|
|
// Install models one by one to avoid network throttling
|
|
for (const modelLanguage of SPACY_MODELS.keys()) {
|
|
const modelInstallationFileUrl = getModelInstallationFileUrl(
|
|
modelLanguage,
|
|
givenMirror
|
|
)
|
|
|
|
await command(`pipenv run pip install ${modelInstallationFileUrl}`, {
|
|
shell: true,
|
|
stdio: 'inherit'
|
|
})
|
|
}
|
|
|
|
LogHelper.success('spaCy models installed')
|
|
} catch (e) {
|
|
LogHelper.error(`Failed to install spaCy models: ${e}`)
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
LogHelper.info('Checking whether all spaCy models are installed...')
|
|
|
|
try {
|
|
for (const { name: modelName } of SPACY_MODELS.values()) {
|
|
const { stderr } = await command(
|
|
`pipenv run python -c "import ${modelName}"`,
|
|
{ shell: true }
|
|
)
|
|
|
|
// Check stderr output for Windows as no exception is thrown
|
|
if (osType === OSTypes.Windows) {
|
|
if (String(stderr).length > 0) {
|
|
await installSpacyModels()
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
LogHelper.success('All spaCy models are already installed')
|
|
} catch (e) {
|
|
LogHelper.info('Not all spaCy models are installed')
|
|
await installSpacyModels()
|
|
}
|
|
}
|
|
|
|
LogHelper.success(`${setupTarget} development environment ready`)
|
|
})()
|