gpt4all/.circleci/continue_config.yml
Jared Van Bortel 6c8a44f6c4
ci: use aws s3 sync to upload docs (#2172)
Signed-off-by: Jared Van Bortel <jared@nomic.ai>
2024-03-27 11:03:10 -04:00

1208 lines
46 KiB
YAML

version: 2.1
orbs:
win: circleci/windows@5.0
python: circleci/python@1.2
node: circleci/node@5.1
parameters:
run-all-workflows:
type: boolean
default: false
run-default-workflow:
type: boolean
default: false
run-python-workflow:
type: boolean
default: false
run-chat-workflow:
type: boolean
default: false
run-ts-workflow:
type: boolean
default: false
run-csharp-workflow:
type: boolean
default: false
jobs:
default-job:
docker:
- image: circleci/python:3.7
steps:
- run: echo "CircleCI pipeline triggered"
build-offline-chat-installer-macos:
macos:
xcode: 14.0.0
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
keys:
- macos-qt-cache-v3
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run:
name: Build
command: |
mkdir build
cd build
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
-DCMAKE_GENERATOR:STRING=Ninja \
-DBUILD_UNIVERSAL=ON \
-DMACDEPLOYQT=~/Qt/6.5.1/macos/bin/macdeployqt \
-DGPT4ALL_OFFLINE_INSTALLER=ON \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-S ../gpt4all-chat \
-B .
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
mkdir upload
cp gpt4all-installer-* upload
- store_artifacts:
path: build/upload
build-offline-chat-installer-linux:
machine:
image: ubuntu-2204:2023.04.2
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
keys:
- linux-qt-cache-v2
- run:
name: Setup Linux and Dependencies
command: |
wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run:
name: Build linuxdeployqt
command: |
git clone https://github.com/nomic-ai/linuxdeployqt
cd linuxdeployqt && qmake && sudo make install
- run:
name: Build
command: |
set -eo pipefail
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
mkdir build
cd build
mkdir upload
~/Qt/Tools/CMake/bin/cmake -DGPT4ALL_OFFLINE_INSTALLER=ON -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
~/Qt/Tools/CMake/bin/cmake --build . --target all
~/Qt/Tools/CMake/bin/cmake --build . --target install
~/Qt/Tools/CMake/bin/cmake --build . --target package
cp gpt4all-installer-* upload
- store_artifacts:
path: build/upload
build-offline-chat-installer-windows:
machine:
image: 'windows-server-2019-vs2019:2022.08.1'
resource_class: windows.large
shell: powershell.exe -ExecutionPolicy Bypass
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
keys:
- windows-qt-cache-v2
- run:
name: Installing Qt
command: |
if (-not (Test-Path C:\Qt)) {
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run:
name: Install VulkanSDK
command: |
Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
.\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
- run:
name: Build
command: |
$Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
$Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
$Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
$Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
$Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
mkdir build
cd build
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
"-DCMAKE_GENERATOR:STRING=Ninja" `
"-DCMAKE_BUILD_TYPE=Release" `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
"-DGPT4ALL_OFFLINE_INSTALLER=ON" `
"-S ..\gpt4all-chat" `
"-B ."
& "C:\Qt\Tools\Ninja\ninja.exe"
& "C:\Qt\Tools\Ninja\ninja.exe" install
& "C:\Qt\Tools\Ninja\ninja.exe" package
mkdir upload
copy gpt4all-installer-win64.exe upload
- store_artifacts:
path: build/upload
build-gpt4all-chat-linux:
machine:
image: ubuntu-2204:2023.04.2
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
keys:
- linux-qt-cache-v2
- run:
name: Setup Linux and Dependencies
command: |
wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run:
name: Build
command: |
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S gpt4all-chat -B build
~/Qt/Tools/CMake/bin/cmake --build build --target all
build-gpt4all-chat-windows:
machine:
image: 'windows-server-2019-vs2019:2022.08.1'
resource_class: windows.large
shell: powershell.exe -ExecutionPolicy Bypass
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
keys:
- windows-qt-cache-v2
- run:
name: Installing Qt
command: |
if (-not (Test-Path C:\Qt)) {
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run:
name: Install VulkanSDK
command: |
Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
.\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
- run:
name: Build
command: |
$Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
$Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
$Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
$Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
$Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
$Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
"-DCMAKE_GENERATOR:STRING=Ninja" `
"-DCMAKE_BUILD_TYPE=Release" `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
"-S gpt4all-chat" `
"-B build"
& "C:\Qt\Tools\Ninja\ninja.exe" -C build
build-gpt4all-chat-macos:
macos:
xcode: 14.0.0
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
keys:
- macos-qt-cache-v3
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run:
name: Build
command: |
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
-DCMAKE_GENERATOR:STRING=Ninja \
-DBUILD_UNIVERSAL=ON \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-S gpt4all-chat \
-B build
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build build --target all
build-ts-docs:
docker:
- image: cimg/base:stable
steps:
- checkout
- node/install:
node-version: "18.16"
- run: node --version
- run: corepack enable
- node/install-packages:
pkg-manager: npm
app-dir: gpt4all-bindings/typescript
override-ci-command: npm install --ignore-scripts
- run:
name: build docs ts yo
command: |
cd gpt4all-bindings/typescript
npm run docs:build
build-py-docs:
docker:
- image: circleci/python:3.8
steps:
- checkout
- run:
name: Install dependencies
command: |
sudo apt-get update
sudo apt-get -y install python3 python3-pip
sudo pip3 install awscli --upgrade
sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
- run:
name: Make Documentation
command: |
cd gpt4all-bindings/python
mkdocs build
- run:
name: Deploy Documentation
command: |
cd gpt4all-bindings/python
aws s3 sync --delete site/ s3://docs.gpt4all.io/
- run:
name: Invalidate docs.gpt4all.io cloudfront
command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
build-py-linux:
machine:
image: ubuntu-2204:2023.04.2
steps:
- checkout
- run:
name: Set Python Version
command: pyenv global 3.11.2
- run:
name: Install dependencies
command: |
wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
sudo apt-get update
sudo apt-get install -y cmake build-essential vulkan-sdk
pip install setuptools wheel cmake
- run:
name: Build C library
command: |
git submodule update --init --recursive
cd gpt4all-backend
cmake -B build
cmake --build build --parallel
- run:
name: Build wheel
command: |
cd gpt4all-bindings/python/
python setup.py bdist_wheel --plat-name=manylinux1_x86_64
- store_artifacts:
path: gpt4all-bindings/python/dist
- persist_to_workspace:
root: gpt4all-bindings/python/dist
paths:
- "*.whl"
build-py-macos:
macos:
xcode: "14.2.0"
resource_class: macos.m1.large.gen1
steps:
- checkout
- run:
name: Install dependencies
command: |
brew install cmake
pip install setuptools wheel cmake
- run:
name: Build C library
command: |
git submodule update --init # don't use --recursive because macOS doesn't use Kompute
cd gpt4all-backend
cmake -B build -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
cmake --build build --parallel
- run:
name: Build wheel
command: |
cd gpt4all-bindings/python
python setup.py bdist_wheel --plat-name=macosx_10_15_universal2
- store_artifacts:
path: gpt4all-bindings/python/dist
- persist_to_workspace:
root: gpt4all-bindings/python/dist
paths:
- "*.whl"
build-py-windows:
executor:
name: win/default
steps:
- checkout
- run:
name: Install MinGW64
command: choco install -y mingw --force --no-progress
- run:
name: Install VulkanSDK
command: |
Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
.\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
- run:
name: Install dependencies
command:
choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
- run:
name: Install Python dependencies
command: pip install setuptools wheel cmake
- run:
name: Build C library
command: |
git submodule update --init --recursive
cd gpt4all-backend
$Env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
$Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
cmake -G "MinGW Makefiles" -B build -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
cmake --build build --parallel
- run:
name: Build wheel
# TODO: As part of this task, we need to move mingw64 binaries into package.
# This is terrible and needs a more robust solution eventually.
command: |
cd gpt4all-bindings/python
cd gpt4all
mkdir llmodel_DO_NOT_MODIFY
mkdir llmodel_DO_NOT_MODIFY/build/
cp 'C:\ProgramData\mingw64\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
cd ..
python setup.py bdist_wheel --plat-name=win_amd64
- store_artifacts:
path: gpt4all-bindings/python/dist
- persist_to_workspace:
root: gpt4all-bindings/python/dist
paths:
- "*.whl"
store-and-upload-wheels:
docker:
- image: circleci/python:3.8
steps:
- setup_remote_docker
- attach_workspace:
at: /tmp/workspace
- run:
name: Install dependencies
command: |
sudo apt-get update
sudo apt-get install -y cmake build-essential
pip install setuptools wheel twine
- run:
name: Upload Python package
command: |
twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
- store_artifacts:
path: /tmp/workspace
build-bindings-backend-linux:
machine:
image: ubuntu-2204:2023.04.2
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- run:
name: Install dependencies
command: |
wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
sudo apt-get update
sudo apt-get install -y cmake build-essential vulkan-sdk
- run:
name: Build Libraries
command: |
cd gpt4all-backend
mkdir -p runtimes/build
cd runtimes/build
cmake ../..
cmake --build . --parallel --config Release
mkdir ../linux-x64
cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
- persist_to_workspace:
root: gpt4all-backend
paths:
- runtimes/linux-x64/*.so
build-bindings-backend-macos:
macos:
xcode: "14.0.0"
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- run:
name: Install dependencies
command: |
brew install cmake
- run:
name: Build Libraries
command: |
cd gpt4all-backend
mkdir -p runtimes/build
cd runtimes/build
cmake ../.. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
cmake --build . --parallel --config Release
mkdir ../osx-x64
cp -L *.dylib ../osx-x64
cp ../../llama.cpp-mainline/*.metal ../osx-x64
ls ../osx-x64
- persist_to_workspace:
root: gpt4all-backend
paths:
- runtimes/osx-x64/*.dylib
- runtimes/osx-x64/*.metal
build-bindings-backend-windows:
executor:
name: win/default
size: large
shell: powershell.exe -ExecutionPolicy Bypass
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- run:
name: Install MinGW64
command: choco install -y mingw --force --no-progress
- run:
name: Install VulkanSDK
command: |
Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
.\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
- run:
name: Install dependencies
command: |
choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
- run:
name: Build Libraries
command: |
$MinGWBin = "C:\ProgramData\mingw64\mingw64\bin"
$Env:Path += ";$MinGwBin"
$Env:Path += ";C:\Program Files\CMake\bin"
$Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
cd gpt4all-backend
mkdir runtimes/win-x64
cd runtimes/win-x64
cmake -G "MinGW Makefiles" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON ../..
cmake --build . --parallel --config Release
cp "$MinGWBin\libgcc*.dll" .
cp "$MinGWBin\libstdc++*.dll" .
cp "$MinGWBin\libwinpthread*.dll" .
cp bin/*.dll .
- persist_to_workspace:
root: gpt4all-backend
paths:
- runtimes/win-x64/*.dll
build-bindings-backend-windows-msvc:
machine:
image: 'windows-server-2022-gui:2023.03.1'
resource_class: windows.large
shell: powershell.exe -ExecutionPolicy Bypass
steps:
- checkout
- run:
name: Update Submodules
command: |
git submodule sync
git submodule update --init --recursive
- run:
name: Install VulkanSDK
command: |
Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
.\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
- run:
name: Install dependencies
command: |
choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
- run:
name: Build Libraries
command: |
$Env:Path += ";C:\Program Files\CMake\bin"
$Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
cd gpt4all-backend
mkdir runtimes/win-x64_msvc
cd runtimes/win-x64_msvc
cmake -G "Visual Studio 17 2022" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -A X64 ../..
cmake --build . --parallel --config Release
cp bin/Release/*.dll .
- persist_to_workspace:
root: gpt4all-backend
paths:
- runtimes/win-x64_msvc/*.dll
build-csharp-linux:
docker:
- image: mcr.microsoft.com/dotnet/sdk:8.0
steps:
- checkout
- attach_workspace:
at: /tmp/workspace
- run:
name: "Prepare Native Libs"
command: |
cd gpt4all-bindings/csharp
mkdir -p runtimes/linux-x64/native
cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
ls -R runtimes
- restore_cache:
keys:
- gpt4all-csharp-nuget-packages-nix
- run:
name: "Install project dependencies"
command: |
cd gpt4all-bindings/csharp
dotnet restore Gpt4All
- save_cache:
paths:
- ~/.nuget/packages
key: gpt4all-csharp-nuget-packages-nix
- run:
name: Build C# Project
command: |
cd gpt4all-bindings/csharp
dotnet build Gpt4All --configuration Release --nologo
- run:
name: "Run C# Tests"
command: |
cd gpt4all-bindings/csharp
dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
- run:
name: Test results
command: |
cd gpt4all-bindings/csharp/Gpt4All.Tests
dotnet tool install -g trx2junit
export PATH="$PATH:$HOME/.dotnet/tools"
trx2junit TestResults/*.trx
- store_test_results:
path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
build-csharp-windows:
executor:
name: win/default
size: large
shell: powershell.exe -ExecutionPolicy Bypass
steps:
- checkout
- restore_cache:
keys:
- gpt4all-csharp-nuget-packages-win
- attach_workspace:
at: C:\Users\circleci\workspace
- run:
name: "Install .NET"
command: |
choco install -y dotnet-8.0-sdk
- run:
name: "Prepare Native Libs"
command: |
cd gpt4all-bindings/csharp
mkdir -p runtimes\win-x64\native
cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\
ls -R runtimes
- run:
name: "Install project dependencies"
command: |
cd gpt4all-bindings/csharp
dotnet.exe restore Gpt4All
- save_cache:
paths:
- C:\Users\circleci\.nuget\packages
key: gpt4all-csharp-nuget-packages-win
- run:
name: Build C# Project
command: |
cd gpt4all-bindings/csharp
dotnet.exe build Gpt4All --configuration Release --nologo
- run:
name: "Run C# Tests"
command: |
cd gpt4all-bindings/csharp
dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
- run:
name: Test results
command: |
cd gpt4all-bindings/csharp/Gpt4All.Tests
dotnet tool install -g trx2junit
$Env:Path += ";$Env:USERPROFILE\.dotnet\tools"
trx2junit TestResults/*.trx
- store_test_results:
path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
build-csharp-macos:
macos:
xcode: "14.0.0"
steps:
- checkout
- restore_cache:
keys:
- gpt4all-csharp-nuget-packages-nix
- run:
name: Install dependencies
command: |
brew tap isen-ng/dotnet-sdk-versions
brew install --cask dotnet-sdk8-0-100
- attach_workspace:
at: /tmp/workspace
- run:
name: "Prepare Native Libs"
command: |
cd gpt4all-bindings/csharp
mkdir -p runtimes/osx/native
cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
ls -R runtimes
- run:
name: "Install project dependencies"
command: |
cd gpt4all-bindings/csharp
dotnet restore Gpt4All
- save_cache:
paths:
- ~/.nuget/packages
key: gpt4all-csharp-nuget-packages-nix
- run:
name: Build C# Project
command: |
cd gpt4all-bindings/csharp
dotnet build Gpt4All --configuration Release --nologo
- run:
name: "Run C# Tests"
command: |
cd gpt4all-bindings/csharp
dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
- run:
name: Test results
command: |
cd gpt4all-bindings/csharp/Gpt4All.Tests
dotnet tool install -g trx2junit
export PATH="$PATH:$HOME/.dotnet/tools"
trx2junit TestResults/*.trx
- store_test_results:
path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
store-and-upload-nupkgs:
docker:
- image: mcr.microsoft.com/dotnet/sdk:8.0
steps:
- attach_workspace:
at: /tmp/workspace
- checkout
- restore_cache:
keys:
- gpt4all-csharp-nuget-packages-nix
- run:
name: NuGet Pack
command: |
cd gpt4all-bindings/csharp
mkdir -p runtimes/linux-x64/native
cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
mkdir -p runtimes/win-x64/native
cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/
#mkdir -p runtimes/osx/native
#cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
#cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release
dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate
- store_artifacts:
path: gpt4all-bindings/csharp/Gpt4All/bin/Release
build-nodejs-linux:
docker:
- image: cimg/base:stable
steps:
- checkout
- attach_workspace:
at: /tmp/gpt4all-backend
- node/install:
install-yarn: true
node-version: "18.16"
- run: node --version
- run: corepack enable
- node/install-packages:
app-dir: gpt4all-bindings/typescript
pkg-manager: yarn
override-ci-command: yarn install
- run:
command: |
cd gpt4all-bindings/typescript
yarn prebuildify -t 18.16.0 --napi
- run:
command: |
mkdir -p gpt4all-backend/prebuilds/linux-x64
mkdir -p gpt4all-backend/runtimes/linux-x64
cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so gpt4all-backend/runtimes/linux-x64
cp gpt4all-bindings/typescript/prebuilds/linux-x64/*.node gpt4all-backend/prebuilds/linux-x64
- persist_to_workspace:
root: gpt4all-backend
paths:
- prebuilds/linux-x64/*.node
- runtimes/linux-x64/*-*.so
build-nodejs-macos:
macos:
xcode: "14.0.0"
steps:
- checkout
- attach_workspace:
at: /tmp/gpt4all-backend
- node/install:
install-yarn: true
node-version: "18.16"
- run: node --version
- run: corepack enable
- node/install-packages:
app-dir: gpt4all-bindings/typescript
pkg-manager: yarn
override-ci-command: yarn install
- run:
command: |
cd gpt4all-bindings/typescript
yarn prebuildify -t 18.16.0 --napi
- run:
name: "Persisting all necessary things to workspace"
command: |
mkdir -p gpt4all-backend/prebuilds/darwin-x64
mkdir -p gpt4all-backend/runtimes/darwin
cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin
cp gpt4all-bindings/typescript/prebuilds/darwin-x64/*.node gpt4all-backend/prebuilds/darwin-x64
- persist_to_workspace:
root: gpt4all-backend
paths:
- prebuilds/darwin-x64/*.node
- runtimes/darwin/*-*.*
build-nodejs-windows:
executor:
name: win/default
size: large
shell: powershell.exe -ExecutionPolicy Bypass
steps:
- checkout
- attach_workspace:
at: /tmp/gpt4all-backend
- run: choco install wget -y
- run:
command: wget https://nodejs.org/dist/v18.16.0/node-v18.16.0-x86.msi -P C:\Users\circleci\Downloads\
shell: cmd.exe
- run: MsiExec.exe /i C:\Users\circleci\Downloads\node-v18.16.0-x86.msi /qn
- run:
command: |
Start-Process powershell -verb runAs -Args "-start GeneralProfile"
nvm install 18.16.0
nvm use 18.16.0
- run: node --version
- run: corepack enable
- run:
command: |
npm install -g yarn
cd gpt4all-bindings/typescript
yarn install
- run:
command: |
cd gpt4all-bindings/typescript
yarn prebuildify -t 18.16.0 --napi
- run:
command: |
mkdir -p gpt4all-backend/prebuilds/win32-x64
mkdir -p gpt4all-backend/runtimes/win32-x64
cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll gpt4all-backend/runtimes/win32-x64
cp gpt4all-bindings/typescript/prebuilds/win32-x64/*.node gpt4all-backend/prebuilds/win32-x64
- persist_to_workspace:
root: gpt4all-backend
paths:
- prebuilds/win32-x64/*.node
- runtimes/win32-x64/*-*.dll
prepare-npm-pkg:
docker:
- image: cimg/base:stable
steps:
- attach_workspace:
at: /tmp/gpt4all-backend
- checkout
- node/install:
install-yarn: true
node-version: "18.16"
- run: node --version
- run: corepack enable
- run:
command: |
cd gpt4all-bindings/typescript
# excluding llmodel. nodejs bindings dont need llmodel.dll
mkdir -p runtimes/win32-x64/native
mkdir -p prebuilds/win32-x64/
cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll runtimes/win32-x64/native/
cp /tmp/gpt4all-backend/prebuilds/win32-x64/*.node prebuilds/win32-x64/
mkdir -p runtimes/linux-x64/native
mkdir -p prebuilds/linux-x64/
cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so runtimes/linux-x64/native/
cp /tmp/gpt4all-backend/prebuilds/linux-x64/*.node prebuilds/linux-x64/
# darwin has univeral runtime libraries
mkdir -p runtimes/darwin/native
mkdir -p prebuilds/darwin-x64/
cp /tmp/gpt4all-backend/runtimes/darwin/*-*.* runtimes/darwin/native/
cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/
# Fallback build if user is not on above prebuilds
mv -f binding.ci.gyp binding.gyp
mkdir gpt4all-backend
cd ../../gpt4all-backend
mv llmodel.h llmodel.cpp llmodel_c.cpp llmodel_c.h sysinfo.h dlhandle.h ../gpt4all-bindings/typescript/gpt4all-backend/
# Test install
- node/install-packages:
app-dir: gpt4all-bindings/typescript
pkg-manager: yarn
override-ci-command: yarn install
- run:
command: |
cd gpt4all-bindings/typescript
yarn run test
- run:
command: |
cd gpt4all-bindings/typescript
npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
npm publish
workflows:
version: 2
default:
when:
or:
- << pipeline.parameters.run-all-workflows >>
- << pipeline.parameters.run-default-workflow >>
jobs:
- default-job
build-chat-offline-installers:
when:
or:
- << pipeline.parameters.run-all-workflows >>
- << pipeline.parameters.run-chat-workflow >>
jobs:
- hold:
type: approval
- build-offline-chat-installer-macos:
requires:
- hold
- build-offline-chat-installer-windows:
requires:
- hold
- build-offline-chat-installer-linux:
requires:
- hold
build-and-test-gpt4all-chat:
when:
or:
- << pipeline.parameters.run-all-workflows >>
- << pipeline.parameters.run-chat-workflow >>
jobs:
- hold:
type: approval
- build-gpt4all-chat-linux:
requires:
- hold
- build-gpt4all-chat-windows:
requires:
- hold
- build-gpt4all-chat-macos:
requires:
- hold
deploy-docs:
when:
or:
- << pipeline.parameters.run-all-workflows >>
- << pipeline.parameters.run-python-workflow >>
jobs:
- build-ts-docs:
filters:
branches:
only:
- main
- build-py-docs:
filters:
branches:
only:
- main
build-py-deploy:
when:
or:
- << pipeline.parameters.run-all-workflows >>
- << pipeline.parameters.run-python-workflow >>
jobs:
- pypi-hold:
type: approval
- hold:
type: approval
- build-py-linux:
filters:
branches:
only:
requires:
- hold
- build-py-macos:
filters:
branches:
only:
requires:
- hold
- build-py-windows:
filters:
branches:
only:
requires:
- hold
- store-and-upload-wheels:
filters:
branches:
only:
requires:
- pypi-hold
- build-py-windows
- build-py-linux
- build-py-macos
build-bindings:
when:
or:
- << pipeline.parameters.run-all-workflows >>
- << pipeline.parameters.run-python-workflow >>
- << pipeline.parameters.run-csharp-workflow >>
- << pipeline.parameters.run-ts-workflow >>
jobs:
- hold:
type: approval
- csharp-hold:
type: approval
- nuget-hold:
type: approval
- nodejs-hold:
type: approval
- npm-hold:
type: approval
- build-bindings-backend-linux:
filters:
branches:
only:
requires:
- hold
- build-bindings-backend-macos:
filters:
branches:
only:
requires:
- hold
- build-bindings-backend-windows:
filters:
branches:
only:
requires:
- hold
- build-bindings-backend-windows-msvc:
filters:
branches:
only:
requires:
- hold
# NodeJs Jobs
- prepare-npm-pkg:
filters:
branches:
only:
requires:
- npm-hold
- build-nodejs-linux
- build-nodejs-windows
- build-nodejs-macos
- build-nodejs-linux:
filters:
branches:
only:
requires:
- nodejs-hold
- build-bindings-backend-linux
- build-nodejs-windows:
filters:
branches:
only:
requires:
- nodejs-hold
- build-bindings-backend-windows-msvc
- build-nodejs-macos:
filters:
branches:
only:
requires:
- nodejs-hold
- build-bindings-backend-macos
# CSharp Jobs
- build-csharp-linux:
filters:
branches:
only:
requires:
- csharp-hold
- build-bindings-backend-linux
- build-csharp-windows:
filters:
branches:
only:
requires:
- csharp-hold
- build-bindings-backend-windows
- build-csharp-macos:
filters:
branches:
only:
requires:
- csharp-hold
- build-bindings-backend-macos
- store-and-upload-nupkgs:
filters:
branches:
only:
requires:
- nuget-hold
- build-csharp-windows
- build-csharp-linux
#- build-csharp-macos