Implement the first real test of gpt4all-chat (#3116)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.gitignore

@@ -182,6 +182,7 @@ gpt4all-chat/models/*
 build_*
 build-*
 cmake-build-*
+/gpt4all-chat/tests/python/config.py
 
 # IntelliJ
 .idea/

gpt4all-chat/.flake8

@@ -0,0 +1,5 @@
+# vim: set syntax=dosini:
+[flake8]
+exclude = .*,__pycache__
+max-line-length = 120
+extend-ignore = B001,C408,D,DAR,E221,E303,E722,E741,E800,N801,N806,P101,S101,S324,S404,S406,S410,S603,WPS100,WPS110,WPS111,WPS113,WPS114,WPS115,WPS120,WPS2,WPS300,WPS301,WPS304,WPS305,WPS306,WPS309,WPS316,WPS317,WPS318,WPS319,WPS322,WPS323,WPS326,WPS329,WPS330,WPS332,WPS336,WPS337,WPS347,WPS360,WPS361,WPS414,WPS420,WPS421,WPS429,WPS430,WPS431,WPS432,WPS433,WPS437,WPS440,WPS440,WPS441,WPS442,WPS457,WPS458,WPS460,WPS462,WPS463,WPS473,WPS501,WPS504,WPS505,WPS508,WPS509,WPS510,WPS515,WPS516,WPS519,WPS529,WPS531,WPS602,WPS604,WPS605,WPS608,WPS609,WPS613,WPS615

gpt4all-chat/CMakeLists.txt

@@ -22,7 +22,7 @@ if(APPLE)
   endif()
 endif()
 
-find_package(Python3 QUIET COMPONENTS Interpreter)
+find_package(Python3 3.12 QUIET COMPONENTS Interpreter)
 
 option(GPT4ALL_TEST "Build the tests" ${Python3_FOUND})
 option(GPT4ALL_LOCALHOST "Build installer for localhost repo" OFF)
@@ -101,7 +101,7 @@ if (GPT4ALL_TEST)
     add_subdirectory(tests)
 
     # The 'check' target makes sure the tests and their dependencies are up-to-date before running them
-    add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} DEPENDS chat gpt4all_tests)
+    add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS chat gpt4all_tests)
 endif()
 
 set(CHAT_EXE_RESOURCES)

gpt4all-chat/dev-requirements.txt

@@ -0,0 +1,11 @@
+-r test-requirements.txt
+
+# dev tools
+flake8~=7.1
+mypy~=1.12
+pytype>=2024.10.11
+wemake-python-styleguide~=0.19.2
+
+# type stubs and other optional modules
+types-requests~=2.32
+urllib3[socks]

gpt4all-chat/pyproject.toml

@@ -0,0 +1,29 @@
+[tool.pytest.ini_options]
+addopts = ['--import-mode=importlib']
+
+[tool.mypy]
+files = 'tests/python'
+pretty = true
+strict = true
+warn_unused_ignores = false
+
+[tool.pytype]
+inputs = ['tests/python']
+jobs = 'auto'
+bind_decorated_methods = true
+none_is_not_bool = true
+overriding_renamed_parameter_count_checks = true
+strict_none_binding = true
+precise_return = true
+# protocols:
+# - https://github.com/google/pytype/issues/1423
+# - https://github.com/google/pytype/issues/1424
+strict_import = true
+strict_parameter_checks = true
+strict_primitive_comparisons = true
+# strict_undefined_checks: too many false positives
+
+[tool.isort]
+src_paths = ['tests/python']
+line_length = 120
+combine_as_imports = true

gpt4all-chat/src/main.cpp

@@ -26,6 +26,8 @@
 
 #ifdef Q_OS_WINDOWS
 #   include <windows.h>
+#else
+#   include <signal.h>
 #endif
 
 using namespace Qt::Literals::StringLiterals;
@@ -130,6 +132,17 @@ int main(int argc, char *argv[])
     }
 #endif
 
+#ifndef Q_OS_WINDOWS
+    // handle signals gracefully
+    struct sigaction sa;
+    sa.sa_handler = [](int s) { QCoreApplication::exit(s == SIGINT ? 0 : 1); };
+    sa.sa_flags   = SA_RESETHAND;
+    sigemptyset(&sa.sa_mask);
+    sigaction(SIGINT,  &sa, nullptr);
+    sigaction(SIGTERM, &sa, nullptr);
+    sigaction(SIGHUP,  &sa, nullptr);
+#endif
+
     int res = app.exec();
 
     // Make sure ChatLLM threads are joined before global destructors run.

gpt4all-chat/test-requirements.txt

@@ -0,0 +1,2 @@
+pytest~=8.3
+requests~=2.32

gpt4all-chat/tests/CMakeLists.txt

@@ -1,6 +1,6 @@
 include(FetchContent)
 
-find_package(Python3 REQUIRED COMPONENTS Interpreter)
+find_package(Python3 3.12 REQUIRED COMPONENTS Interpreter)
 
 # Google test download and setup
 FetchContent_Declare(
@@ -9,8 +9,10 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(googletest)
 
+configure_file(python/config.py.in "${CMAKE_CURRENT_SOURCE_DIR}/python/config.py")
+
 add_test(NAME ChatPythonTests
-    COMMAND ${Python3_EXECUTABLE} -m pytest ${CMAKE_SOURCE_DIR}/tests/python_tests
+    COMMAND ${Python3_EXECUTABLE} -m pytest --color=yes "${CMAKE_CURRENT_SOURCE_DIR}/python"
 )
 set_tests_properties(ChatPythonTests PROPERTIES
     ENVIRONMENT "CHAT_EXECUTABLE=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/chat"
@@ -18,8 +20,8 @@ set_tests_properties(ChatPythonTests PROPERTIES
 )
 
 add_executable(gpt4all_tests
-    test_main.cpp
-    basic_test.cpp
+    cpp/test_main.cpp
+    cpp/basic_test.cpp
 )
 
 target_link_libraries(gpt4all_tests PRIVATE gtest gtest_main)

gpt4all-chat/tests/python/config.py.in

@@ -0,0 +1 @@
+APP_VERSION = '@APP_VERSION@'

gpt4all-chat/tests/python/test_server_api.py

@@ -0,0 +1,87 @@
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import textwrap
+from pathlib import Path
+from subprocess import CalledProcessError
+from typing import Any, Iterator
+
+import pytest
+import requests
+from urllib3 import Retry
+
+from . import config
+
+
+class Requestor:
+    def __init__(self) -> None:
+        self.session = requests.Session()
+        self.http_adapter = self.session.adapters['http://']
+
+    def get(self, path: str, *, wait: bool = False) -> Any:
+        return self._request('GET', path, wait)
+
+    def _request(self, method: str, path: str, wait: bool) -> Any:
+        if wait:
+            retry = Retry(total=None, connect=10, read=False, status=0, other=0, backoff_factor=.01)
+        else:
+            retry = Retry(total=False)
+        self.http_adapter.max_retries = retry  # type: ignore[attr-defined]
+
+        resp = self.session.request(method, f'http://localhost:4891/v1/{path}')
+        resp.raise_for_status()
+        return resp.json()
+
+
+request = Requestor()
+
+
+@pytest.fixture
+def chat_server_config() -> Iterator[dict[str, str]]:
+    if os.name != 'posix' or sys.platform == 'darwin':
+        pytest.skip('Need non-Apple Unix to use alternate config path')
+
+    with tempfile.TemporaryDirectory(prefix='gpt4all-test') as td:
+        tmpdir = Path(td)
+        xdg_confdir = tmpdir / 'config'
+        app_confdir = xdg_confdir / 'nomic.ai'
+        app_confdir.mkdir(parents=True)
+        with open(app_confdir / 'GPT4All.ini', 'w') as conf:
+            conf.write(textwrap.dedent(f"""\
+                [General]
+                serverChat=true
+
+                [download]
+                lastVersionStarted={config.APP_VERSION}
+
+                [network]
+                isActive=false
+                usageStatsActive=false
+            """))
+        yield dict(
+            os.environ,
+            XDG_CACHE_HOME=str(tmpdir / 'cache'),
+            XDG_DATA_HOME=str(tmpdir / 'share'),
+            XDG_CONFIG_HOME=str(xdg_confdir),
+            APPIMAGE=str(tmpdir),  # hack to bypass SingleApplication
+        )
+
+
+@pytest.fixture
+def chat_server(chat_server_config: dict[str, str]) -> Iterator[None]:
+    chat_executable = Path(os.environ['CHAT_EXECUTABLE']).absolute()
+    with subprocess.Popen(chat_executable, env=chat_server_config) as process:
+        try:
+            yield
+        except:
+            process.kill()
+            raise
+        process.send_signal(signal.SIGINT)
+        if retcode := process.wait():
+            raise CalledProcessError(retcode, process.args)
+
+
+def test_list_models_empty(chat_server: None) -> None:
+    assert request.get('models', wait=True) == {'object': 'list', 'data': []}

gpt4all-chat/tests/python_tests/test_executable.py

@@ -1,7 +0,0 @@
-import os
-
-import pytest
-
-# test that the chat executable exists
-def test_chat_environment():
-    assert os.path.exists(os.environ['CHAT_EXECUTABLE'])