mirror of
https://github.com/SilasMarvin/lsp-ai.git
synced 2024-09-11 04:15:26 +03:00
Added some better testing
This commit is contained in:
Silas Marvin
parent
6627da705e
commit
d818cdca6d
1
.gitignore
vendored
1
.gitignore
vendored
@ -2,3 +2,4 @@
|
||||
/models
|
||||
node_modules
|
||||
out
|
||||
lsp-ai.log
|
||||
|
||||
81
Cargo.lock
generated
81
Cargo.lock
generated
@ -71,6 +71,21 @@ version = "1.0.80"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1"
|
||||
|
||||
[[package]]
|
||||
name = "assert_cmd"
|
||||
version = "2.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"bstr",
|
||||
"doc-comment",
|
||||
"predicates",
|
||||
"predicates-core",
|
||||
"predicates-tree",
|
||||
"wait-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.1.0"
|
||||
@ -124,6 +139,17 @@ version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"regex-automata 0.4.5",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
@ -355,6 +381,12 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "difflib"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
|
||||
|
||||
[[package]]
|
||||
name = "directories"
|
||||
version = "5.0.1"
|
||||
@ -385,6 +417,12 @@ dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "doc-comment"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.10.0"
|
||||
@ -658,6 +696,7 @@ name = "lsp-ai"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"assert_cmd",
|
||||
"directories",
|
||||
"hf-hub",
|
||||
"llama-cpp-2",
|
||||
@ -963,6 +1002,33 @@ version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
|
||||
|
||||
[[package]]
|
||||
name = "predicates"
|
||||
version = "3.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"difflib",
|
||||
"predicates-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "predicates-core"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174"
|
||||
|
||||
[[package]]
|
||||
name = "predicates-tree"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf"
|
||||
dependencies = [
|
||||
"predicates-core",
|
||||
"termtree",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.16"
|
||||
@ -1391,6 +1457,12 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termtree"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.57"
|
||||
@ -1635,6 +1707,15 @@ version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "wait-timeout"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
|
||||
@ -27,3 +27,6 @@ tracing = "0.1.40"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
[dev-dependencies]
|
||||
assert_cmd = "2.0.14"
|
||||
|
||||
26
src/main.rs
26
src/main.rs
@ -48,6 +48,7 @@ fn main() -> Result<()> {
|
||||
FmtSubscriber::builder()
|
||||
.with_writer(std::io::stderr)
|
||||
.with_env_filter(EnvFilter::from_env("LSP_AI_LOG"))
|
||||
.with_max_level(tracing::Level::TRACE)
|
||||
.init();
|
||||
|
||||
let (connection, io_threads) = Connection::stdio();
|
||||
@ -159,13 +160,28 @@ fn main_loop(connection: Connection, args: serde_json::Value) -> Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::memory_backends::Prompt;
|
||||
|
||||
use super::*;
|
||||
use crate::memory_backends::Prompt;
|
||||
use serde_json::json;
|
||||
|
||||
//////////////////////////////////////
|
||||
//////////////////////////////////////
|
||||
/// Some basic gguf model tests //////
|
||||
//////////////////////////////////////
|
||||
//////////////////////////////////////
|
||||
|
||||
#[test]
|
||||
fn custom_mac_gguf_model() {
|
||||
fn completion_with_default_arguments() {
|
||||
let args = json!({});
|
||||
let configuration = Configuration::new(args).unwrap();
|
||||
let backend: Box<dyn TransformerBackend + Send> = configuration.clone().try_into().unwrap();
|
||||
let prompt = Prompt::new("".to_string(), "def fibn".to_string());
|
||||
let response = backend.do_completion(&prompt).unwrap();
|
||||
assert!(!response.insert_text.is_empty())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_with_custom_gguf_model() {
|
||||
let args = json!({
|
||||
"initializationOptions": {
|
||||
"memory": {
|
||||
@ -175,8 +191,6 @@ mod tests {
|
||||
"model_gguf": {
|
||||
"repository": "TheBloke/deepseek-coder-6.7B-instruct-GGUF",
|
||||
"name": "deepseek-coder-6.7b-instruct.Q5_K_S.gguf",
|
||||
// "repository": "stabilityai/stablelm-2-zephyr-1_6b",
|
||||
// "name": "stablelm-2-zephyr-1_6b-Q5_K_M.gguf",
|
||||
"max_new_tokens": {
|
||||
"completion": 32,
|
||||
"generation": 256,
|
||||
@ -219,6 +233,6 @@ mod tests {
|
||||
let backend: Box<dyn TransformerBackend + Send> = configuration.clone().try_into().unwrap();
|
||||
let prompt = Prompt::new("".to_string(), "def fibn".to_string());
|
||||
let response = backend.do_completion(&prompt).unwrap();
|
||||
eprintln!("\nRESPONSE:\n{:?}", response.insert_text);
|
||||
assert!(!response.insert_text.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
114
tests/integration_tests.rs
Normal file
114
tests/integration_tests.rs
Normal file
@ -0,0 +1,114 @@
|
||||
use anyhow::Result;
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
process::{ChildStdin, ChildStdout, Command, Stdio},
|
||||
};
|
||||
|
||||
// Note if you get an empty response with no error, that typically means
|
||||
// the language server died
|
||||
fn read_response(stdout: &mut ChildStdout) -> Result<String> {
|
||||
eprintln!("READING RESPONSE");
|
||||
let mut content_length = None;
|
||||
let mut buf = vec![];
|
||||
loop {
|
||||
let mut buf2 = vec![0];
|
||||
stdout.read_exact(&mut buf2)?;
|
||||
buf.push(buf2[0]);
|
||||
if let Some(content_length) = content_length {
|
||||
if buf.len() == content_length {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
let len = buf.len();
|
||||
if len > 4
|
||||
&& buf[len - 4] == 13
|
||||
&& buf[len - 3] == 10
|
||||
&& buf[len - 2] == 13
|
||||
&& buf[len - 1] == 10
|
||||
{
|
||||
content_length =
|
||||
Some(String::from_utf8(buf[16..len - 4].to_vec())?.parse::<usize>()?);
|
||||
println!("SETTING CONTENT-LENGTH: {:?}", content_length);
|
||||
buf = vec![];
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(String::from_utf8(buf)?)
|
||||
}
|
||||
|
||||
fn send_message(stdin: &mut ChildStdin, message: &str) -> Result<()> {
|
||||
stdin.write_all(format!("Content-Length: {}\r\n", message.as_bytes().len(),).as_bytes())?;
|
||||
stdin.write_all("\r\n".as_bytes())?;
|
||||
stdin.write_all(message.as_bytes())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// This completion sequence was created using helix with the lsp-ai analyzer and reading the logs
|
||||
// It starts with a Python file:
|
||||
// ```
|
||||
// # Multiplies two numbers
|
||||
// def multiply_two_numbers(x, y):
|
||||
//
|
||||
// # A singular test
|
||||
// assert multiply_two_numbers(2, 3) == 6
|
||||
// ```
|
||||
// And has the following sequence of key strokes:
|
||||
// o on line 2 (this creates an indented new line and enters insert mode)
|
||||
// r
|
||||
// e
|
||||
// The sequence has:
|
||||
// - 1 textDocument/DidOpen notification
|
||||
// - 3 textDocument/didChange notifications
|
||||
// - 1 textDocument/completion requests
|
||||
// This test can fail if the model gives a different response than normal, but that seems reasonably unlikely
|
||||
// I guess we should hardcode the seed or something if we want to do more of these
|
||||
#[test]
|
||||
fn test_completion_sequence() -> Result<()> {
|
||||
let mut child = Command::new("cargo")
|
||||
.arg("run")
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
|
||||
let mut stdin = child.stdin.take().unwrap();
|
||||
let mut stdout = child.stdout.take().unwrap();
|
||||
|
||||
let initialization_message = r##"{"jsonrpc":"2.0","method":"initialize","params":{"capabilities":{"general":{"positionEncodings":["utf-8","utf-32","utf-16"]},"textDocument":{"codeAction":{"codeActionLiteralSupport":{"codeActionKind":{"valueSet":["","quickfix","refactor","refactor.extract","refactor.inline","refactor.rewrite","source","source.organizeImports"]}},"dataSupport":true,"disabledSupport":true,"isPreferredSupport":true,"resolveSupport":{"properties":["edit","command"]}},"completion":{"completionItem":{"deprecatedSupport":true,"insertReplaceSupport":true,"resolveSupport":{"properties":["documentation","detail","additionalTextEdits"]},"snippetSupport":true,"tagSupport":{"valueSet":[1]}},"completionItemKind":{}},"hover":{"contentFormat":["markdown"]},"inlayHint":{"dynamicRegistration":false},"publishDiagnostics":{"versionSupport":true},"rename":{"dynamicRegistration":false,"honorsChangeAnnotations":false,"prepareSupport":true},"signatureHelp":{"signatureInformation":{"activeParameterSupport":true,"documentationFormat":["markdown"],"parameterInformation":{"labelOffsetSupport":true}}}},"window":{"workDoneProgress":true},"workspace":{"applyEdit":true,"configuration":true,"didChangeConfiguration":{"dynamicRegistration":false},"didChangeWatchedFiles":{"dynamicRegistration":true,"relativePatternSupport":false},"executeCommand":{"dynamicRegistration":false},"inlayHint":{"refreshSupport":false},"symbol":{"dynamicRegistration":false},"workspaceEdit":{"documentChanges":true,"failureHandling":"abort","normalizesLineEndings":false,"resourceOperations":["create","rename","delete"]},"workspaceFolders":true}},"clientInfo":{"name":"helix","version":"23.10 (f6021dd0)"},"processId":70007,"rootPath":"/Users/silas/Projects/Tests/lsp-ai-tests","rootUri":null,"workspaceFolders":[]},"id":0}"##;
|
||||
send_message(&mut stdin, initialization_message)?;
|
||||
let _ = read_response(&mut stdout)?;
|
||||
|
||||
send_message(
|
||||
&mut stdin,
|
||||
r#"{"jsonrpc":"2.0","method":"initialized","params":{}}"#,
|
||||
)?;
|
||||
send_message(
|
||||
&mut stdin,
|
||||
r##"{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"languageId":"python","text":"# Multiplies two numbers\ndef multiply_two_numbers(x, y):\n\n# A singular test\nassert multiply_two_numbers(2, 3) == 6\n","uri":"file:///fake.py","version":0}}}"##,
|
||||
)?;
|
||||
send_message(
|
||||
&mut stdin,
|
||||
r##"{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"contentChanges":[{"range":{"end":{"character":31,"line":1},"start":{"character":31,"line":1}},"text":"\n "}],"textDocument":{"uri":"file:///fake.py","version":1}}}"##,
|
||||
)?;
|
||||
send_message(
|
||||
&mut stdin,
|
||||
r##"{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"contentChanges":[{"range":{"end":{"character":4,"line":2},"start":{"character":4,"line":2}},"text":"r"}],"textDocument":{"uri":"file:///fake.py","version":2}}}"##,
|
||||
)?;
|
||||
send_message(
|
||||
&mut stdin,
|
||||
r##"{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"contentChanges":[{"range":{"end":{"character":5,"line":2},"start":{"character":5,"line":2}},"text":"e"}],"textDocument":{"uri":"file:///fake.py","version":3}}}"##,
|
||||
)?;
|
||||
send_message(
|
||||
&mut stdin,
|
||||
r##"{"jsonrpc":"2.0","method":"textDocument/completion","params":{"position":{"character":6,"line":2},"textDocument":{"uri":"file:///fake.py"}},"id":1}"##,
|
||||
)?;
|
||||
|
||||
let output = read_response(&mut stdout)?;
|
||||
assert_eq!(
|
||||
output,
|
||||
r##"{"jsonrpc":"2.0","id":1,"result":{"isIncomplete":false,"items":[{"filterText":" re\n","kind":1,"label":"ai - turn x * y","textEdit":{"newText":"turn x * y","range":{"end":{"character":6,"line":2},"start":{"character":6,"line":2}}}}]}}"##
|
||||
);
|
||||
|
||||
child.kill()?;
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user