From d818cdca6ddd25ce91651d14c361bebed3a417f3 Mon Sep 17 00:00:00 2001 From: Silas Marvin <19626586+SilasMarvin@users.noreply.github.com> Date: Sun, 3 Mar 2024 17:27:37 -0800 Subject: [PATCH] Added some better testing --- .gitignore | 1 + Cargo.lock | 81 ++++++++++++++++++++++++++ Cargo.toml | 3 + src/main.rs | 26 +++++++-- tests/integration_tests.rs | 114 +++++++++++++++++++++++++++++++++++++ 5 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 tests/integration_tests.rs diff --git a/.gitignore b/.gitignore index a7196cf..dc051d7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /models node_modules out +lsp-ai.log diff --git a/Cargo.lock b/Cargo.lock index 5baeb0b..f623ed5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,21 @@ version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +[[package]] +name = "assert_cmd" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -124,6 +139,17 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +[[package]] +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +dependencies = [ + "memchr", + "regex-automata 0.4.5", + "serde", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -355,6 +381,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "directories" version = "5.0.1" @@ -385,6 +417,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "either" version = "1.10.0" @@ -658,6 +696,7 @@ name = "lsp-ai" version = "0.1.0" dependencies = [ "anyhow", + "assert_cmd", "directories", "hf-hub", "llama-cpp-2", @@ -963,6 +1002,33 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prettyplease" version = "0.2.16" @@ -1391,6 +1457,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "thiserror" version = "1.0.57" @@ -1635,6 +1707,15 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 659feb3..06fa1d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,6 @@ tracing = "0.1.40" [features] default = [] + +[dev-dependencies] +assert_cmd = "2.0.14" diff --git a/src/main.rs b/src/main.rs index ad9e81a..9e32eba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,6 +48,7 @@ fn main() -> Result<()> { FmtSubscriber::builder() .with_writer(std::io::stderr) .with_env_filter(EnvFilter::from_env("LSP_AI_LOG")) + .with_max_level(tracing::Level::TRACE) .init(); let (connection, io_threads) = Connection::stdio(); @@ -159,13 +160,28 @@ fn main_loop(connection: Connection, args: serde_json::Value) -> Result<()> { #[cfg(test)] mod tests { - use crate::memory_backends::Prompt; - use super::*; + use crate::memory_backends::Prompt; use serde_json::json; + ////////////////////////////////////// + ////////////////////////////////////// + /// Some basic gguf model tests ////// + ////////////////////////////////////// + ////////////////////////////////////// + #[test] - fn custom_mac_gguf_model() { + fn completion_with_default_arguments() { + let args = json!({}); + let configuration = Configuration::new(args).unwrap(); + let backend: Box = configuration.clone().try_into().unwrap(); + let prompt = Prompt::new("".to_string(), "def fibn".to_string()); + let response = backend.do_completion(&prompt).unwrap(); + assert!(!response.insert_text.is_empty()) + } + + #[test] + fn completion_with_custom_gguf_model() { let args = json!({ "initializationOptions": { "memory": { @@ -175,8 +191,6 @@ mod tests { "model_gguf": { "repository": "TheBloke/deepseek-coder-6.7B-instruct-GGUF", "name": "deepseek-coder-6.7b-instruct.Q5_K_S.gguf", - // "repository": "stabilityai/stablelm-2-zephyr-1_6b", - // "name": "stablelm-2-zephyr-1_6b-Q5_K_M.gguf", "max_new_tokens": { "completion": 32, "generation": 256, @@ -219,6 +233,6 @@ mod tests { let backend: Box = configuration.clone().try_into().unwrap(); let prompt = Prompt::new("".to_string(), "def fibn".to_string()); let response = backend.do_completion(&prompt).unwrap(); - eprintln!("\nRESPONSE:\n{:?}", response.insert_text); + assert!(!response.insert_text.is_empty()); } } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs new file mode 100644 index 0000000..2b9e0e7 --- /dev/null +++ b/tests/integration_tests.rs @@ -0,0 +1,114 @@ +use anyhow::Result; +use std::{ + io::{Read, Write}, + process::{ChildStdin, ChildStdout, Command, Stdio}, +}; + +// Note if you get an empty response with no error, that typically means +// the language server died +fn read_response(stdout: &mut ChildStdout) -> Result { + eprintln!("READING RESPONSE"); + let mut content_length = None; + let mut buf = vec![]; + loop { + let mut buf2 = vec![0]; + stdout.read_exact(&mut buf2)?; + buf.push(buf2[0]); + if let Some(content_length) = content_length { + if buf.len() == content_length { + break; + } + } else { + let len = buf.len(); + if len > 4 + && buf[len - 4] == 13 + && buf[len - 3] == 10 + && buf[len - 2] == 13 + && buf[len - 1] == 10 + { + content_length = + Some(String::from_utf8(buf[16..len - 4].to_vec())?.parse::()?); + println!("SETTING CONTENT-LENGTH: {:?}", content_length); + buf = vec![]; + } + } + } + Ok(String::from_utf8(buf)?) +} + +fn send_message(stdin: &mut ChildStdin, message: &str) -> Result<()> { + stdin.write_all(format!("Content-Length: {}\r\n", message.as_bytes().len(),).as_bytes())?; + stdin.write_all("\r\n".as_bytes())?; + stdin.write_all(message.as_bytes())?; + Ok(()) +} + +// This completion sequence was created using helix with the lsp-ai analyzer and reading the logs +// It starts with a Python file: +// ``` +// # Multiplies two numbers +// def multiply_two_numbers(x, y): +// +// # A singular test +// assert multiply_two_numbers(2, 3) == 6 +// ``` +// And has the following sequence of key strokes: +// o on line 2 (this creates an indented new line and enters insert mode) +// r +// e +// The sequence has: +// - 1 textDocument/DidOpen notification +// - 3 textDocument/didChange notifications +// - 1 textDocument/completion requests +// This test can fail if the model gives a different response than normal, but that seems reasonably unlikely +// I guess we should hardcode the seed or something if we want to do more of these +#[test] +fn test_completion_sequence() -> Result<()> { + let mut child = Command::new("cargo") + .arg("run") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = child.stdout.take().unwrap(); + + let initialization_message = r##"{"jsonrpc":"2.0","method":"initialize","params":{"capabilities":{"general":{"positionEncodings":["utf-8","utf-32","utf-16"]},"textDocument":{"codeAction":{"codeActionLiteralSupport":{"codeActionKind":{"valueSet":["","quickfix","refactor","refactor.extract","refactor.inline","refactor.rewrite","source","source.organizeImports"]}},"dataSupport":true,"disabledSupport":true,"isPreferredSupport":true,"resolveSupport":{"properties":["edit","command"]}},"completion":{"completionItem":{"deprecatedSupport":true,"insertReplaceSupport":true,"resolveSupport":{"properties":["documentation","detail","additionalTextEdits"]},"snippetSupport":true,"tagSupport":{"valueSet":[1]}},"completionItemKind":{}},"hover":{"contentFormat":["markdown"]},"inlayHint":{"dynamicRegistration":false},"publishDiagnostics":{"versionSupport":true},"rename":{"dynamicRegistration":false,"honorsChangeAnnotations":false,"prepareSupport":true},"signatureHelp":{"signatureInformation":{"activeParameterSupport":true,"documentationFormat":["markdown"],"parameterInformation":{"labelOffsetSupport":true}}}},"window":{"workDoneProgress":true},"workspace":{"applyEdit":true,"configuration":true,"didChangeConfiguration":{"dynamicRegistration":false},"didChangeWatchedFiles":{"dynamicRegistration":true,"relativePatternSupport":false},"executeCommand":{"dynamicRegistration":false},"inlayHint":{"refreshSupport":false},"symbol":{"dynamicRegistration":false},"workspaceEdit":{"documentChanges":true,"failureHandling":"abort","normalizesLineEndings":false,"resourceOperations":["create","rename","delete"]},"workspaceFolders":true}},"clientInfo":{"name":"helix","version":"23.10 (f6021dd0)"},"processId":70007,"rootPath":"/Users/silas/Projects/Tests/lsp-ai-tests","rootUri":null,"workspaceFolders":[]},"id":0}"##; + send_message(&mut stdin, initialization_message)?; + let _ = read_response(&mut stdout)?; + + send_message( + &mut stdin, + r#"{"jsonrpc":"2.0","method":"initialized","params":{}}"#, + )?; + send_message( + &mut stdin, + r##"{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"languageId":"python","text":"# Multiplies two numbers\ndef multiply_two_numbers(x, y):\n\n# A singular test\nassert multiply_two_numbers(2, 3) == 6\n","uri":"file:///fake.py","version":0}}}"##, + )?; + send_message( + &mut stdin, + r##"{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"contentChanges":[{"range":{"end":{"character":31,"line":1},"start":{"character":31,"line":1}},"text":"\n "}],"textDocument":{"uri":"file:///fake.py","version":1}}}"##, + )?; + send_message( + &mut stdin, + r##"{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"contentChanges":[{"range":{"end":{"character":4,"line":2},"start":{"character":4,"line":2}},"text":"r"}],"textDocument":{"uri":"file:///fake.py","version":2}}}"##, + )?; + send_message( + &mut stdin, + r##"{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"contentChanges":[{"range":{"end":{"character":5,"line":2},"start":{"character":5,"line":2}},"text":"e"}],"textDocument":{"uri":"file:///fake.py","version":3}}}"##, + )?; + send_message( + &mut stdin, + r##"{"jsonrpc":"2.0","method":"textDocument/completion","params":{"position":{"character":6,"line":2},"textDocument":{"uri":"file:///fake.py"}},"id":1}"##, + )?; + + let output = read_response(&mut stdout)?; + assert_eq!( + output, + r##"{"jsonrpc":"2.0","id":1,"result":{"isIncomplete":false,"items":[{"filterText":" re\n","kind":1,"label":"ai - turn x * y","textEdit":{"newText":"turn x * y","range":{"end":{"character":6,"line":2},"start":{"character":6,"line":2}}}}]}}"## + ); + + child.kill()?; + Ok(()) +}