Add nls benchmarks (#1659)

Cargo.lock

@@ -1738,9 +1738,11 @@ dependencies = [
  "codespan",
  "codespan-lsp",
  "codespan-reporting",
+ "criterion",
  "csv",
  "derive_more",
  "env_logger",
+ "glob",
  "insta",
  "lalrpop",
  "lalrpop-util",

lsp/lsp-harness/Cargo.toml

@@ -12,6 +12,7 @@ lsp-server.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 toml.workspace = true
+assert_cmd.workspace = true
 
 [dev-dependencies]
 assert_cmd.workspace = true

lsp/lsp-harness/src/jsonrpc.rs

@@ -68,10 +68,10 @@ struct SendNotification<T: LspNotification> {
 pub struct Notification {
     /// The string "2.0", hopefully. (We aren't strict about checking it.)
     jsonrpc: String,
-    method: String,
+    pub method: String,
     /// The notification parameters. The structure of this should be determined
     /// by `method`, but it hasn't been checked yet.
-    params: serde_json::Value,
+    pub params: serde_json::Value,
 }
 
 /// An untyped request response from the LS.
@@ -240,7 +240,7 @@ impl Server {
     }
 
     /// Receive a single JSON RPC message from the server.
-    fn recv(&mut self) -> Result<ServerMessage> {
+    pub(crate) fn recv(&mut self) -> Result<ServerMessage> {
         let mut buf = String::new();
         let mut content_length: Option<usize> = None;
 

lsp/lsp-harness/src/lib.rs

@@ -1,11 +1,19 @@
 mod jsonrpc;
 mod output;
 
+use std::collections::{hash_map::Entry, HashMap};
+
+use assert_cmd::prelude::CommandCargoExt;
 pub use jsonrpc::Server;
 use log::error;
 use lsp_types::{
+    notification::{Notification, PublishDiagnostics},
+    request::{
+        Completion, DocumentSymbolRequest, Formatting, GotoDefinition, HoverRequest, References,
+        Request as LspRequest,
+    },
     CompletionParams, DocumentFormattingParams, DocumentSymbolParams, GotoDefinitionParams,
-    HoverParams, ReferenceParams, Url,
+    HoverParams, PublishDiagnosticsParams, ReferenceParams, Url,
 };
 pub use output::LspDebug;
 use serde::Deserialize;
@@ -27,7 +35,7 @@ pub struct TestFile {
 }
 
 /// A subset of LSP requests that our harness supports.
-#[derive(Deserialize, Debug)]
+#[derive(Clone, Deserialize, Debug)]
 #[serde(tag = "type")]
 pub enum Request {
     GotoDefinition(GotoDefinitionParams),
@@ -96,3 +104,99 @@ impl TestFixture {
         }
     }
 }
+
+pub struct TestHarness {
+    srv: Server,
+    pub out: Vec<u8>,
+}
+
+impl Default for TestHarness {
+    fn default() -> Self {
+        TestHarness::new()
+    }
+}
+
+impl TestHarness {
+    pub fn new() -> Self {
+        let cmd = std::process::Command::cargo_bin("nls").unwrap();
+        let srv = Server::new(cmd).unwrap();
+        Self {
+            srv,
+            out: Vec::new(),
+        }
+    }
+
+    pub fn request<T: LspRequest>(&mut self, params: T::Params)
+    where
+        T::Result: LspDebug,
+    {
+        let result = self.srv.send_request::<T>(params).unwrap();
+        result.debug(&mut self.out).unwrap();
+        self.out.push(b'\n');
+    }
+
+    pub fn request_dyn(&mut self, req: Request) {
+        match req {
+            Request::GotoDefinition(d) => self.request::<GotoDefinition>(d),
+            Request::Completion(c) => self.request::<Completion>(c),
+            Request::Formatting(f) => self.request::<Formatting>(f),
+            Request::Hover(h) => self.request::<HoverRequest>(h),
+            Request::References(r) => self.request::<References>(r),
+            Request::Symbols(s) => self.request::<DocumentSymbolRequest>(s),
+        }
+    }
+
+    pub fn prepare_files(&mut self, fixture: &TestFixture) {
+        let mut file_versions = HashMap::new();
+
+        if fixture.files.is_empty() {
+            panic!("no files");
+        }
+
+        for file in &fixture.files {
+            match file_versions.entry(file.uri.clone()) {
+                Entry::Occupied(mut version) => {
+                    *version.get_mut() += 1;
+                    self.srv
+                        .replace_file(file.uri.clone(), *version.get(), &file.contents)
+                        .unwrap();
+                }
+                Entry::Vacant(entry) => {
+                    self.send_file(file.uri.clone(), &file.contents);
+                    entry.insert(1);
+                }
+            }
+        }
+    }
+
+    pub fn send_file(&mut self, uri: Url, contents: &str) {
+        self.srv.send_file(uri.clone(), contents).unwrap();
+    }
+
+    // Waits (until forever, if necessary) for the first diagnostics, and then
+    // returns them.
+    pub fn wait_for_diagnostics(&mut self) -> PublishDiagnosticsParams {
+        loop {
+            match self.srv.recv().unwrap() {
+                jsonrpc::ServerMessage::Notification(note) => {
+                    if note.method == PublishDiagnostics::METHOD {
+                        return serde_json::value::from_value(note.params).unwrap();
+                    }
+                }
+                jsonrpc::ServerMessage::Response(_) => {}
+            }
+        }
+    }
+
+    // For debug purposes, drain and print notifications.
+    pub fn drain_notifications(&mut self) {
+        // FIXME: nls doesn't report progress, so we have no way to check whether
+        // it's finished sending notifications. We just retrieve any that we've already
+        // received.
+        // We should also have a better format for printing diagnostics and other
+        // notifications.
+        for msg in self.srv.pending_notifications() {
+            eprintln!("{msg:?}");
+        }
+    }
+}

lsp/nls/Cargo.toml

@@ -18,6 +18,10 @@ path = "src/main.rs"
 default = ["format"]
 format = ["nickel-lang-core/format"]
 
+[[bench]]
+name = "main"
+harness = false
+
 [build-dependencies]
 lalrpop.workspace = true
 
@@ -45,6 +49,8 @@ thiserror.workspace = true
 [dev-dependencies]
 assert_cmd.workspace = true
 assert_matches.workspace = true
+criterion.workspace = true
+glob = "0.3.1"
 insta.workspace = true
 lsp-harness.workspace = true
 nickel-lang-utils.workspace = true

lsp/nls/benches/main.rs

@@ -0,0 +1,105 @@
+use std::{
+    path::{Path, PathBuf},
+    time::Duration,
+};
+
+use criterion::{criterion_group, criterion_main, Criterion};
+
+use glob::glob;
+use lsp_harness::{TestFixture, TestHarness};
+use nickel_lang_core::cache;
+use nickel_lang_utils::project_root::project_root;
+
+criterion_main!(test_request_benches, test_init_benches);
+
+criterion_group! {
+    name = test_request_benches;
+    // There are a lot of these and none is particularly high-value, so turn down
+    // the default benchmark time.
+    config = Criterion::default()
+        .measurement_time(Duration::from_secs(1))
+        .warm_up_time(Duration::from_secs_f64(0.5));
+    targets = test_requests
+}
+
+criterion_group! {
+    name = test_init_benches;
+    // There are a lot of these and none is particularly high-value, so turn down
+    // the default benchmark time.
+    config = Criterion::default()
+        .measurement_time(Duration::from_secs(1))
+        .warm_up_time(Duration::from_secs_f64(0.5));
+    targets = test_init
+}
+
+fn friendly_path(path: &Path) -> String {
+    let path = cache::normalize_path(path).unwrap();
+    let components: Vec<_> = path.components().rev().take(3).collect();
+    let path: PathBuf = components.into_iter().rev().collect();
+    path.to_str().unwrap().to_owned()
+}
+
+fn test_requests(c: &mut Criterion) {
+    let files = project_root()
+        .join("lsp/nls/tests/inputs/*.ncl")
+        .to_str()
+        .unwrap()
+        .to_owned();
+    for f in glob(&files).unwrap() {
+        benchmark_one_test(c, f.unwrap().to_str().unwrap());
+    }
+}
+
+fn benchmark_one_test(c: &mut Criterion, path: &str) {
+    let full_path = project_root().join(path);
+    let contents = std::fs::read_to_string(&full_path).unwrap();
+    let fixture = TestFixture::parse(&contents).unwrap();
+    let mut harness = TestHarness::new();
+
+    harness.prepare_files(&fixture);
+
+    for (i, req) in fixture.reqs.iter().enumerate() {
+        let path = friendly_path(&full_path);
+        let name = format!("requests-{path}-{i:03}");
+        c.bench_function(&name, |b| b.iter(|| harness.request_dyn(req.clone())));
+    }
+}
+
+fn test_init(c: &mut Criterion) {
+    let files = project_root()
+        .join("lsp/nls/tests/inputs/*.ncl")
+        .to_str()
+        .unwrap()
+        .to_owned();
+    for f in glob(&files).unwrap() {
+        benchmark_diagnostics(c, f.unwrap().to_str().unwrap());
+    }
+}
+
+// Measure how long it takes from the time the file is sent to the LSP
+// until its diagnostics are received.
+fn benchmark_diagnostics(c: &mut Criterion, path: &str) {
+    let full_path = project_root().join(path);
+    let contents = std::fs::read_to_string(&full_path).unwrap();
+    let fixture = TestFixture::parse(&contents).unwrap();
+
+    for (i, f) in fixture.files.into_iter().enumerate() {
+        let path = friendly_path(&full_path);
+        let name = format!("init-diagnostics-{path}-{i:03}");
+        c.bench_function(&name, |b| {
+            let mut harness = TestHarness::new();
+            b.iter(|| {
+                harness.send_file(f.uri.clone(), &f.contents);
+                loop {
+                    // Check the uri of the diagnostics, because in
+                    // the presence of imports we want to wait until
+                    // the main file sends its diagnostics back.
+                    let diags = harness.wait_for_diagnostics();
+                    if diags.uri == f.uri {
+                        break;
+                    }
+                }
+            });
+        });
+    }
+}

lsp/nls/src/server.rs

@@ -221,13 +221,13 @@ impl Server {
             }
 
             GotoDefinition::METHOD => {
-                debug!("handle goto defnition");
+                debug!("handle goto definition");
                 let params: GotoDefinitionParams = serde_json::from_value(req.params).unwrap();
                 goto::handle_to_definition(params, req.id.clone(), self)
             }
 
             References::METHOD => {
-                debug!("handle goto defnition");
+                debug!("handle goto definition");
                 let params: ReferenceParams = serde_json::from_value(req.params).unwrap();
                 goto::handle_references(params, req.id.clone(), self)
             }

lsp/nls/tests/main.rs

@@ -1,62 +1,7 @@
-use std::collections::{hash_map::Entry, HashMap};
-
-use assert_cmd::cargo::CommandCargoExt;
-use lsp_types::request::{
-    Completion, DocumentSymbolRequest, Formatting, GotoDefinition, HoverRequest, References,
-    Request as LspRequest,
-};
 use nickel_lang_utils::project_root::project_root;
 use test_generator::test_resources;
 
-use lsp_harness::{LspDebug, Request, Server, TestFixture};
-
-struct TestHarness {
-    srv: Server,
-    out: Vec<u8>,
-}
-
-impl TestHarness {
-    fn new() -> Self {
-        let cmd = std::process::Command::cargo_bin("nls").unwrap();
-        let srv = Server::new(cmd).unwrap();
-        Self {
-            srv,
-            out: Vec::new(),
-        }
-    }
-
-    fn request<T: LspRequest>(&mut self, params: T::Params)
-    where
-        T::Result: LspDebug,
-    {
-        let result = self.srv.send_request::<T>(params).unwrap();
-        result.debug(&mut self.out).unwrap();
-        self.out.push(b'\n');
-    }
-
-    fn request_dyn(&mut self, req: Request) {
-        match req {
-            Request::GotoDefinition(d) => self.request::<GotoDefinition>(d),
-            Request::Completion(c) => self.request::<Completion>(c),
-            Request::Formatting(f) => self.request::<Formatting>(f),
-            Request::Hover(h) => self.request::<HoverRequest>(h),
-            Request::References(r) => self.request::<References>(r),
-            Request::Symbols(s) => self.request::<DocumentSymbolRequest>(s),
-        }
-    }
-
-    // For debug purposes, drain and print notifications.
-    fn drain_notifications(&mut self) {
-        // FIXME: nls doesn't report progress, so we have no way to check whether
-        // it's finished sending notifications. We just retrieve any that we've already
-        // received.
-        // We should also have a better format for printing diagnostics and other
-        // notifications.
-        for msg in self.srv.pending_notifications() {
-            eprintln!("{msg:?}");
-        }
-    }
-}
+use lsp_harness::{TestFixture, TestHarness};
 
 #[test_resources("lsp/nls/tests/inputs/*.ncl")]
 fn check_snapshots(path: &str) {
@@ -68,27 +13,7 @@ fn check_snapshots(path: &str) {
     let fixture = TestFixture::parse(&contents).unwrap();
     let mut harness = TestHarness::new();
 
-    let mut file_versions = HashMap::new();
-
-    if fixture.files.is_empty() {
-        panic!("no files");
-    }
-
-    for file in fixture.files {
-        match file_versions.entry(file.uri.clone()) {
-            Entry::Occupied(mut version) => {
-                *version.get_mut() += 1;
-                harness
-                    .srv
-                    .replace_file(file.uri, *version.get(), &file.contents)
-                    .unwrap();
-            }
-            Entry::Vacant(entry) => {
-                harness.srv.send_file(file.uri, &file.contents).unwrap();
-                entry.insert(1);
-            }
-        }
-    }
+    harness.prepare_files(&fixture);
     for req in fixture.reqs {
         harness.request_dyn(req);
     }