Replace Go with Java-based httpbin server (#3894)

build/build/src/engine/context.rs

@@ -391,7 +391,7 @@ impl RunContext {
 
         let enso = BuiltEnso { paths: self.paths.clone() };
         if self.config.test_standard_library {
-            enso.run_tests(IrCaches::No, PARALLEL_ENSO_TESTS).await?;
+            enso.run_tests(IrCaches::No, &sbt, PARALLEL_ENSO_TESTS).await?;
         }
 
         if self.config.build_engine_package() {
@@ -406,7 +406,7 @@ impl RunContext {
         }
 
         if self.config.test_standard_library {
-            enso.run_tests(IrCaches::Yes, PARALLEL_ENSO_TESTS).await?;
+            enso.run_tests(IrCaches::Yes, &sbt, PARALLEL_ENSO_TESTS).await?;
         }
 
         // if build_native_runner {

build/build/src/enso.rs

@@ -74,7 +74,12 @@ impl BuiltEnso {
         Ok(command)
     }
 
-    pub async fn run_tests(&self, ir_caches: IrCaches, async_policy: AsyncPolicy) -> Result {
+    pub async fn run_tests(
+        &self,
+        ir_caches: IrCaches,
+        sbt: &crate::engine::sbt::Context,
+        async_policy: AsyncPolicy,
+    ) -> Result {
         let paths = &self.paths;
         // Prepare Engine Test Environment
         if let Ok(gdoc_key) = std::env::var("GDOC_KEY") {
@@ -84,7 +89,7 @@ impl BuiltEnso {
             ide_ci::fs::write(google_api_test_data_dir.join("secret.json"), &gdoc_key)?;
         }
 
-        let _httpbin = crate::httpbin::get_and_spawn_httpbin_on_free_port().await?;
+        let _httpbin = crate::httpbin::get_and_spawn_httpbin_on_free_port(sbt).await?;
         let _postgres = match TARGET_OS {
             OS::Linux => {
                 let runner_context_string = crate::env::ENSO_RUNNER_CONTAINER_NAME

build/build/src/httpbin.rs

@@ -1,6 +1,6 @@
 use crate::prelude::*;
 
-use ide_ci::programs::Go;
+use ide_ci::extensions::child::ChildExt;
 use tokio::process::Child;
 
 
@@ -14,27 +14,24 @@ pub mod env {
     }
 }
 
+/// Handle to the spawned httpbin server.
+///
+/// It kills the process when dropped.
 #[derive(Debug)]
 pub struct Spawned {
     pub process: Child,
     pub url:     Url,
 }
 
-pub async fn get_and_spawn_httpbin(port: u16) -> Result<Spawned> {
+pub async fn get_and_spawn_httpbin(
-    Go.cmd()?
+    sbt: &crate::engine::sbt::Context,
-        .args(["install", "-v", "github.com/ahmetb/go-httpbin/cmd/httpbin@latest"])
+    port: u16,
-        .run_ok()
+) -> Result<Spawned> {
-        .await?;
+    let process = sbt
-    let gopath = Go.cmd()?.args(["env", "GOPATH"]).run_stdout().await?;
+        .command()?
-    let gopath = gopath.trim();
+        .arg(format!("simple-httpbin/run localhost {port}"))
-    let gopath = PathBuf::from(gopath); // be careful of trailing newline!
-    let program = gopath.join("bin").join("httpbin");
-    debug!("Will spawn {}", program.display());
-    let process = Command::new(program) // TODO? wrap in Program?
-        .args(["-host", &format!(":{port}")])
         .kill_on_drop(true)
-        .spawn_intercepting()
+        .spawn()?;
-        .anyhow_err()?;
 
     let url_string = format!("http://localhost:{port}");
     let url = Url::parse(&url_string)?;
@@ -46,16 +43,22 @@ impl Drop for Spawned {
     fn drop(&mut self) {
         debug!("Dropping the httpbin wrapper.");
         env::ENSO_HTTP_TEST_HTTPBIN_URL.remove();
+        self.process.kill_subtree();
     }
 }
 
-pub async fn get_and_spawn_httpbin_on_free_port() -> Result<Spawned> {
+pub async fn get_and_spawn_httpbin_on_free_port(
-    get_and_spawn_httpbin(ide_ci::get_free_port()?).await
+    sbt: &crate::engine::sbt::Context,
+) -> Result<Spawned> {
+    get_and_spawn_httpbin(sbt, ide_ci::get_free_port()?).await
 }
 
+
 #[cfg(test)]
 mod tests {
-    use crate::project::ProcessWrapper;
+    use ide_ci::cache;
+    use ide_ci::env::current_dir;
+    use std::env::set_current_dir;
 
     use super::*;
 
@@ -63,9 +66,21 @@ mod tests {
     #[tokio::test]
     #[ignore]
     async fn spawn() -> Result {
-        let mut spawned = get_and_spawn_httpbin_on_free_port().await?;
+        setup_logging()?;
+        set_current_dir(r"H:\NBO\enso5")?;
+        let cache = cache::Cache::new_default().await?;
+        cache::goodie::sbt::Sbt.install_if_missing(&cache).await?;
+
+        let sbt = crate::engine::sbt::Context {
+            repo_root:         current_dir()?,
+            system_properties: vec![],
+        };
+
+        let spawned = get_and_spawn_httpbin_on_free_port(&sbt).await?;
+        std::thread::sleep(std::time::Duration::from_secs(20));
         dbg!(&spawned);
-        spawned.process.wait_ok().await?;
+
+
         Ok(())
     }
 }

build/ci_utils/src/extensions/child.rs

@@ -1,13 +1,32 @@
 use crate::prelude::*;
 
+use sysinfo::Pid;
 
 
+
+/// Extension methods for [`tokio::process::Child`].
 pub trait ChildExt {
+    /// Wait for the process completion and represent non-zero exit code as an error.
     fn wait_ok(&mut self) -> BoxFuture<Result>;
+
+    /// Kill the process and all its descendants.
+    ///
+    /// Note that in case of partial failures, the function will at most log the error and continue.
+    fn kill_subtree(&self);
 }
 
 impl ChildExt for tokio::process::Child {
     fn wait_ok(&mut self) -> BoxFuture<Result> {
         async move { self.wait().await?.exit_ok().anyhow_err() }.boxed()
     }
+
+    fn kill_subtree(&self) {
+        let Some(pid) = self.id().map(Pid::from_u32) else {
+            // Not necessarily that bad, as the process might have already exited. 
+            // Still, we don't know about its descendants, so we cannot kill them.
+            warn!("Failed to get PID of the process.");
+            return;
+        };
+        crate::process::kill_process_subtree(pid)
+    }
 }

build/ci_utils/src/lib.rs

@@ -62,6 +62,7 @@ pub mod os;
 pub mod path;
 pub mod paths;
 pub mod platform;
+pub mod process;
 pub mod program;
 pub mod programs;
 pub mod reqwest;
@@ -85,6 +86,9 @@ pub mod prelude {
     pub use platforms::target::OS;
     pub use semver::Version;
     pub use shrinkwraprs::Shrinkwrap;
+    pub use sysinfo::PidExt as _;
+    pub use sysinfo::ProcessExt as _;
+    pub use sysinfo::SystemExt as _;
     pub use tokio::io::AsyncWriteExt as _;
     pub use url::Url;
     pub use uuid::Uuid;

build/ci_utils/src/process.rs

@@ -0,0 +1,22 @@
+use crate::prelude::*;
+
+use sysinfo::Pid;
+
+
+// ==============
+// === Export ===
+// ==============
+
+pub mod hierarchy;
+
+
+
+/// Kills the process and all its descendants.
+///
+/// Note that in case of partial failures, the function will at most log the error and continue.
+/// As much processes as possible will receive the kill signal.
+#[instrument]
+pub fn kill_process_subtree(pid: Pid) {
+    let mut system = sysinfo::System::new();
+    hierarchy::Hierarchy::new(&mut system).kill_process_subtree(pid);
+}

build/ci_utils/src/process/hierarchy.rs

@@ -0,0 +1,63 @@
+use crate::prelude::*;
+
+use sysinfo::Pid;
+use sysinfo::Process;
+use sysinfo::ProcessRefreshKind;
+use sysinfo::System;
+
+
+
+/// A wrapper over [`System`] that represents information about the process hierarchy.
+#[derive(Debug, Clone)]
+pub struct Hierarchy<'a> {
+    /// Data about all known processes.
+    pub processes: &'a HashMap<Pid, Process>,
+    /// Children processes of each process.
+    pub children:  HashMap<Pid, HashSet<Pid>>,
+}
+
+impl<'a> Hierarchy<'a> {
+    /// Creates a new instance of the process hierarchy.
+    ///
+    /// The system will be used to refresh the process information.
+    pub fn new(system: &'a mut System) -> Self {
+        trace!("Refreshing system information.");
+        system.refresh_processes_specifics(ProcessRefreshKind::default());
+        let processes = system.processes();
+        let mut children = HashMap::<_, HashSet<Pid>>::new();
+        for (pid, process) in processes {
+            let parent_pid = process.parent();
+            if let Some(parent_pid) = parent_pid {
+                children.entry(parent_pid).or_default().insert(*pid);
+            } else {
+                // Not really an error, some processes might not have a parent, like "System" or
+                // "System Idle Process". Also, we might not have permissions to see the parent of
+                // some processes.
+                trace!(%pid, "Process has no parent information.");
+            }
+        }
+        Self { processes, children }
+    }
+
+    /// Kills the process and all its descendants.
+    ///
+    /// Note that in case of partial failures, the function will at most log the error and continue.
+    /// As much processes as possible will receive the kill signal.
+    pub fn kill_process_subtree(&self, pid: Pid) {
+        if let Some(children) = self.children.get(&pid) {
+            for child in children {
+                self.kill_process_subtree(*child);
+            }
+        }
+        if let Some(process) = self.processes.get(&pid) {
+            let name = process.name();
+            let command = process.cmd();
+            trace!(%pid, %name, ?command, "Killing process.");
+            if !process.kill() {
+                warn!(%pid, %name, ?command, "Failed to kill process.");
+            }
+        } else {
+            warn!(%pid, "Failed to kill process. It does not exist.");
+        }
+    }
+}