Merge branch 'trunk' into layout-cache

.github/workflows/ci.yml

@@ -13,8 +13,8 @@ jobs:
       - name: Verify compiler/builtin/bitcode/regenerate.sh was run if necessary
         run: pushd compiler/builtins/bitcode && ./regenerate.sh && git diff --exit-code ../../gen/src/llvm/builtins.bc && popd
 
-      - name: Install LLVM
-        run: sudo ./ci/install-llvm.sh 10
+      - name: Install CI Libraries
+        run: sudo ./ci/install-ci-libraries.sh 10
 
       - name: Enable LLD
         run: sudo ./ci/enable-lld.sh

BUILDING_FROM_SOURCE.md

@@ -1,13 +1,17 @@
 # Building the Roc compiler from source
 
 
-## Installing LLVM, libunwind, and libc++-dev
+## Installing LLVM, valgrind, libunwind, and libc++-dev
 
 To build the compiler, you need these installed:
 
 * `libunwind` (macOS should already have this one installed)
 * `libc++-dev`
-* a particular version of LLVM 
+* a particular version of LLVM
+
+To run the test suite (via `cargo test`), you additionally need to install:
+
+* [`valgrind`](https://www.valgrind.org/)
 
 Some systems may already have `libc++-dev` on them, but if not, you may need to install it. (On Ubuntu, this can be done with `sudo apt-get install libc++-dev`.) macOS systems
 should already have `libunwind`, but other systems will need to install it

Cargo.lock

@@ -2317,6 +2317,8 @@ dependencies = [
  "roc_types",
  "roc_unify",
  "roc_uniq",
+ "serde",
+ "serde-xml-rs",
  "strip-ansi-escapes",
  "target-lexicon",
  "tokio",
@@ -2748,6 +2750,21 @@ name = "serde"
 version = "1.0.116"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde-xml-rs"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efe415925cf3d0bbb2fc47d09b56ce03eef51c5d56846468a39bcc293c7a846c"
+dependencies = [
+ "log",
+ "serde",
+ "thiserror",
+ "xml-rs",
+]
 
 [[package]]
 name = "serde_cbor"
@@ -2980,6 +2997,26 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
+dependencies = [
+ "proc-macro2 1.0.21",
+ "quote 1.0.7",
+ "syn 1.0.40",
+]
+
 [[package]]
 name = "thread-id"
 version = "3.3.0"

ci/install-ci-libraries.sh

@@ -59,4 +59,4 @@ esac
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 add-apt-repository "${REPO_NAME}"
 apt-get update
-apt-get install -y clang-$LLVM_VERSION lldb-$LLVM_VERSION lld-$LLVM_VERSION clangd-$LLVM_VERSION libc++abi-dev libunwind-dev
+apt-get install -y clang-$LLVM_VERSION lldb-$LLVM_VERSION lld-$LLVM_VERSION clangd-$LLVM_VERSION libc++abi-dev libunwind-dev valgrind

cli/Cargo.toml

@@ -86,3 +86,5 @@ indoc = "0.3.3"
 quickcheck = "0.8"
 quickcheck_macros = "0.8"
 strip-ansi-escapes = "0.1"
+serde = { version = "1.0", features = ["derive"] }
+serde-xml-rs = "0.4"

cli/tests/cli_run.rs

@@ -11,55 +11,119 @@ mod helpers;
 
 #[cfg(test)]
 mod cli_run {
-    use crate::helpers::{example_file, run_roc};
+    use crate::helpers::{example_file, extract_valgrind_errors, run_roc, run_with_valgrind, Out};
 
     #[test]
     fn run_hello_world() {
-        let out = run_roc(&[
-            "run",
-            example_file("hello-world", "Hello.roc").to_str().unwrap(),
-        ]);
+        fn check_hello_world_output(out: Out) {
+            if !out.stderr.is_empty() {
+                panic!(out.stderr);
+            }
+            assert!(out.status.success());
 
-        if !out.stderr.is_empty() {
-            panic!(out.stderr);
+            let valgrind_out =
+                run_with_valgrind(&[example_file("hello-world", "app").to_str().unwrap()]);
+            assert!(valgrind_out.status.success());
+            let ending = "Hello, World!!!!!!!!!!!!!\n";
+            if !&valgrind_out.stdout.ends_with(ending) {
+                panic!(
+                    "expected output to end with {:?} but instead got {:?}",
+                    ending, &valgrind_out.stdout
+                );
+            }
+            let memory_errors = extract_valgrind_errors(&valgrind_out.stderr);
+            if !memory_errors.is_empty() {
+                panic!("{:?}", memory_errors);
+            }
         }
-        assert!(&out.stdout.ends_with("Hello, World!!!!!!!!!!!!!\n"));
-        assert!(out.status.success());
+        check_hello_world_output(run_roc(&[
+            "build",
+            example_file("hello-world", "Hello.roc").to_str().unwrap(),
+        ]));
+        check_hello_world_output(run_roc(&[
+            "build",
+            "--optimize",
+            example_file("hello-world", "Hello.roc").to_str().unwrap(),
+        ]));
     }
 
     #[test]
     fn run_quicksort() {
-        let out = run_roc(&[
-            "run",
-            example_file("quicksort", "Quicksort.roc").to_str().unwrap(),
-            "--optimize",
-        ]);
+        fn check_quicksort_output(out: Out) {
+            if !out.stderr.is_empty() {
+                panic!(out.stderr);
+            }
+            assert!(out.status.success());
 
-        if !out.stderr.is_empty() {
-            panic!(out.stderr);
+            let valgrind_out =
+                run_with_valgrind(&[example_file("quicksort", "app").to_str().unwrap()]);
+            assert!(valgrind_out.status.success());
+            let ending = "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n";
+            if !&valgrind_out.stdout.ends_with(ending) {
+                panic!(
+                    "expected output to end with {:?} but instead got {:?}",
+                    ending, &valgrind_out.stdout
+                );
+            }
+            let memory_errors = extract_valgrind_errors(&valgrind_out.stderr);
+            if !memory_errors.is_empty() {
+                panic!("{:?}", memory_errors);
+            }
         }
-        assert!(&out
-            .stdout
-            .ends_with("[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"));
-        assert!(out.status.success());
+
+        // TODO: Uncomment this once we are correctly freeing the RocList even when in dev build.
+        /*
+        check_quicksort_output(run_roc(&[
+            "build",
+            example_file("quicksort", "Quicksort.roc").to_str().unwrap(),
+        ]));
+        */
+        check_quicksort_output(run_roc(&[
+            "build",
+            "--optimize",
+            example_file("quicksort", "Quicksort.roc").to_str().unwrap(),
+        ]));
     }
 
     #[test]
     fn run_multi_module() {
-        let out = run_roc(&[
+        fn check_muti_module_output(out: Out) {
+            if !out.stderr.is_empty() {
+                panic!(out.stderr);
+            }
+            assert!(out.status.success());
+
+            let valgrind_out =
+                run_with_valgrind(&[example_file("multi-module", "app").to_str().unwrap()]);
+            assert!(valgrind_out.status.success());
+            let ending = "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n";
+            if !&valgrind_out.stdout.ends_with(ending) {
+                panic!(
+                    "expected output to end with {:?} but instead got {:?}",
+                    ending, &valgrind_out.stdout
+                );
+            }
+            let memory_errors = extract_valgrind_errors(&valgrind_out.stderr);
+            if !memory_errors.is_empty() {
+                panic!("{:?}", memory_errors);
+            }
+        }
+
+        // TODO: Uncomment this once we are correctly freeing the RocList even when in dev build.
+        /*
+        check_muti_module_output(run_roc(&[
+            "run",
+            example_file("multi-module", "Quicksort.roc")
+                .to_str()
+                .unwrap(),
+        ]));
+        */
+        check_muti_module_output(run_roc(&[
             "run",
             example_file("multi-module", "Quicksort.roc")
                 .to_str()
                 .unwrap(),
             "--optimize",
-        ]);
-
-        if !out.stderr.is_empty() {
-            panic!(out.stderr);
-        }
-        assert!(&out
-            .stdout
-            .ends_with("[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"));
-        assert!(out.status.success());
+        ]));
     }
 }

cli/tests/helpers.rs

@@ -5,6 +5,8 @@ extern crate roc_load;
 extern crate roc_module;
 
 use roc_cli::repl::{INSTRUCTIONS, PROMPT, WELCOME_MESSAGE};
+use serde::Deserialize;
+use serde_xml_rs::from_str;
 use std::env;
 use std::io::Write;
 use std::path::PathBuf;
@@ -54,6 +56,86 @@ pub fn run_roc(args: &[&str]) -> Out {
     }
 }
 
+#[allow(dead_code)]
+pub fn run_with_valgrind(args: &[&str]) -> Out {
+    //TODO: figure out if there is a better way to get the valgrind executable.
+    let mut cmd = Command::new("valgrind");
+
+    cmd.arg("--tool=memcheck");
+    cmd.arg("--xml=yes");
+    cmd.arg("--xml-fd=2");
+    for arg in args {
+        cmd.arg(arg);
+    }
+
+    let output = cmd
+        .output()
+        .expect("failed to execute compiled `valgrind` binary in CLI test");
+
+    Out {
+        stdout: String::from_utf8(output.stdout).unwrap(),
+        stderr: String::from_utf8(output.stderr).unwrap(),
+        status: output.status,
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct ValgrindOutput {
+    #[serde(rename = "$value")]
+    pub fields: Vec<ValgrindField>,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "lowercase")]
+enum ValgrindField {
+    ProtocolVersion(isize),
+    ProtocolTool(String),
+    Preamble(ValgrindDummyStruct),
+    Pid(isize),
+    PPid(isize),
+    Tool(String),
+    Args(ValgrindDummyStruct),
+    Error(ValgrindError),
+    Status(ValgrindDummyStruct),
+    ErrorCounts(ValgrindDummyStruct),
+    SuppCounts(ValgrindDummyStruct),
+}
+
+#[derive(Debug, Deserialize)]
+struct ValgrindDummyStruct {}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct ValgrindError {
+    kind: String,
+    #[serde(default)]
+    what: Option<String>,
+    #[serde(default)]
+    xwhat: Option<ValgrindErrorXWhat>,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct ValgrindErrorXWhat {
+    text: String,
+    #[serde(default)]
+    leakedbytes: Option<isize>,
+    #[serde(default)]
+    leakedblocks: Option<isize>,
+}
+
+#[allow(dead_code)]
+pub fn extract_valgrind_errors(xml: &str) -> Vec<ValgrindError> {
+    let parsed_xml: ValgrindOutput =
+        from_str(xml).expect("failed to parse the `valgrind` xml output");
+    parsed_xml
+        .fields
+        .iter()
+        .filter_map(|field| match field {
+            ValgrindField::Error(err) => Some(err.clone()),
+            _ => None,
+        })
+        .collect()
+}
+
 #[allow(dead_code)]
 pub fn example_dir(dir_name: &str) -> PathBuf {
     let mut path = env::current_exe().ok().unwrap();

examples/multi-module/platform/src/lib.rs

@@ -6,7 +6,7 @@ extern "C" {
     fn quicksort(list: RocList<i64>) -> RocList<i64>;
 }
 
-const NUM_NUMS: usize = 1_000_000;
+const NUM_NUMS: usize = 10_000;
 
 #[no_mangle]
 pub fn rust_main() -> isize {
@@ -14,7 +14,7 @@ pub fn rust_main() -> isize {
         let mut nums = Vec::with_capacity(NUM_NUMS);
 
         for index in 0..nums.capacity() {
-            let num = index as i64 % 12345;
+            let num = index as i64 % 123;
 
             nums.push(num);
         }

examples/shared-quicksort/platform/host.rs

@@ -8,7 +8,7 @@ extern "C" {
     fn quicksort(list: Box<[i64]>) -> Box<[i64]>;
 }
 
-const NUM_NUMS: usize = 1_000_000;
+const NUM_NUMS: usize = 10_000;
 
 #[no_mangle]
 pub fn rust_main() -> isize {
@@ -16,7 +16,7 @@ pub fn rust_main() -> isize {
         let mut nums = Vec::with_capacity(NUM_NUMS);
 
         for index in 0..nums.capacity() {
-            let num = index as i64 % 12345;
+            let num = index as i64 % 123;
 
             nums.push(num);
         }