roc-lang/roc
1
1
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2024-09-20 23:37:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into simplify_examples

Browse Source 
Signed-off-by: Anton-4 <17049058+Anton-4@users.noreply.github.com>
This commit is contained in:
Anton-4 2022-09-28 11:54:44 +02:00 committed by GitHub
commit 735d4c13bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 3568 additions and 3077 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -96,3 +96,4 @@ Rod <randomer@users.noreply.github.com>
Marko Vujanic <crashxx@gmail.com>
KilianVounckx <kilianvounckx@hotmail.be>
David Dunn <26876072+doubledup@users.noreply.github.com>
Jelle Besseling <jelle@pingiun.com>

20
Cargo.lock generated
View File

@ -1871,9 +1871,9 @@ checksum = "90953f308a79fe6d62a4643e51f848fbfddcd05975a38e69fdf4ab86a7baf7ca"
[[package]]
name = "insta"
version = "1.19.0"
version = "1.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f37dc501f12ec0c339b385787fa89ffda3d5d2caa62e558da731134c24d6e0c4"
checksum = "58a931b01c76064c5be919faa2ef0dc570e9a889dcd1e5fef08a8ca6eb4d6c0b"
dependencies = [
"console",
"linked-hash-map",
@ -2776,9 +2776,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "peg"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af728fe826811af3b38c37e93de6d104485953ea373d656eebae53d6987fcd2c"
checksum = "a07f2cafdc3babeebc087e499118343442b742cc7c31b4d054682cc598508554"
dependencies = [
"peg-macros",
"peg-runtime",
@ -2786,9 +2786,9 @@ dependencies = [
[[package]]
name = "peg-macros"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4536be147b770b824895cbad934fccce8e49f14b4c4946eaa46a6e4a12fcdc16"
checksum = "4a90084dc05cf0428428e3d12399f39faad19b0909f64fb9170c9fdd6d9cd49b"
dependencies = [
"peg-runtime",
"proc-macro2",
@ -2797,9 +2797,9 @@ dependencies = [
[[package]]
name = "peg-runtime"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9b0efd3ba03c3a409d44d60425f279ec442bcf0b9e63ff4e410da31c8b0f69f"
checksum = "9fa00462b37ead6d11a82c9d568b26682d78e0477dc02d1966c013af80969739"
[[package]]
name = "percent-encoding"
@ -5341,9 +5341,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.32"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad"
checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
dependencies = [
"cfg-if 1.0.0",
"js-sys",

53
TUTORIAL.md
View File

@ -116,10 +116,10 @@ Create a new file called `Hello.roc` and put this inside it:
```coffee
app "hello"
packages { pf: "examples/cli/cli-platform/main.roc" }
imports [pf.Stdout]
imports [pf.Stdout, pf.Program]
provides [main] to pf
main = Stdout.line "I'm a Roc application!"
main = Stdout.line "I'm a Roc application!" |> Program.quick
```
> **NOTE:** This assumes you've put Hello.roc in the root directory of the Roc
@ -146,7 +146,7 @@ this file above `main` do later, but first let's play around a bit.
Try replacing the `main` line with this:
```coffee
main = Stdout.line "There are \(total) animals."
main = Stdout.line "There are \(total) animals." |> Program.quick
birds = 3
@ -166,7 +166,7 @@ short - namely, `main`, `birds`, `iguanas`, and `total`.
A definition names an expression.
- The first def assigns the name `main` to the expression `Stdout.line "There are \(total) animals."`. The `Stdout.line` function takes a string and prints it as a line to [`stdout`] (the terminal's standard output device).
- The first def assigns the name `main` to the expression `Stdout.line "There are \(total) animals." |> Program.quick`. The `Stdout.line` function takes a string and prints it as a line to [`stdout`] (the terminal's standard output device). Then `Program.quick` wrap this expression into an executable Roc program.
- The next two defs assign the names `birds` and `iguanas` to the expressions `3` and `2`.
- The last def assigns the name `total` to the expression `Num.toStr (birds + iguanas)`.
@ -201,7 +201,7 @@ So far we've called functions like `Num.toStr`, `Str.concat`, and `Stdout.line`.
Next let's try defining a function of our own.
```coffee
main = Stdout.line "There are \(total) animals."
main = Stdout.line "There are \(total) animals." |> Program.quick
birds = 3
@ -1258,7 +1258,7 @@ Let's take a closer look at the part of `Hello.roc` above `main`:
```coffee
app "hello"
packages { pf: "examples/cli/cli-platform/main.roc" }
imports [pf.Stdout]
imports [pf.Stdout, pf.Program]
provides main to pf
```
@ -1276,7 +1276,7 @@ The remaining lines all involve the *platform* this application is built on:
```coffee
packages { pf: "examples/cli/cli-platform/main.roc" }
imports [pf.Stdout]
imports [pf.Stdout, pf.Program]
provides main to pf
```
@ -1285,22 +1285,24 @@ The `packages { pf: "examples/cli/cli-platform/main.roc" }` part says two things
- We're going to be using a *package* (that is, a collection of modules) called `"examples/cli/cli-platform/main.roc"`
- We're going to name that package `pf` so we can refer to it more concisely in the future.
The `imports [pf.Stdout]` line says that we want to import the `Stdout` module
from the `pf` package, and make it available in the current module.
The `imports [pf.Stdout, pf.Program]` line says that we want to import the `Stdout` and `Program` modules
from the `pf` package, and make them available in the current module.
This import has a direct interaction with our definition of `main`. Let's look
at that again:
```coffee
main = Stdout.line "I'm a Roc application!"
main = Stdout.line "I'm a Roc application!" |> Program.quick
```
Here, `main` is calling a function called `Stdout.line`. More specifically, it's
calling a function named `line` which is exposed by a module named
`Stdout`.
Then the result of that function call is passed to the `quick` function of the `Program` module,
which effectively makes it a simple Roc program.
When we write `imports [pf.Stdout]`, it specifies that the `Stdout`
module comes from the `pf` package.
When we write `imports [pf.Stdout, pf.Program]`, it specifies that the `Stdout`
and `Program` modules come from the `pf` package.
Since `pf` was the name we chose for the `examples/cli/cli-platform/main.roc`
package (when we wrote `packages { pf: "examples/cli/cli-platform/main.roc" }`),
@ -1327,11 +1329,12 @@ First, let's do a basic "Hello World" using the tutorial app.
```coffee
app "cli-tutorial"
packages { pf: "examples/cli/cli-platform/main.roc" }
imports [pf.Stdout]
imports [pf.Stdout, pf.Program]
provides [main] to pf
main =
Stdout.line "Hello, World!"
|> Program.quick
```
The `Stdout.line` function takes a `Str` and writes it to [standard output](https://en.wikipedia.org/wiki/Standard_streams#Standard_output_(stdout)).
@ -1364,10 +1367,12 @@ Let's change `main` to read a line from `stdin`, and then print it back out agai
```swift
app "cli-tutorial"
packages { pf: "examples/cli/cli-platform/main.roc" }
imports [pf.Stdout, pf.Stdin, pf.Task]
imports [pf.Stdout, pf.Stdin, pf.Task, pf.Program]
provides [main] to pf
main =
main = Program.quick task
task =
Task.await Stdin.line \text ->
Stdout.line "You just entered: \(text)"
```
@ -1395,7 +1400,7 @@ we did in our `\text -> …` callback function here:
Stdout.line "You just entered: \(text)"
```
Notice that, just like before, we're still setting `main` to be a single `Task`. This is how we'll
Notice that, just like before, we're still building `main` from a single `Task`. This is how we'll
always do it! We'll keep building up bigger and bigger `Task`s out of smaller tasks, and then setting
`main` to be that one big `Task`.
@ -1403,7 +1408,7 @@ For example, we can print a prompt before we pause to read from `stdin`, so it n
the program isn't doing anything when we start it up:
```swift
main =
task =
Task.await (Stdout.line "Type something press Enter:") \_ ->
Task.await Stdin.line \text ->
Stdout.line "You just entered: \(text)"
@ -1414,10 +1419,12 @@ This works, but we can make it a little nicer to read. Let's change it to the fo
```haskell
app "cli-tutorial"
packages { pf: "examples/cli/cli-platform/main.roc" }
imports [pf.Stdout, pf.Stdin, pf.Task.{ await }]
imports [pf.Stdout, pf.Stdin, pf.Task.{ await }, pf.Program]
provides [main] to pf
main =
main = Program.quick task
task =
await (Stdout.line "Type something press Enter:") \_ ->
await Stdin.line \text ->
Stdout.line "You just entered: \(text)"
@ -1436,10 +1443,10 @@ across a small number of lines of code.
Speaking of calling `await` repeatedly, if we keep calling it more and more on this
code, we'll end up doing a lot of indenting. If we'd rather not indent so much, we
can rewrite `main` into this style which looks different but does the same thing:
can rewrite `task` into this style which looks different but does the same thing:
```swift
main =
task =
_ <- await (Stdout.line "Type something press Enter:")
text <- await Stdin.line
@ -1512,11 +1519,11 @@ which is totally allowed! Since backpassing is nothing more than syntax sugar fo
defining a function and passing back as an argument to another function, there's no
reason we can't mix and match if we like.
That said, the typical style in which this `main` would be written in Roc is using
That said, the typical style in which this `task` would be written in Roc is using
backpassing for all the `await` calls, like we had above:
```swift
main =
task =
_ <- await (Stdout.line "Type something press Enter:")
text <- await Stdin.line

38
crates/cli/src/build.rs
View File

@ -191,12 +191,6 @@ pub fn build_file<'a>(
exposed_closure_types,
);
let app_o_file = Builder::new()
.prefix("roc_app")
.suffix(&format!(".{}", app_extension))
.tempfile()
.map_err(|err| todo!("TODO Gracefully handle tempfile creation error {:?}", err))?;
let app_o_file = app_o_file.path();
let buf = &mut String::with_capacity(1024);
let mut it = loaded.timings.iter().peekable();
@ -270,12 +264,11 @@ pub fn build_file<'a>(
HostRebuildTiming::ConcurrentWithApp(rebuild_thread)
};
let code_gen_timing = program::gen_from_mono_module(
let (roc_app_bytes, code_gen_timing) = program::gen_from_mono_module(
arena,
loaded,
&app_module_path,
target,
app_o_file,
opt_level,
emit_debug_info,
&preprocessed_host_path,
@ -292,18 +285,10 @@ pub fn build_file<'a>(
"Generate Assembly from Mono IR",
code_gen_timing.code_gen,
);
report_timing(buf, "Emit .o file", code_gen_timing.emit_o_file);
let compilation_end = compilation_start.elapsed();
let size = std::fs::metadata(&app_o_file)
.unwrap_or_else(|err| {
panic!(
"Could not open {:?} - which was supposed to have been generated. Error: {:?}",
app_o_file, err
);
})
.len();
let size = roc_app_bytes.len();
if emit_timings {
println!(
@ -332,16 +317,31 @@ pub fn build_file<'a>(
let link_start = Instant::now();
let problems = match (linking_strategy, link_type) {
(LinkingStrategy::Surgical, _) => {
roc_linker::link_preprocessed_host(target, &host_input_path, app_o_file, &binary_path);
roc_linker::link_preprocessed_host(
target,
&host_input_path,
&roc_app_bytes,
&binary_path,
);
problems
}
(LinkingStrategy::Additive, _) | (LinkingStrategy::Legacy, LinkType::None) => {
// Just copy the object file to the output folder.
binary_path.set_extension(app_extension);
std::fs::copy(app_o_file, &binary_path).unwrap();
std::fs::write(&binary_path, &*roc_app_bytes).unwrap();
problems
}
(LinkingStrategy::Legacy, _) => {
let app_o_file = Builder::new()
.prefix("roc_app")
.suffix(&format!(".{}", app_extension))
.tempfile()
.map_err(|err| todo!("TODO Gracefully handle tempfile creation error {:?}", err))?;
let app_o_file = app_o_file.path();
std::fs::write(app_o_file, &*roc_app_bytes).unwrap();
let mut inputs = vec![
host_input_path.as_path().to_str().unwrap(),
app_o_file.to_str().unwrap(),

106
crates/compiler/build/src/program.rs
View File

@ -1,3 +1,4 @@
use inkwell::memory_buffer::MemoryBuffer;
pub use roc_gen_llvm::llvm::build::FunctionIterator;
use roc_gen_llvm::llvm::build::{module_from_builtins, LlvmBackendMode};
use roc_gen_llvm::llvm::externs::add_default_roc_externs;
@ -6,6 +7,7 @@ use roc_module::symbol::{Interns, ModuleId};
use roc_mono::ir::OptLevel;
use roc_region::all::LineInfo;
use roc_solve_problem::TypeError;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
@ -16,7 +18,6 @@ use roc_collections::all::MutSet;
#[derive(Debug, Clone, Copy, Default)]
pub struct CodeGenTiming {
pub code_gen: Duration,
pub emit_o_file: Duration,
}
pub fn report_problems_monomorphized(loaded: &mut MonomorphizedModule) -> Problems {
@ -156,25 +157,39 @@ fn report_problems_help(
}
}
pub enum CodeObject {
MemoryBuffer(MemoryBuffer),
Vector(Vec<u8>),
}
impl Deref for CodeObject {
type Target = [u8];
fn deref(&self) -> &Self::Target {
match self {
CodeObject::MemoryBuffer(memory_buffer) => memory_buffer.as_slice(),
CodeObject::Vector(vector) => vector.as_slice(),
}
}
}
#[allow(clippy::too_many_arguments)]
pub fn gen_from_mono_module(
arena: &bumpalo::Bump,
loaded: MonomorphizedModule,
roc_file_path: &Path,
target: &target_lexicon::Triple,
app_o_file: &Path,
opt_level: OptLevel,
emit_debug_info: bool,
preprocessed_host_path: &Path,
wasm_dev_stack_bytes: Option<u32>,
) -> CodeGenTiming {
) -> (CodeObject, CodeGenTiming) {
match opt_level {
OptLevel::Normal | OptLevel::Size | OptLevel::Optimize => gen_from_mono_module_llvm(
arena,
loaded,
roc_file_path,
target,
app_o_file,
opt_level,
emit_debug_info,
),
@ -182,7 +197,6 @@ pub fn gen_from_mono_module(
arena,
loaded,
target,
app_o_file,
preprocessed_host_path,
wasm_dev_stack_bytes,
),
@ -192,15 +206,14 @@ pub fn gen_from_mono_module(
// TODO how should imported modules factor into this? What if those use builtins too?
// TODO this should probably use more helper functions
// TODO make this polymorphic in the llvm functions so it can be reused for another backend.
pub fn gen_from_mono_module_llvm(
fn gen_from_mono_module_llvm(
arena: &bumpalo::Bump,
loaded: MonomorphizedModule,
roc_file_path: &Path,
target: &target_lexicon::Triple,
app_o_file: &Path,
opt_level: OptLevel,
emit_debug_info: bool,
) -> CodeGenTiming {
) -> (CodeObject, CodeGenTiming) {
use crate::target::{self, convert_opt_level};
use inkwell::attributes::{Attribute, AttributeLoc};
use inkwell::context::Context;
@ -312,12 +325,9 @@ pub fn gen_from_mono_module_llvm(
// Uncomment this to see the module's optimized LLVM instruction output:
// env.module.print_to_stderr();
let code_gen = code_gen_start.elapsed();
let emit_o_file_start = Instant::now();
// annotate the LLVM IR output with debug info
// so errors are reported with the line number of the LLVM source
if emit_debug_info {
let memory_buffer = if emit_debug_info {
module.strip_debug_info();
let mut app_ll_dbg_file = PathBuf::from(roc_file_path);
@ -326,6 +336,9 @@ pub fn gen_from_mono_module_llvm(
let mut app_bc_file = PathBuf::from(roc_file_path);
app_bc_file.set_extension("bc");
let mut app_o_file = PathBuf::from(roc_file_path);
app_o_file.set_extension("o");
use std::process::Command;
// write the ll code to a file, so we can modify it
@ -384,6 +397,8 @@ pub fn gen_from_mono_module_llvm(
}
_ => unreachable!(),
}
MemoryBuffer::create_from_file(&app_o_file).expect("memory buffer creation works")
} else {
// Emit the .o file
use target_lexicon::Architecture;
@ -394,49 +409,48 @@ pub fn gen_from_mono_module_llvm(
target::target_machine(target, convert_opt_level(opt_level), reloc).unwrap();
target_machine
.write_to_file(env.module, FileType::Object, app_o_file)
.expect("Writing .o file failed");
.write_to_memory_buffer(env.module, FileType::Object)
.expect("Writing .o file failed")
}
Architecture::Wasm32 => {
// Useful for debugging
// module.print_to_file(app_ll_file);
module.write_bitcode_to_path(app_o_file);
module.write_bitcode_to_memory()
}
_ => panic!(
"TODO gracefully handle unsupported architecture: {:?}",
target.architecture
),
}
}
};
let emit_o_file = emit_o_file_start.elapsed();
let code_gen = code_gen_start.elapsed();
CodeGenTiming {
code_gen,
emit_o_file,
}
(
CodeObject::MemoryBuffer(memory_buffer),
CodeGenTiming { code_gen },
)
}
#[cfg(feature = "target-wasm32")]
pub fn gen_from_mono_module_dev(
fn gen_from_mono_module_dev(
arena: &bumpalo::Bump,
loaded: MonomorphizedModule,
target: &target_lexicon::Triple,
app_o_file: &Path,
preprocessed_host_path: &Path,
wasm_dev_stack_bytes: Option<u32>,
) -> CodeGenTiming {
) -> (CodeObject, CodeGenTiming) {
use target_lexicon::Architecture;
match target.architecture {
Architecture::Wasm32 => gen_from_mono_module_dev_wasm32(
arena,
loaded,
app_o_file,
preprocessed_host_path,
wasm_dev_stack_bytes,
),
Architecture::X86_64 | Architecture::Aarch64(_) => {
gen_from_mono_module_dev_assembly(arena, loaded, target, app_o_file)
gen_from_mono_module_dev_assembly(arena, loaded, target)
}
_ => todo!(),
}
@ -447,15 +461,14 @@ pub fn gen_from_mono_module_dev(
arena: &bumpalo::Bump,
loaded: MonomorphizedModule,
target: &target_lexicon::Triple,
app_o_file: &Path,
_host_input_path: &Path,
_wasm_dev_stack_bytes: Option<u32>,
) -> CodeGenTiming {
) -> (CodeObject, CodeGenTiming) {
use target_lexicon::Architecture;
match target.architecture {
Architecture::X86_64 | Architecture::Aarch64(_) => {
gen_from_mono_module_dev_assembly(arena, loaded, target, app_o_file)
gen_from_mono_module_dev_assembly(arena, loaded, target)
}
_ => todo!(),
}
@ -465,10 +478,9 @@ pub fn gen_from_mono_module_dev(
fn gen_from_mono_module_dev_wasm32(
arena: &bumpalo::Bump,
loaded: MonomorphizedModule,
app_o_file: &Path,
preprocessed_host_path: &Path,
wasm_dev_stack_bytes: Option<u32>,
) -> CodeGenTiming {
) -> (CodeObject, CodeGenTiming) {
let code_gen_start = Instant::now();
let MonomorphizedModule {
module_id,
@ -513,31 +525,18 @@ fn gen_from_mono_module_dev_wasm32(
roc_gen_wasm::build_app_binary(&env, &mut interns, host_module, procedures);
let code_gen = code_gen_start.elapsed();
let emit_o_file_start = Instant::now();
// The app_o_file is actually the final binary
std::fs::write(&app_o_file, &final_binary_bytes).unwrap_or_else(|e| {
panic!(
"I wasn't able to write to the output file {}\n{}",
app_o_file.display(),
e
)
});
let emit_o_file = emit_o_file_start.elapsed();
CodeGenTiming {
code_gen,
emit_o_file,
}
(
CodeObject::Vector(final_binary_bytes),
CodeGenTiming { code_gen },
)
}
fn gen_from_mono_module_dev_assembly(
arena: &bumpalo::Bump,
loaded: MonomorphizedModule,
target: &target_lexicon::Triple,
app_o_file: &Path,
) -> CodeGenTiming {
) -> (CodeObject, CodeGenTiming) {
let code_gen_start = Instant::now();
let lazy_literals = true;
@ -564,17 +563,10 @@ fn gen_from_mono_module_dev_assembly(
let module_object = roc_gen_dev::build_module(&env, &mut interns, target, procedures);
let code_gen = code_gen_start.elapsed();
let emit_o_file_start = Instant::now();
let module_out = module_object
.write()
.expect("failed to build output object");
std::fs::write(&app_o_file, module_out).expect("failed to write object to file");
let emit_o_file = emit_o_file_start.elapsed();
CodeGenTiming {
code_gen,
emit_o_file,
}
(CodeObject::Vector(module_out), CodeGenTiming { code_gen })
}

2
crates/compiler/solve/Cargo.toml
View File

@ -37,4 +37,4 @@ tempfile = "3.2.0"
bumpalo = { version = "3.11.0", features = ["collections"] }
regex = "1.5.5"
lazy_static = "1.4.0"
insta = "1.19.0"
insta = "1.20.0"

2
crates/compiler/test_derive/Cargo.toml
View File

@ -29,4 +29,4 @@ lazy_static = "1.4.0"
indoc = "1.0.7"
ven_pretty = { path = "../../vendor/pretty" }
pretty_assertions = "1.3.0"
insta = "1.19.0"
insta = "1.20.0"

2
crates/docs/Cargo.toml
View File

@ -22,7 +22,7 @@ roc_highlight = { path = "../highlight"}
roc_reporting = { path = "../reporting"}
bumpalo = { version = "3.11.0", features = ["collections"] }
snafu = { version = "0.7.1", features = ["backtraces"] }
peg = "0.8.0"
peg = "0.8.1"
[dev-dependencies]
pretty_assertions = "1.3.0"

2
crates/highlight/Cargo.toml
View File

@ -7,5 +7,5 @@ edition = "2021"
description = "For syntax highlighting, starts with a string and returns our markup nodes."
[dependencies]
peg = "0.8.0"
peg = "0.8.1"
roc_code_markup = { path = "../code_markup"}

1540
crates/linker/src/elf.rs Normal file
View File

File diff suppressed because it is too large Load Diff

2922
crates/linker/src/lib.rs
View File

File diff suppressed because it is too large Load Diff

1639
crates/linker/src/macho.rs Normal file
View File

File diff suppressed because it is too large Load Diff

29
crates/linker/src/metadata.rs
View File

@ -1,4 +1,11 @@
use std::{
io::{BufReader, BufWriter},
path::Path,
};
use bincode::{deserialize_from, serialize_into};
use roc_collections::all::MutMap;
use roc_error_macros::internal_error;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, PartialEq, Debug)]
@ -36,3 +43,25 @@ pub struct Metadata {
pub symbol_table_size: u64,
pub macho_cmd_loc: u64,
}
impl Metadata {
pub fn write_to_file(&self, metadata_filename: &Path) {
let metadata_file =
std::fs::File::create(metadata_filename).unwrap_or_else(|e| internal_error!("{}", e));
serialize_into(BufWriter::new(metadata_file), self)
.unwrap_or_else(|err| internal_error!("Failed to serialize metadata: {err}"));
}
pub fn read_from_file(metadata_filename: &Path) -> Self {
let input =
std::fs::File::open(metadata_filename).unwrap_or_else(|e| internal_error!("{}", e));
match deserialize_from(BufReader::new(input)) {
Ok(data) => data,
Err(err) => {
internal_error!("Failed to deserialize metadata: {}", err);
}
}
}
}

285
crates/linker/src/pe.rs
View File

@ -2,7 +2,6 @@ use std::{
io::{BufReader, BufWriter},
ops::Range,
path::Path,
time::Instant,
};
use bincode::{deserialize_from, serialize_into};
@ -20,7 +19,9 @@ use serde::{Deserialize, Serialize};
use roc_collections::MutMap;
use roc_error_macros::internal_error;
use crate::{generate_dylib::APP_DLL, load_struct_inplace, load_struct_inplace_mut};
use crate::{
generate_dylib::APP_DLL, load_struct_inplace, load_struct_inplace_mut, open_mmap, open_mmap_mut,
};
/// The metadata stores information about/from the host .exe because
///
@ -39,6 +40,9 @@ struct PeMetadata {
last_host_section_size: u64,
last_host_section_address: u64,
/// Number of sections in the unmodified host .exe
host_section_count: usize,
optional_header_offset: usize,
dynamic_relocations: DynamicRelocationsPe,
@ -58,128 +62,16 @@ struct PeMetadata {
exports: MutMap<String, i64>,
}
pub(crate) fn preprocess_windows(
host_exe_filename: &str,
metadata_filename: &Path,
out_filename: &Path,
_shared_lib: &Path,
_verbose: bool,
_time: bool,
) -> object::read::Result<()> {
use object::ObjectSection;
impl PeMetadata {
fn write_to_file(&self, metadata_filename: &Path) {
let metadata_file =
std::fs::File::create(metadata_filename).unwrap_or_else(|e| internal_error!("{}", e));
let total_start = Instant::now();
let exec_parsing_start = total_start;
let _exec_parsing_duration = exec_parsing_start.elapsed();
let data = std::fs::read(host_exe_filename).unwrap();
let new_sections = [*b".text\0\0\0", *b".rdata\0\0"];
let mut dynhost = Preprocessor::preprocess(out_filename, &data, &new_sections);
let dynhost_data: &[u8] = &*dynhost;
let dynhost_obj = match object::read::pe::PeFile64::parse(dynhost_data) {
Ok(obj) => obj,
Err(err) => {
internal_error!("Failed to parse executable file: {}", err);
}
};
// -2 because we added 2 sections, -1 because we have a count and want an index
let last_host_section = dynhost_obj
.sections()
.nth(dynhost_obj.sections().count() - 2 - 1)
.unwrap();
let dynamic_relocations = DynamicRelocationsPe::new(dynhost_data);
let thunks_start_offset = find_thunks_start_offset(dynhost_data, &dynamic_relocations);
let optional_header = dynhost_obj.nt_headers().optional_header;
let optional_header_offset = dynhost_obj.dos_header().nt_headers_offset() as usize
+ std::mem::size_of::<u32>()
+ std::mem::size_of::<ImageFileHeader>();
let exports: MutMap<String, i64> = dynhost_obj
.exports()
.unwrap()
.into_iter()
.map(|e| {
(
String::from_utf8(e.name().to_vec()).unwrap(),
(e.address() - optional_header.image_base.get(LE)) as i64,
)
})
.collect();
let imports: Vec<_> = dynhost_obj
.imports()
.unwrap()
.iter()
.filter(|import| import.library() == APP_DLL.as_bytes())
.map(|import| {
std::str::from_utf8(import.name())
.unwrap_or_default()
.to_owned()
})
.collect();
let last_host_section_size = last_host_section.size();
let last_host_section_address = last_host_section.address();
// in the data directories, update the length of the imports (there is one fewer now)
{
let start = dynamic_relocations.data_directories_offset_in_file as usize
+ object::pe::IMAGE_DIRECTORY_ENTRY_IMPORT
* std::mem::size_of::<pe::ImageDataDirectory>();
let dir = load_struct_inplace_mut::<pe::ImageDataDirectory>(&mut dynhost, start);
let new = dir.size.get(LE) - std::mem::size_of::<pe::ImageImportDescriptor>() as u32;
dir.size.set(LE, new);
serialize_into(BufWriter::new(metadata_file), self)
.unwrap_or_else(|err| internal_error!("Failed to serialize metadata: {err}"));
}
// clear out the import table entry. we do implicitly assume that our dummy .dll is the last
{
const W: usize = std::mem::size_of::<ImageImportDescriptor>();
let start = dynamic_relocations.imports_offset_in_file as usize
+ W * dynamic_relocations.dummy_import_index as usize;
for b in dynhost[start..][..W].iter_mut() {
*b = 0;
}
}
let metadata = PeMetadata {
dynhost_file_size: std::fs::metadata(out_filename).unwrap().len() as usize,
image_base: optional_header.image_base.get(LE),
file_alignment: optional_header.file_alignment.get(LE),
section_alignment: optional_header.section_alignment.get(LE),
last_host_section_size,
last_host_section_address,
optional_header_offset,
imports,
exports,
dynamic_relocations,
thunks_start_offset,
};
let metadata_file =
std::fs::File::create(metadata_filename).unwrap_or_else(|e| internal_error!("{}", e));
serialize_into(BufWriter::new(metadata_file), &metadata)
.unwrap_or_else(|err| internal_error!("Failed to serialize metadata: {err}"));
Ok(())
}
pub(crate) fn surgery_pe(
out_filename: &str,
metadata_filename: &str,
app_bytes: &[u8],
_verbose: bool,
) {
let md: PeMetadata = {
fn read_from_file(metadata_filename: &Path) -> Self {
let input =
std::fs::File::open(metadata_filename).unwrap_or_else(|e| internal_error!("{}", e));
@ -189,9 +81,119 @@ pub(crate) fn surgery_pe(
internal_error!("Failed to deserialize metadata: {}", err);
}
}
};
}
let app_obj_sections = AppSections::from_data(app_bytes);
fn from_preprocessed_host(preprocessed_data: &[u8], new_sections: &[[u8; 8]]) -> Self {
use object::ObjectSection;
let dynhost_obj = object::read::pe::PeFile64::parse(preprocessed_data)
.unwrap_or_else(|err| internal_error!("Failed to parse executable file: {}", err));
let host_section_count = dynhost_obj.sections().count() - new_sections.len();
let last_host_section = dynhost_obj
.sections()
.nth(host_section_count - 1) // -1 because we have a count and want an index
.unwrap();
let dynamic_relocations = DynamicRelocationsPe::new(preprocessed_data);
let thunks_start_offset = find_thunks_start_offset(preprocessed_data, &dynamic_relocations);
let optional_header = dynhost_obj.nt_headers().optional_header;
let optional_header_offset = dynhost_obj.dos_header().nt_headers_offset() as usize
+ std::mem::size_of::<u32>()
+ std::mem::size_of::<ImageFileHeader>();
let exports: MutMap<String, i64> = dynhost_obj
.exports()
.unwrap()
.into_iter()
.map(|e| {
(
String::from_utf8(e.name().to_vec()).unwrap(),
(e.address() - optional_header.image_base.get(LE)) as i64,
)
})
.collect();
let imports: Vec<_> = dynhost_obj
.imports()
.unwrap()
.iter()
.filter(|import| import.library() == APP_DLL.as_bytes())
.map(|import| {
std::str::from_utf8(import.name())
.unwrap_or_default()
.to_owned()
})
.collect();
let last_host_section_size = last_host_section.size();
let last_host_section_address = last_host_section.address();
PeMetadata {
dynhost_file_size: preprocessed_data.len(),
image_base: optional_header.image_base.get(LE),
file_alignment: optional_header.file_alignment.get(LE),
section_alignment: optional_header.section_alignment.get(LE),
last_host_section_size,
last_host_section_address,
host_section_count,
optional_header_offset,
imports,
exports,
dynamic_relocations,
thunks_start_offset,
}
}
}
pub(crate) fn preprocess_windows(
host_exe_filename: &Path,
metadata_filename: &Path,
preprocessed_filename: &Path,
_verbose: bool,
_time: bool,
) -> object::read::Result<()> {
let data = open_mmap(host_exe_filename);
let new_sections = [*b".text\0\0\0", *b".rdata\0\0"];
let mut preprocessed = Preprocessor::preprocess(preprocessed_filename, &data, &new_sections);
// get the metadata from the preprocessed executable before the destructive operations below
let md = PeMetadata::from_preprocessed_host(&preprocessed, &new_sections);
// in the data directories, update the length of the imports (there is one fewer now)
{
let start = md.dynamic_relocations.data_directories_offset_in_file as usize
+ object::pe::IMAGE_DIRECTORY_ENTRY_IMPORT
* std::mem::size_of::<pe::ImageDataDirectory>();
let dir = load_struct_inplace_mut::<pe::ImageDataDirectory>(&mut preprocessed, start);
let new = dir.size.get(LE) - std::mem::size_of::<pe::ImageImportDescriptor>() as u32;
dir.size.set(LE, new);
}
// clear out the import table entry. we do implicitly assume that our dummy .dll is the last
{
const W: usize = std::mem::size_of::<ImageImportDescriptor>();
let start = md.dynamic_relocations.imports_offset_in_file as usize
+ W * md.dynamic_relocations.dummy_import_index as usize;
for b in preprocessed[start..][..W].iter_mut() {
*b = 0;
}
}
md.write_to_file(metadata_filename);
Ok(())
}
pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_bytes: &[u8]) {
let md = PeMetadata::read_from_file(metadata_path);
let app_obj_sections = AppSections::from_data(roc_app_bytes);
let mut symbols = app_obj_sections.symbols;
let image_base: u64 = md.image_base;
@ -204,10 +206,7 @@ pub(crate) fn surgery_pe(
.map(|s| next_multiple_of(s.file_range.end - s.file_range.start, file_alignment))
.sum();
let executable = &mut mmap_mut(
Path::new(out_filename),
md.dynhost_file_size + app_sections_size,
);
let executable = &mut open_mmap_mut(executable_path, md.dynhost_file_size + app_sections_size);
let app_code_section_va = md.last_host_section_address
+ next_multiple_of(
@ -215,10 +214,13 @@ pub(crate) fn surgery_pe(
section_alignment as usize,
) as u64;
let mut section_header_start = 624;
let mut section_file_offset = md.dynhost_file_size;
let mut virtual_address = (app_code_section_va - image_base) as u32;
// find the location to write the section headers for our new sections
let mut section_header_start = md.dynamic_relocations.section_headers_offset_in_file as usize
+ md.host_section_count * std::mem::size_of::<ImageSectionHeader>();
let mut code_bytes_added = 0;
let mut data_bytes_added = 0;
let mut file_bytes_added = 0;
@ -275,7 +277,7 @@ pub(crate) fn surgery_pe(
let mut offset = section_file_offset;
let it = app_obj_sections.sections.iter().filter(|s| s.kind == kind);
for section in it {
let slice = &app_bytes[section.file_range.start..section.file_range.end];
let slice = &roc_app_bytes[section.file_range.start..section.file_range.end];
executable[offset..][..slice.len()].copy_from_slice(slice);
for (name, app_relocation) in section.relocations.iter() {
@ -368,6 +370,9 @@ struct DynamicRelocationsPe {
/// The dummy .dll is the `dummy_import_index`th import of the host .exe
dummy_import_index: u32,
/// Start of the first ImageSectionHeader of the file
section_headers_offset_in_file: u64,
}
impl DynamicRelocationsPe {
@ -437,6 +442,7 @@ impl DynamicRelocationsPe {
let (nt_headers, data_directories) = ImageNtHeaders64::parse(data, &mut offset)?;
let sections = nt_headers.sections(data, offset)?;
let section_headers_offset_in_file = offset;
let data_dir = match data_directories.get(pe::IMAGE_DIRECTORY_ENTRY_IMPORT) {
Some(data_dir) => data_dir,
@ -475,6 +481,7 @@ impl DynamicRelocationsPe {
imports_offset_in_file,
data_directories_offset_in_file,
dummy_import_index,
section_headers_offset_in_file,
};
this.append_roc_imports(&import_table, &descriptor)?;
@ -506,7 +513,7 @@ impl Preprocessor {
fn preprocess(output_path: &Path, data: &[u8], extra_sections: &[[u8; 8]]) -> MmapMut {
let this = Self::new(data, extra_sections);
let mut result = mmap_mut(output_path, data.len() + this.additional_length);
let mut result = open_mmap_mut(output_path, data.len() + this.additional_length);
this.copy(&mut result, data);
this.fix(&mut result, extra_sections);
@ -579,8 +586,6 @@ impl Preprocessor {
}
fn write_dummy_sections(&self, result: &mut MmapMut, extra_sections: &[[u8; 8]]) {
// start of the first new section
for (i, name) in extra_sections.iter().enumerate() {
let header = ImageSectionHeader {
name: *name,
@ -657,20 +662,6 @@ impl Preprocessor {
}
}
fn mmap_mut(path: &Path, length: usize) -> MmapMut {
let out_file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.open(path)
.unwrap_or_else(|e| internal_error!("{e}"));
out_file
.set_len(length as u64)
.unwrap_or_else(|e| internal_error!("{e}"));
unsafe { MmapMut::map_mut(&out_file).unwrap_or_else(|e| internal_error!("{e}")) }
}
/// Find the place in the executable where the thunks for our dummy .dll are stored
fn find_thunks_start_offset(
executable: &[u8],
@ -1446,7 +1437,7 @@ mod test {
let dynhost_bytes = std::fs::read(dir.join("dynhost.exe")).unwrap();
let mut executable = mmap_mut(
let mut executable = open_mmap_mut(
&dir.join("app.exe"),
dynhost_bytes.len() + roc_app_sections_size,
);

2
crates/repl_wasm/Cargo.toml
View File

@ -18,7 +18,7 @@ console_error_panic_hook = {version = "0.1.7", optional = true}
futures = {version = "0.3.24", optional = true}
js-sys = "0.3.60"
wasm-bindgen = "0.2.79"
wasm-bindgen-futures = "0.4.32"
wasm-bindgen-futures = "0.4.33"
roc_collections = {path = "../compiler/collections"}
roc_gen_wasm = {path = "../compiler/gen_wasm"}

2
crates/reporting/Cargo.toml
View File

@ -34,4 +34,4 @@ roc_test_utils = { path = "../test_utils" }
roc_solve = { path = "../compiler/solve" }
pretty_assertions = "1.3.0"
indoc = "1.0.7"
insta = "1.19.0"
insta = "1.20.0"