feat(dbg-swc): Make reducer parallel (#4676)

Cargo.lock

@@ -802,6 +802,7 @@ dependencies = [
  "swc_ecma_visit",
  "swc_error_reporters",
  "swc_timer",
+ "tempdir",
  "tracing",
  "tracing-subscriber",
  "url",
@@ -981,6 +982,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
 
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+
 [[package]]
 name = "futures-channel"
 version = "0.3.19"
@@ -2016,7 +2023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
 dependencies = [
  "phf_shared",
- "rand",
+ "rand 0.8.4",
 ]
 
 [[package]]
@@ -2241,6 +2248,19 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce082a9940a7ace2ad4a8b7d0b1eac6aa378895f18be598230c5f2284ac05426"
 
+[[package]]
+name = "rand"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
+dependencies = [
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.1",
+ "rdrand",
+ "winapi",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.4"
@@ -2249,7 +2269,7 @@ checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
 dependencies = [
  "libc",
  "rand_chacha",
- "rand_core",
+ "rand_core 0.6.3",
  "rand_hc",
 ]
 
@@ -2260,9 +2280,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.3",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+dependencies = [
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+
 [[package]]
 name = "rand_core"
 version = "0.6.3"
@@ -2278,7 +2313,7 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
 dependencies = [
- "rand_core",
+ "rand_core 0.6.3",
 ]
 
 [[package]]
@@ -2305,6 +2340,15 @@ dependencies = [
  "num_cpus",
 ]
 
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.10"
@@ -4139,6 +4183,16 @@ version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9bffcddbc2458fa3e6058414599e3c838a022abae82e5c67b4f7f80298d5bff"
 
+[[package]]
+name = "tempdir"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
+dependencies = [
+ "rand 0.4.6",
+ "remove_dir_all",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.2.0"
@@ -4147,7 +4201,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "rand",
+ "rand 0.8.4",
  "redox_syscall",
  "remove_dir_all",
  "winapi",

crates/dbg-swc/Cargo.toml

@@ -28,6 +28,7 @@ swc_ecma_transforms_base = { version = "0.85.0", path = "../swc_ecma_transforms_
 swc_ecma_visit = { version = "0.64.0", path = "../swc_ecma_visit" }
 swc_error_reporters = { version = "0.2.0", path = "../swc_error_reporters" }
 swc_timer = { version = "0.6.0", path = "../swc_timer" }
+tempdir = "0.3.7"
 tracing = "0.1.34"
 tracing-subscriber = { version = "0.3.11", features = ["fmt", "env-filter"] }
 url = "2"

crates/dbg-swc/scripts/reduce-again.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 set -eu
 
-find .input -type f | xargs -L 1 -I {} cargo run --release -- minify reduce --mode size '{}' && rm '{}'
+cargo run --release -- minify reduce --mode size --remove .input
 # mv data/*.js ./.input/

crates/dbg-swc/src/main.rs

@@ -1,6 +1,6 @@
 #![feature(box_syntax)]
 
-use std::{env, str::FromStr, sync::Arc};
+use std::{env, path::PathBuf, str::FromStr, sync::Arc};
 
 use anyhow::{bail, Result};
 use clap::{StructOpt, Subcommand};
@@ -24,6 +24,10 @@ mod minify;
 mod test;
 mod util;
 
+const CREDUCE_INPUT_ENV_VAR: &str = "CREDUCE_INPUT";
+
+const CREDUCE_MODE_ENV_VAR: &str = "CREDUCE_COMPARE";
+
 #[derive(Debug, clap::Parser)]
 struct AppArgs {
     #[clap(subcommand)]
@@ -63,7 +67,7 @@ fn main() -> Result<()> {
 
     let cm = Arc::new(SourceMap::default());
 
-    if let Ok(mode) = env::var("CREDUCE_COMPARE") {
+    if let Ok(mode) = env::var(CREDUCE_MODE_ENV_VAR) {
         return try_with_handler(
             cm.clone(),
             HandlerOpts {
@@ -74,14 +78,18 @@ fn main() -> Result<()> {
                 GLOBALS.set(&Globals::default(), || {
                     HANDLER.set(handler, || {
                         //
+                        let input = PathBuf::from(
+                            env::var(CREDUCE_INPUT_ENV_VAR)
+                                .expect("creduce is invoked without the name of input file"),
+                        );
 
                         if mode == "SIZE" {
-                            let m = get_minified(cm.clone(), "input.js".as_ref(), true, true)?;
+                            let m = get_minified(cm.clone(), &input, true, true)?;
 
                             let swc_output = print_js(cm.clone(), &m.module, true)?;
 
-                            let terser_output = get_terser_output("input.js".as_ref(), true, true)?;
+                            let terser_output = get_terser_output(&input, true, true)?;
-                            if swc_output.len() > terser_output.len() {
+                            if swc_output.trim().len() > terser_output.trim().len() {
                                 return Ok(());
                             }
 
@@ -90,7 +98,7 @@ fn main() -> Result<()> {
                             // We target es5, but esbuild does not support it
                             let swc_output = swc_output.replace("\\n", "_");
 
-                            let esbuild_output = get_esbuild_output("input.js".as_ref(), true)?;
+                            let esbuild_output = get_esbuild_output(&input, true)?;
 
                             if swc_output.len() > esbuild_output.len() {
                                 return Ok(());
@@ -104,12 +112,11 @@ fn main() -> Result<()> {
 
                             bail!("We don't care about this file")
                         } else if mode == "SEMANTICS" {
-                            let m = get_minified(cm.clone(), "input.js".as_ref(), true, false)?;
+                            let m = get_minified(cm.clone(), &input, true, false)?;
 
                             let swc_output = print_js(cm.clone(), &m.module, true)?;
 
-                            let terser_output =
+                            let terser_output = get_terser_output(&input, true, false)?;
-                                get_terser_output("input.js".as_ref(), true, false)?;
 
                             if swc_output.len() <= 1000 || terser_output.len() <= 1000 {
                                 bail!("We don't care about this file because it's too small")

crates/dbg-swc/src/minify/reduce.rs

@@ -8,8 +8,12 @@ use std::{
 
 use anyhow::{Context, Result};
 use clap::{ArgEnum, Args};
+use rayon::prelude::*;
 use sha1::{Digest, Sha1};
 use swc_common::SourceMap;
+use tempdir::TempDir;
+
+use crate::{util::all_js_files, CREDUCE_INPUT_ENV_VAR, CREDUCE_MODE_ENV_VAR};
 
 #[derive(Debug, Args)]
 pub struct ReduceCommand {
@@ -17,6 +21,10 @@ pub struct ReduceCommand {
 
     #[clap(long, arg_enum)]
     pub mode: ReduceMode,
+
+    /// If true, the input file will be removed after the reduction.
+    #[clap(long)]
+    pub remove: bool,
 }
 
 #[derive(Debug, Clone, Copy, ArgEnum)]
@@ -27,24 +35,45 @@ pub enum ReduceMode {
 
 impl ReduceCommand {
     pub fn run(self, _cm: Arc<SourceMap>) -> Result<()> {
-        fs::copy(&self.path, "input.js").context("failed to copy")?;
+        let js_files = all_js_files(&self.path)?;
+
+        js_files
+            .into_par_iter()
+            .map(|path| self.reduce_file(&path))
+            .collect::<Result<Vec<_>>>()?;
+
+        Ok(())
+    }
+
+    fn reduce_file(&self, src_path: &Path) -> Result<()> {
+        let dir = TempDir::new("dbg-swc").context("failed to create a temp directory")?;
+
+        let input = dir.path().join("input.js");
+
+        fs::copy(&src_path, &input).context("failed to copy")?;
 
         let mut c = Command::new("creduce");
 
         c.env(
-            "CREDUCE_COMPARE",
+            CREDUCE_MODE_ENV_VAR,
             match self.mode {
                 ReduceMode::Size => "SIZE",
                 ReduceMode::Semantics => "SEMANTICS",
             },
         );
+        c.env(CREDUCE_INPUT_ENV_VAR, &input);
+
         let exe = current_exe()?;
         c.arg(&exe);
-        c.arg("input.js");
+        c.arg(&input);
         let status = c.status().context("failed to run creduce")?;
 
         if status.success() {
-            move_to_data_dir("input.js".as_ref())?;
+            move_to_data_dir(&input)?;
+        }
+
+        if self.remove {
+            fs::remove_file(&src_path).context("failed to remove")?;
         }
 
         Ok(())
@@ -68,7 +97,7 @@ fn move_to_data_dir(input_path: &Path) -> Result<PathBuf> {
     create_dir_all(format!("data/{}", hash_str)).context("failed to create `.data`")?;
 
     let to = PathBuf::from(format!("data/{}/input.js", hash_str));
-    fs::copy(input_path, &to).context("failed to copy")?;
+    fs::write(&to, src.as_bytes()).context("failed to write")?;
 
     Ok(to)
 }

crates/swc_ecma_minifier/tests/full/.size/e4dd4373c192c6fe2fc929bc55d1ed625b974338/input.js

@@ -1,5 +1,5 @@
 []({
     c() {
-        var a = {}.b;
+        a = {}.b;
     },
 });