diff --git a/Cargo.lock b/Cargo.lock index 1091e31d348..00f1f29194e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -217,12 +217,24 @@ dependencies = [ "byteorder", ] +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + [[package]] name = "base64" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bimap" version = "0.4.0" @@ -2079,6 +2091,7 @@ version = "0.1.0" dependencies = [ "analytics", "ast", + "base64 0.13.0", "bimap", "engine-protocol", "enso-config", @@ -2098,6 +2111,7 @@ dependencies = [ "ordered-float 1.1.1", "serde", "serde_json", + "sourcemap", "span-tree", "uuid", "wasm-bindgen", @@ -2126,6 +2140,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "if_chain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" + [[package]] name = "ifmt" version = "0.3.3" @@ -3814,6 +3834,22 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" +[[package]] +name = "sourcemap" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e031f2463ecbdd5f34c950f89f5c1e1032f22c0f8e3dc4bdb2e8b6658cf61eb" +dependencies = [ + "base64 0.11.0", + "if_chain", + "lazy_static", + "regex", + "rustc_version 0.2.3", + "serde", + "serde_json", + "url 2.2.2", +] + [[package]] name = "span-tree" version = "0.1.0" diff --git a/app/gui/CHANGELOG.md b/app/gui/CHANGELOG.md index 3ec0b947441..a2527db11fc 100644 --- a/app/gui/CHANGELOG.md +++ b/app/gui/CHANGELOG.md @@ -2,6 +2,7 @@ #### Visual Environment +- [Added support of source maps for JS-based visualizations.][3208] - [Fixed histograms coloring and added a color legend.][3153] - [Fixed broken node whose expression contains non-ASCII characters.][3166] - [Fixed developer console warnings about views being created but not @@ -17,6 +18,7 @@ [3181]: https://github.com/enso-org/enso/pull/3181 [3186]: https://github.com/enso-org/enso/pull/3186 [3193]: https://github.com/enso-org/enso/pull/3193 +[3208]: https://github.com/enso-org/enso/pull/3208 # Enso 2.0.0-alpha.18 (2021-10-12) diff --git a/app/gui/src/controller/visualization.rs b/app/gui/src/controller/visualization.rs index ab575573b24..69fb66d9da8 100644 --- a/app/gui/src/controller/visualization.rs +++ b/app/gui/src/controller/visualization.rs @@ -10,6 +10,7 @@ use crate::constants::VISUALIZATION_DIRECTORY; use engine_protocol::language_server; use ide_view::graph_editor::component::visualization; use ide_view::graph_editor::component::visualization::definition; +use ide_view::graph_editor::component::visualization::java_script::Sources; use std::rc::Rc; @@ -89,6 +90,7 @@ pub struct EmbeddedVisualizations { pub struct Handle { language_server_rpc: Rc, embedded_visualizations: Rc>, + logger: Logger, } impl Handle { @@ -96,9 +98,11 @@ impl Handle { pub fn new( language_server_rpc: Rc, embedded_visualizations: EmbeddedVisualizations, + logger: &Logger, ) -> Self { + let logger = logger.sub("VisualizationController"); let embedded_visualizations = Rc::new(RefCell::new(embedded_visualizations)); - Self { language_server_rpc, embedded_visualizations } + Self { language_server_rpc, embedded_visualizations, logger } } async fn list_project_specific_visualizations(&self) -> FallibleResult> { @@ -155,7 +159,17 @@ impl Handle { let js_code = self.language_server_rpc.read_file(path).await?.contents; let wrap_error = |err| Error::js_preparation_error(visualization.clone(), err).into(); - visualization::java_script::Definition::new(project, &js_code) + let sources = if let Some(file_name) = path.file_name() { + let sources: &[(&str, &str)] = &[(file_name, &js_code)]; + Sources::from_files(sources) + } else { + warning!( + self.logger, + "Unable to get a file name from {path}. Visualization source map will not be provided." + ); + Sources::empty() + }; + visualization::java_script::Definition::new(project, sources) .map(Into::into) .map_err(wrap_error) } @@ -233,7 +247,8 @@ mod tests { let embedded_visualization = builtin::visualization::native::BubbleChart::definition(); embedded_visualizations .insert("[Demo] Bubble Visualization".to_string(), embedded_visualization.clone()); - let vis_controller = Handle::new(language_server, embedded_visualizations); + let logger = Logger::new("Mock logger"); + let vis_controller = Handle::new(language_server, embedded_visualizations, &logger); let visualizations = vis_controller.list_visualizations().await; let visualizations = visualizations.expect("Couldn't list visualizations."); @@ -247,8 +262,10 @@ mod tests { assert_eq!(visualizations.len(), 3); let owner = visualization::Project::CurrentProject; - let javascript_vis0 = js_vis::Definition::new(owner.clone_ref(), &file_content0); - let javascript_vis1 = js_vis::Definition::new(owner, &file_content1); + let sources_vis0 = Sources::from_files(&[("file0.js", &file_content0)]); + let javascript_vis0 = js_vis::Definition::new(owner.clone_ref(), sources_vis0); + let sources_vis1 = Sources::from_files(&[("file0.js", &file_content1)]); + let javascript_vis1 = js_vis::Definition::new(owner, sources_vis1); let javascript_vis0 = javascript_vis0.expect("Couldn't create visualization class."); let javascript_vis1 = javascript_vis1.expect("Couldn't create visualization class."); let javascript_vis0: visualization::Definition = javascript_vis0.into(); diff --git a/app/gui/src/model/project/synchronized.rs b/app/gui/src/model/project/synchronized.rs index 37ccac8a23f..8ede19e708c 100644 --- a/app/gui/src/model/project/synchronized.rs +++ b/app/gui/src/model/project/synchronized.rs @@ -265,7 +265,7 @@ impl Project { let module_registry = default(); let execution_contexts = default(); let visualization = - controller::Visualization::new(language_server, embedded_visualizations); + controller::Visualization::new(language_server, embedded_visualizations, &logger); let parser = Parser::new_or_panic(); let language_server = &*language_server_rpc; let suggestion_db = SuggestionDatabase::create_synchronized(language_server); diff --git a/app/gui/view/debug_scene/visualization/src/lib.rs b/app/gui/view/debug_scene/visualization/src/lib.rs index 99e5758eb99..c9a959ddd9d 100644 --- a/app/gui/view/debug_scene/visualization/src/lib.rs +++ b/app/gui/view/debug_scene/visualization/src/lib.rs @@ -81,7 +81,9 @@ fn constructor_graph() -> visualization::java_script::Definition { return Graph "#; - visualization::java_script::Definition::new_builtin(source).unwrap() + let mut sources = visualization::java_script::Sources::empty(); + sources.add_file("demo.js", source); + visualization::java_script::Definition::new_builtin(sources).unwrap() } #[wasm_bindgen] diff --git a/app/gui/view/graph-editor/Cargo.toml b/app/gui/view/graph-editor/Cargo.toml index 14ec5dff1d8..d9dbcaa2776 100644 --- a/app/gui/view/graph-editor/Cargo.toml +++ b/app/gui/view/graph-editor/Cargo.toml @@ -10,6 +10,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] analytics = { version = "0.1.0", path = "../../analytics" } ast = { version = "0.1.0", path = "../../language/ast/impl" } +base64 = "0.13" bimap = { version = "0.4.0" } enso-config = { version = "0.1.0", path = "../../config" } enso-frp = { version = "0.1.0", path = "../../../../lib/rust/frp" } @@ -30,6 +31,7 @@ js-sys = { version = "0.3.28" } nalgebra = { version = "0.26.1", features = ["serde-serialize"] } serde_json = { version = "1.0" } serde = { version = "1.0", features = ["derive"] } +sourcemap = "6.0" uuid = { version = "0.8", features = ["serde", "v4", "wasm-bindgen"] } wasm-bindgen = { version = "=0.2.58", features = ["nightly", "serde-serialize"] } diff --git a/app/gui/view/graph-editor/src/builtin/visualization/java_script.rs b/app/gui/view/graph-editor/src/builtin/visualization/java_script.rs index 28e8b647bec..2452a552400 100644 --- a/app/gui/view/graph-editor/src/builtin/visualization/java_script.rs +++ b/app/gui/view/graph-editor/src/builtin/visualization/java_script.rs @@ -3,6 +3,7 @@ // These implementations are neither efficient nor pretty, but get the idea across. use crate::component::visualization; +use crate::component::visualization::java_script::source::from_files; @@ -12,60 +13,66 @@ use crate::component::visualization; /// Return a `JavaScript` Table visualization. pub fn table_visualization() -> visualization::java_script::FallibleDefinition { - let loading_scripts = include_str!("java_script/helpers/loading.js"); - let scrollable = include_str!("java_script/helpers/scrollable.js"); - let source = include_str!("java_script/table.js"); - let source = format!("{}{}{}", loading_scripts, scrollable, source); + let source = from_files!( + "java_script/helpers/loading.js", + "java_script/helpers/scrollable.js", + "java_script/table.js" + ); visualization::java_script::Definition::new_builtin(source) } /// Return a `JavaScript` SQL visualization. pub fn sql_visualization() -> visualization::java_script::FallibleDefinition { - let loading_scripts = include_str!("java_script/helpers/loading.js"); - let scrollable = include_str!("java_script/helpers/scrollable.js"); - let source = include_str!("java_script/sql.js"); - let source = format!("{}{}{}", loading_scripts, scrollable, source); + let source = from_files!( + "java_script/helpers/loading.js", + "java_script/helpers/scrollable.js", + "java_script/sql.js" + ); visualization::java_script::Definition::new_builtin(source) } /// Return a `JavaScript` Scatter plot visualization. pub fn scatter_plot_visualization() -> visualization::java_script::FallibleDefinition { - let loading_scripts = include_str!("java_script/helpers/loading.js"); - let number = include_str!("java_script/helpers/number.js"); - let source = include_str!("java_script/scatterPlot.js"); - let source = format!("{}{}{}", loading_scripts, number, source); + let source = from_files!( + "java_script/helpers/loading.js", + "java_script/helpers/number.js", + "java_script/scatterPlot.js" + ); visualization::java_script::Definition::new_builtin(source) } /// Return a `JavaScript` Histogram visualization. pub fn histogram_visualization() -> visualization::java_script::FallibleDefinition { - let loading_scripts = include_str!("java_script/helpers/loading.js"); - let number = include_str!("java_script/helpers/number.js"); - let source = include_str!("java_script/histogram.js"); - let source = format!("{}{}{}", loading_scripts, number, source); + let source = from_files!( + "java_script/helpers/loading.js", + "java_script/helpers/number.js", + "java_script/histogram.js" + ); visualization::java_script::Definition::new_builtin(source) } /// Return a `JavaScript` Heatmap visualization. pub fn heatmap_visualization() -> visualization::java_script::FallibleDefinition { - let loading_scripts = include_str!("java_script/helpers/loading.js"); - let number = include_str!("java_script/helpers/number.js"); - let source = include_str!("java_script/heatmap.js"); - let source = format!("{}{}{}", loading_scripts, number, source); + let source = from_files!( + "java_script/helpers/loading.js", + "java_script/helpers/number.js", + "java_script/heatmap.js" + ); visualization::java_script::Definition::new_builtin(source) } /// Return a `JavaScript` Map visualization. pub fn geo_map_visualization() -> visualization::java_script::FallibleDefinition { - let loading_scripts = include_str!("java_script/helpers/loading.js"); - let number = include_str!("java_script/helpers/number.js"); - let source = include_str!("java_script/geoMap.js"); - let source = format!("{}{}{}", loading_scripts, number, source); + let source = from_files!( + "java_script/helpers/loading.js", + "java_script/helpers/number.js", + "java_script/geoMap.js" + ); visualization::java_script::Definition::new_builtin(source) } @@ -73,16 +80,14 @@ pub fn geo_map_visualization() -> visualization::java_script::FallibleDefinition /// Return a `JavaScript` Bubble visualization. This should not be used as it is a demo /// visualization. pub fn bubble_visualization() -> visualization::java_script::FallibleDefinition { - let source = include_str!("java_script/bubbleVisualization.js"); + let source = from_files!("java_script/bubbleVisualization.js"); visualization::java_script::Definition::new_builtin(source) } /// Return a `JavaScript` Image visualization. pub fn image_base64_visualization() -> visualization::java_script::FallibleDefinition { - let loading_scripts = include_str!("java_script/helpers/loading.js"); - let source = include_str!("java_script/imageBase64.js"); - let source = format!("{}{}", loading_scripts, source); + let source = from_files!("java_script/helpers/loading.js", "java_script/imageBase64.js"); visualization::java_script::Definition::new_builtin(source) } @@ -93,6 +98,8 @@ pub fn empty_visualization() -> visualization::java_script::FallibleDefinition { class EmptyVisualization extends Visualization {} return EmptyVisualization; "#; + let files = [("java_script/empty.js", source)]; + let source = visualization::java_script::Sources::from_files(&files); visualization::java_script::Definition::new_builtin(source) } diff --git a/app/gui/view/graph-editor/src/component/visualization/foreign/java_script.rs b/app/gui/view/graph-editor/src/component/visualization/foreign/java_script.rs index 1d786f24d2f..9dd28677af2 100644 --- a/app/gui/view/graph-editor/src/component/visualization/foreign/java_script.rs +++ b/app/gui/view/graph-editor/src/component/visualization/foreign/java_script.rs @@ -3,6 +3,8 @@ pub mod binding; pub mod definition; pub mod instance; +pub mod source; pub use definition::*; pub use instance::*; +pub use source::Sources; diff --git a/app/gui/view/graph-editor/src/component/visualization/foreign/java_script/definition.rs b/app/gui/view/graph-editor/src/component/visualization/foreign/java_script/definition.rs index 828dc6a059d..865cb3435e6 100644 --- a/app/gui/view/graph-editor/src/component/visualization/foreign/java_script/definition.rs +++ b/app/gui/view/graph-editor/src/component/visualization/foreign/java_script/definition.rs @@ -15,6 +15,7 @@ use super::instance::Instance; use crate::component::visualization; use crate::component::visualization::InstantiationError; use crate::component::visualization::InstantiationResult; +use crate::visualization::foreign::java_script::Sources; use ensogl::display::Scene; use ensogl::system::web::JsValue; @@ -59,14 +60,10 @@ pub struct Definition { impl Definition { /// Create a visualization source from piece of JS source code. Signature needs to be inferred. - pub fn new( - project: visualization::path::Project, - source: impl AsRef, - ) -> Result { - let source = source.as_ref(); - let source = source; + pub fn new(project: visualization::path::Project, sources: Sources) -> Result { + let source = sources.to_string(&project); let context = JsValue::NULL; - let function = Function::new_with_args(binding::JS_CLASS_NAME, source) + let function = Function::new_with_args(binding::JS_CLASS_NAME, &source) .map_err(Error::InvalidFunction)?; let js_class = binding::js_class(); let class = function.call1(&context, &js_class).map_err(Error::InvalidFunction)?; @@ -84,8 +81,8 @@ impl Definition { } /// Create a definition of visualization that is built into the IDE. - pub fn new_builtin(source: impl AsRef) -> Result { - Self::new(visualization::path::Project::Builtin, source) + pub fn new_builtin(sources: Sources) -> Result { + Self::new(visualization::path::Project::Builtin, sources) } fn new_instance(&self, scene: &Scene) -> InstantiationResult { diff --git a/app/gui/view/graph-editor/src/component/visualization/foreign/java_script/source.rs b/app/gui/view/graph-editor/src/component/visualization/foreign/java_script/source.rs new file mode 100644 index 00000000000..8fc85add46e --- /dev/null +++ b/app/gui/view/graph-editor/src/component/visualization/foreign/java_script/source.rs @@ -0,0 +1,165 @@ +//! Contains [`Sources`] definition, a set of JS source code files with attached source map. +use crate::prelude::*; + +use crate::visualization::Project; + +use sourcemap::SourceMapBuilder; +use std::path::PathBuf; + + + +// ============================ +// === Visualizations paths === +// ============================ + +/// Path prefixes for source files depending on `Project` enum. +/// These prefixes form a path to the source file displayed in the DevTools. We provide a specific +/// set of prefixes to make sources navigation in DevTools easier. As we have access to relative +/// file paths only, it would be hard to understand what relative path corresponds to otherwise. +mod path_prefix { + use super::*; + + const BUILTIN: &str = "visualization/builtin"; + const CURRENT_PROJECT: &str = "visualization/project"; + const LIBRARY: &str = "visualization/library"; + + /// Turn `Project` enum into a path prefix. + pub fn from_project(project: &Project) -> PathBuf { + match *project { + Project::Builtin => PathBuf::from(BUILTIN), + Project::CurrentProject => PathBuf::from(CURRENT_PROJECT), + Project::Library(ref name) => { + let name: &str = name.as_ref(); + PathBuf::from(LIBRARY).join(name) + } + } + } +} + + + +// =============== +// === Sources === +// =============== + +/// Internal identifier of the source file. +type SourceId = u32; + +/// A set of JS source code files with attached +/// [source map](https://blog.teamtreehouse.com/introduction-source-maps). +/// +/// It is used to concatenate several source files into a single one and preserve +/// the information about original file names and line numbers. +pub struct Sources { + code: String, + lines_count: u32, + source_map_builder: SourceMapBuilder, + files: Vec<(PathBuf, SourceId)>, +} + +impl Debug for Sources { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Sources") + .field("code", &self.code) + .field("lines_count", &self.lines_count) + .field("files", &self.files) + .finish() + } +} + +impl Default for Sources { + fn default() -> Self { + Self::empty() + } +} + +impl Sources { + /// Constructor. Typically [`Sources::from_files`] should be used instead. + pub fn empty() -> Self { + Self { + code: default(), + lines_count: 0, + source_map_builder: SourceMapBuilder::new(None), + files: default(), + } + } + + /// Constructor from a list of `(file_name, file_content)` pairs. + /// + /// `file_name` should be a file name or a relative path to the file. + pub fn from_files(files: &[(&str, &str)]) -> Self { + let mut sources = Self::empty(); + for (path, content) in files { + sources.add_file(path, content); + } + sources + } + + /// Add a new file to the sources set, concatenating it with other files and updating source + /// map information. + pub fn add_file(&mut self, path: &str, content: &str) { + let source_id = self.source_map_builder.add_source(path); + self.source_map_builder.set_source_contents(source_id, Some(content)); + let added_lines_count = content.lines().count() as u32; + for line in 0..added_lines_count { + // This offset is needed because of how DevTools handle source maps for + // `eval`uated code. Our visualizations source code is turned into a JS object by + // constructing a `Function` object + // (see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function). + // DevTools basically wrap provided source code with `function anonimous() { + // }`, so the actual line numbers in resulted code are off by 2. This was + // calculated empirically, and apparently, this behavior of DevTools is undocumented. + const OFFSET: u32 = 2; + const LINE_START: u32 = 0; + + let dst_line = self.lines_count + line + OFFSET; + let source = Some(source_id); + self.source_map_builder.add_raw(dst_line, LINE_START, line, LINE_START, source, None); + } + self.lines_count += added_lines_count; + self.files.push((PathBuf::from(path), source_id)); + self.code += content; + } + + /// Get the final source string consisting of concatenated files and an inlined source map. + pub fn to_string(mut self, project: &Project) -> String { + self.set_file_names(project); + let encoded_source_map = Self::source_map_to_base64(self.source_map_builder); + const SOURCE_MAPPING_URL: &str = + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"; + [&self.code, SOURCE_MAPPING_URL, &encoded_source_map].concat() + } + + /// Set file names to follow `/` pattern. See `path_prefix` + /// module documentation. + fn set_file_names(&mut self, project: &Project) { + let path_prefix = path_prefix::from_project(project); + for (path, source_id) in &self.files { + let source = path_prefix.join(path); + self.source_map_builder.set_source(*source_id, &source.to_string_lossy()); + } + } + + /// Helper function to convert source map into a base64-encoded JSON. + fn source_map_to_base64(builder: SourceMapBuilder) -> String { + let mut buf = Vec::new(); + let source_map = builder.into_sourcemap(); + source_map.to_writer(&mut buf).expect("Source map serialization failed."); + base64::encode(&buf) + } +} + + + +// ==================== +// === Helper macro === +// ==================== + +/// Transform list of file paths `file1, file2, ...` into +/// `Sources::from_files(&[(file1, include_str!(file1)), (file2, include_str!(file2)), ...])`. +macro_rules! from_files { + ($($file:literal),*) => { + $crate::visualization::java_script::Sources::from_files(&[$(($file, include_str!($file))),*]) + } +} +pub(crate) use from_files;