git.iko.soy/htmlang
on master help sign in
One data directive: @data $name SOURCE
cb6e458iko
crates/htmlang-core/src/parser.rs+153−391
compare with master
unifiedsplit
@@ -120,10 +120,6 @@ struct ParseContext {
120 manifest: Option<crate::ast::ManifestConfig>,
120 manifest: Option<crate::ast::ManifestConfig>,
121 /// Track @import paths for circular dependency detection
121 /// Track @import paths for circular dependency detection
122 import_stack: Vec<PathBuf>,
122 import_stack: Vec<PathBuf>,
123− /// i18n translations: locale -> (key -> value)
124− translations: HashMap<String, HashMap<String, String>>,
125− /// Current active locale for translations
126− active_locale: Option<String>,
127}
123}
128
124
129struct Parser {
125struct Parser {
@@ -222,8 +218,6 @@ pub fn parse_with_base(input: &str, base_path: Option<&Path>) -> ParseResult {
222 json_ld_blocks: Vec::new(),
218 json_ld_blocks: Vec::new(),
223 manifest: None,
219 manifest: None,
224 import_stack: Vec::new(),
220 import_stack: Vec::new(),
225− translations: HashMap::new(),
226− active_locale: None,
227 };
221 };
228 load_prelude(&mut ctx);
222 load_prelude(&mut ctx);
229 let nodes = parser.parse_children(0, &mut ctx);
223 let nodes = parser.parse_children(0, &mut ctx);
@@ -1159,113 +1153,6 @@ impl Parser {
1159 return Ok(Some(included_nodes));
1153 return Ok(Some(included_nodes));
1160 }
1154 }
1161
1155
1162− // --- @env (compile-time environment variables) ---
1163−
1164− if let Some(rest) = content.strip_prefix("@env ") {
1165− let rest = rest.trim();
1166− // @env VAR_NAME default_value OR @env VAR_NAME
1167− let (var_name, default_val) = if let Some((name, default)) = rest.split_once(' ') {
1168− (name.trim(), Some(default.trim()))
1169− } else {
1170− (rest, None)
1171− };
1172− let env_val = std::env::var(var_name)
1173− .ok()
1174− .or_else(|| default_val.map(|d| substitute_vars(d, &ctx.variables)));
1175− match env_val {
1176− Some(val) => {
1177− // Store as variable with lowercase name
1178− let key = var_name.to_lowercase().replace('-', "_");
1179− ctx.variables.insert(key, val);
1180− }
1181− None => {
1182− ctx.diagnostics.push(Diagnostic {
1183− line: line_num,
1184− column: None,
1185− message: format!(
1186− "environment variable '{}' is not set and no default provided",
1187− var_name
1188− ),
1189− severity: Severity::Warning,
1190− source_line: Some(content.clone()),
1191− });
1192− }
1193− }
1194− return Ok(None);
1195− }
1196−
1197− // --- @fetch (compile-time HTTP data fetching) ---
1198−
1199− if let Some(rest) = content.strip_prefix("@fetch ") {
1200− let rest = rest.trim();
1201− // @fetch $prefix https://url OR @fetch https://url
1202− let (prefix, url) = if rest.starts_with('$') {
1203− if let Some((p, u)) = rest.split_once(' ') {
1204− (
1205− p.strip_prefix('$').unwrap_or(p).to_string(),
1206− u.trim().to_string(),
1207− )
1208− } else {
1209− return Err(ParseError {
1210− line: line_num,
1211− message: "@fetch requires: @fetch $prefix url or @fetch url".to_string(),
1212− });
1213− }
1214− } else {
1215− (String::new(), rest.to_string())
1216− };
1217−
1218− let url = substitute_vars(&url, &ctx.variables);
1219−
1220− // Synchronous HTTP GET using std::net
1221− match fetch_url_blocking(&url) {
1222− Ok(body) => {
1223− // Try to parse as JSON
1224− match parse_json(&body) {
1225− Some(json) => {
1226− if prefix.is_empty() {
1227− if let JsonValue::Object(pairs) = &json {
1228− for (key, val) in pairs {
1229− let mut sub = HashMap::new();
1230− flatten_json(key, val, &mut sub);
1231− for (k, v) in sub {
1232− ctx.variables.insert(k, v);
1233− }
1234− }
1235− } else {
1236− // Store the raw body as a single variable
1237− ctx.variables.insert("__fetch_body".to_string(), body);
1238− }
1239− } else {
1240− let mut sub = HashMap::new();
1241− flatten_json(&prefix, &json, &mut sub);
1242− for (k, v) in sub {
1243− ctx.variables.insert(k, v);
1244− }
1245− }
1246− }
1247− None => {
1248− // Not JSON — store raw body
1249− if prefix.is_empty() {
1250− ctx.variables.insert("__fetch_body".to_string(), body);
1251− } else {
1252− ctx.variables.insert(prefix, body);
1253− }
1254− }
1255− }
1256− }
1257− Err(e) => {
1258− ctx.diagnostics.push(Diagnostic {
1259− line: line_num,
1260− column: None,
1261− message: format!("@fetch failed for '{}': {}", url, e),
1262− severity: Severity::Error,
1263− source_line: Some(content.clone()),
1264− });
1265− }
1266− }
1267− return Ok(None);
1268− }
1269
1156
1270 // --- @svg (inline SVG from file) ---
1157 // --- @svg (inline SVG from file) ---
1271
1158
@@ -1345,25 +1232,68 @@ impl Parser {
1345
1232
1346 // --- @data (load JSON file into variables) ---
1233 // --- @data (load JSON file into variables) ---
1347
1234
1235+ // @data file.json top-level keys become variables
1236+ // @data $name file.json values as $name.key
1237+ // @data $name dir/*.json each file as $name.STEM.key; $name
1238+ // lists the stems, $name._count counts them
1239+ // @data $name env:NAME [DEFAULT] an environment variable
1348 if let Some(rest) = content.strip_prefix("@data ") {
1240 if let Some(rest) = content.strip_prefix("@data ") {
1349 let rest = rest.trim();
1241 let rest = rest.trim();
1350− // @data $prefix file.json OR @data file.json (no prefix, top-level keys become vars)
1242+ let (prefix, filename) = match rest.strip_prefix('$') {
1351− let (prefix, filename) = if rest.starts_with('$') {
1243+ Some(named) => match named.split_once(char::is_whitespace) {
1352− if let Some((p, f)) = rest.split_once(' ') {
1244+ Some((name, source)) => (name.to_string(), source.trim().to_string()),
1353− (
1245+ None => {
1354− p.strip_prefix('$').unwrap_or(p).to_string(),
1246+ return Err(ParseError {
1355− f.trim().to_string(),
1247+ line: line_num,
1356− )
1248+ message: "@data requires: @data $name SOURCE or @data file.json"
1357− } else {
1249+ .to_string(),
1250+ });
1251+ }
1252+ },
1253+ None => (String::new(), rest.to_string()),
1254+ };
1255+
1256+ if let Some(env) = filename.strip_prefix("env:") {
1257+ let (var, default) = match env.split_once(char::is_whitespace) {
1258+ Some((var, default)) => (var, Some(default.trim())),
1259+ None => (env, None),
1260+ };
1261+ if prefix.is_empty() {
1358 return Err(ParseError {
1262 return Err(ParseError {
1359 line: line_num,
1263 line: line_num,
1360− message: "@data requires: @data $prefix file.json or @data file.json"
1264+ message: format!("@data env:{} needs a name: @data $name env:{}", var, var),
1361− .to_string(),
1362 });
1265 });
1363 }
1266 }
1364− } else {
1267+ let value = std::env::var(var)
1365− (String::new(), rest.to_string())
1268+ .ok()
1269+ .or_else(|| default.map(|d| substitute_vars(d, &ctx.variables)));
1270+ if value.is_none() {
1271+ ctx.diagnostics.push(Diagnostic {
1272+ line: line_num,
1273+ column: None,
1274+ message: format!(
1275+ "environment variable '{}' is not set and has no default",
1276+ var
1277+ ),
1278+ severity: Severity::Warning,
1279+ source_line: Some(content.clone()),
1280+ });
1281+ }
1282+ ctx.variables.insert(prefix, value.unwrap_or_default());
1283+ return Ok(None);
1366− };
1284+ }
1285+
1286+ let filename = substitute_vars(&filename, &ctx.variables);
1287+ if filename.contains('*') {
1288+ if prefix.is_empty() {
1289+ return Err(ParseError {
1290+ line: line_num,
1291+ message: format!("@data {} needs a name: @data $name {}", filename, filename),
1292+ });
1293+ }
1294+ self.load_data_glob(&prefix, &filename, line_num, &content, ctx);
1295+ return Ok(None);
1296+ }
1367
1297
1368 let filename = substitute_vars(&filename, &ctx.variables);
1298 let filename = substitute_vars(&filename, &ctx.variables);
1369 let resolved = match &ctx.base_path {
1299 let resolved = match &ctx.base_path {
@@ -1431,96 +1361,6 @@ impl Parser {
1431 }
1361 }
1432
1362
1433
1363
1434− // --- @collection (load multiple JSON files matching a glob pattern) ---
1435−
1436− if let Some(rest) = content.strip_prefix("@collection ") {
1437− let rest = rest.trim();
1438− // @collection $varname "pattern" OR @collection "pattern" as varname
1439− let (var_name, pattern) = if rest.starts_with('$') {
1440− let parts: Vec<&str> = rest.splitn(2, ' ').collect();
1441− if parts.len() == 2 {
1442− let name = parts[0].strip_prefix('$').unwrap_or(parts[0]);
1443− let pat = parts[1].trim().trim_matches('"');
1444− (name.to_string(), pat.to_string())
1445− } else {
1446− return Err(ParseError {
1447− line: line_num,
1448− message: "@collection requires: @collection $var \"pattern\"".to_string(),
1449− });
1450− }
1451− } else {
1452− let pat = rest.trim_matches('"');
1453− ("_items".to_string(), pat.to_string())
1454− };
1455−
1456− let pattern = substitute_vars(&pattern, &ctx.variables);
1457− let base = ctx.base_path.clone().unwrap_or_else(|| PathBuf::from("."));
1458− let glob_pattern = base.join(&pattern);
1459−
1460− // Simple glob: support * in filename part
1461− let parent = glob_pattern.parent().unwrap_or(Path::new("."));
1462− let file_pattern = glob_pattern
1463− .file_name()
1464− .map(|f| f.to_string_lossy().to_string())
1465− .unwrap_or_default();
1466−
1467− let mut items = Vec::new();
1468− if let Ok(entries) = std::fs::read_dir(parent) {
1469− let mut paths: Vec<PathBuf> = entries
1470− .flatten()
1471− .map(|e| e.path())
1472− .filter(|p| {
1473− if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
1474− if file_pattern.contains('*') {
1475− let parts: Vec<&str> = file_pattern.split('*').collect();
1476− if parts.len() == 2 {
1477− name.starts_with(parts[0]) && name.ends_with(parts[1])
1478− } else {
1479− true
1480− }
1481− } else {
1482− name == file_pattern
1483− }
1484− } else {
1485− false
1486− }
1487− })
1488− .collect();
1489− paths.sort();
1490− for path in &paths {
1491− if let Ok(text) = std::fs::read_to_string(path) {
1492− // Store the file content as a JSON string for each item
1493− let stem = path
1494− .file_stem()
1495− .and_then(|s| s.to_str())
1496− .unwrap_or("")
1497− .to_string();
1498− items.push((stem, text));
1499− }
1500− }
1501− }
1502−
1503− // Store item count and serialized data as variables
1504− ctx.variables
1505− .insert(format!("{}_count", var_name), items.len().to_string());
1506− // Store as space-separated list of stems for @each
1507− let stems: Vec<String> = items.iter().map(|(stem, _)| stem.clone()).collect();
1508− ctx.variables.insert(var_name.clone(), stems.join(" "));
1509− // Also parse each JSON file and store keys as prefix_stem_key
1510− for (stem, text) in &items {
1511− if let Some(json) = parse_json(text.trim())
1512− && let JsonValue::Object(ref obj) = json
1513− {
1514− for (key, val) in obj {
1515− let var_key = format!("{}_{}", stem, key);
1516− ctx.variables.insert(var_key, json_value_to_string(val));
1517− }
1518− }
1519− }
1520−
1521− return Ok(None);
1522− }
1523−
1524 // --- @if / @else ---
1364 // --- @if / @else ---
1525
1365
1526 if let Some(rest) = content.strip_prefix("@if ") {
1366 if let Some(rest) = content.strip_prefix("@if ") {
@@ -2016,58 +1856,6 @@ impl Parser {
2016 return Ok(None);
1856 return Ok(None);
2017 }
1857 }
2018
1858
2019− // --- @translations (i18n) ---
2020−
2021− if content.trim() == "@translations" || content.starts_with("@translations ") {
2022− let locale = content
2023− .strip_prefix("@translations")
2024− .unwrap_or("")
2025− .trim()
2026− .to_string();
2027− let active_locale = if locale.is_empty() {
2028− ctx.lang.clone().unwrap_or_else(|| "en".to_string())
2029− } else {
2030− locale
2031− };
2032−
2033− // Collect indented body: key value pairs, grouped by locale headers
2034− let mut current_locale = active_locale.clone();
2035− let mut first_locale_indent: Option<usize> = None;
2036− while self.pos < self.lines.len() && self.lines[self.pos].indent > current_indent {
2037− if let LineContent::Normal(ref s) = self.lines[self.pos].content {
2038− let trimmed = s.trim();
2039− // Check for locale sub-header: e.g., "en:" or "fr:" or "es:"
2040− if trimmed.ends_with(':') && trimmed.len() <= 10 && !trimmed.contains(' ') {
2041− current_locale = trimmed[..trimmed.len() - 1].to_string();
2042− first_locale_indent = Some(self.lines[self.pos].indent);
2043− self.pos += 1;
2044− continue;
2045− }
2046− // key value pair
2047− if let Some((key, value)) = trimmed.split_once(' ') {
2048− let key = key.trim().to_string();
2049− let value = substitute_vars(value.trim(), &ctx.variables);
2050− ctx.translations
2051− .entry(current_locale.clone())
2052− .or_default()
2053− .insert(key, value);
2054− }
2055− }
2056− self.pos += 1;
2057− }
2058− // Set the active locale
2059− if ctx.active_locale.is_none() {
2060− ctx.active_locale = Some(active_locale.clone());
2061− }
2062− let _ = first_locale_indent; // suppress unused
2063− // Inject active locale's translations as $t.key variables
2064− if let Some(strings) = ctx.translations.get(&active_locale) {
2065− for (key, value) in strings {
2066− ctx.variables.insert(format!("t.{}", key), value.clone());
2067− }
2068− }
2069− return Ok(None);
2070− }
2071
1859
2072 // --- @deprecated annotation ---
1860 // --- @deprecated annotation ---
2073
1861
@@ -2258,6 +2046,90 @@ impl Parser {
2258 Ok(Some(vec![Node::Text(segments)]))
2046 Ok(Some(vec![Node::Text(segments)]))
2259 }
2047 }
2260
2048
2049+ /// `@data $name dir/*.json`: load every matching file as
2050+ /// `$name.STEM.key`, list the stems in `$name` and count them.
2051+ fn load_data_glob(
2052+ &mut self,
2053+ name: &str,
2054+ pattern: &str,
2055+ line_num: usize,
2056+ content: &str,
2057+ ctx: &mut ParseContext,
2058+ ) {
2059+ let base = ctx.base_path.clone().unwrap_or_else(|| PathBuf::from("."));
2060+ let pattern_path = Path::new(pattern);
2061+ let dir = match pattern_path.parent() {
2062+ Some(dir) if !dir.as_os_str().is_empty() => base.join(dir),
2063+ _ => base,
2064+ };
2065+ let file_pattern = pattern_path
2066+ .file_name()
2067+ .unwrap_or_default()
2068+ .to_string_lossy()
2069+ .to_string();
2070+ let mut files: Vec<PathBuf> = match std::fs::read_dir(&dir) {
2071+ Ok(entries) => entries
2072+ .flatten()
2073+ .map(|e| e.path())
2074+ .filter(|p| {
2075+ p.file_name()
2076+ .is_some_and(|n| glob_match(&file_pattern, &n.to_string_lossy()))
2077+ })
2078+ .collect(),
2079+ Err(e) => {
2080+ ctx.diagnostics.push(Diagnostic {
2081+ line: line_num,
2082+ column: None,
2083+ message: format!("cannot read directory for '{}': {}", pattern, e),
2084+ severity: Severity::Error,
2085+ source_line: Some(content.to_string()),
2086+ });
2087+ return;
2088+ }
2089+ };
2090+ files.sort();
2091+ let mut stems = Vec::new();
2092+ for file in files {
2093+ let stem = file
2094+ .file_stem()
2095+ .unwrap_or_default()
2096+ .to_string_lossy()
2097+ .to_string();
2098+ let text = match std::fs::read_to_string(&file) {
2099+ Ok(text) => text,
2100+ Err(e) => {
2101+ ctx.diagnostics.push(Diagnostic {
2102+ line: line_num,
2103+ column: None,
2104+ message: format!("cannot load data '{}': {}", file.display(), e),
2105+ severity: Severity::Error,
2106+ source_line: Some(content.to_string()),
2107+ });
2108+ continue;
2109+ }
2110+ };
2111+ match parse_json_with_error(&text) {
2112+ Ok(json) => {
2113+ let mut vars = HashMap::new();
2114+ flatten_json(&format!("{}.{}", name, stem), &json, &mut vars);
2115+ ctx.variables.extend(vars);
2116+ stems.push(stem);
2117+ }
2118+ Err(detail) => ctx.diagnostics.push(Diagnostic {
2119+ line: line_num,
2120+ column: None,
2121+ message: format!("invalid JSON in '{}': {}", file.display(), detail),
2122+ severity: Severity::Error,
2123+ source_line: Some(content.to_string()),
2124+ }),
2125+ }
2126+ ctx.included_files.push(file);
2127+ }
2128+ ctx.variables
2129+ .insert(format!("{}._count", name), stems.len().to_string());
2130+ ctx.variables.insert(name.to_string(), stems.join(", "));
2131+ }
2132+
2261 fn expand_fn_call(
2133 fn expand_fn_call(
2262 &mut self,
2134 &mut self,
2263 name: &str,
2135 name: &str,
@@ -2663,9 +2535,7 @@ const KNOWN_DIRECTIVES: &[&str] = &[
2663 "json-ld",
2535 "json-ld",
2664 "assert",
2536 "assert",
2665 "markdown",
2537 "markdown",
2666− "collection",
2667 "manifest",
2538 "manifest",
2668− "translations",
2669];
2539];
2670
2540
2671fn parse_element_kind(s: &str, line_num: usize) -> Result<ElementKind, ParseError> {
2541fn parse_element_kind(s: &str, line_num: usize) -> Result<ElementKind, ParseError> {
@@ -2726,6 +2596,19 @@ const REMOVED_SYNTAX: &[(&str, &str)] = &[
2726 ("@base", "use `@page [base ...] Title`"),
2596 ("@base", "use `@page [base ...] Title`"),
2727 ("@og", "use `@meta og:NAME VALUE`"),
2597 ("@og", "use `@meta og:NAME VALUE`"),
2728 ("@debug", "use `@warn`"),
2598 ("@debug", "use `@warn`"),
2599+ (
2600+ "@collection",
2601+ "use `@data $name dir/*.json` (each file becomes `$name.STEM.key`)",
2602+ ),
2603+ ("@env", "use `@data $name env:NAME [default]`"),
2604+ (
2605+ "@fetch",
2606+ "download the data before building and use `@data $name file.json`",
2607+ ),
2608+ (
2609+ "@translations",
2610+ "put each locale's strings in a JSON file and use `@data $t locales/$lang.json`",
2611+ ),
2729 ("@defer", "remove it: the content is already in the page"),
2612 ("@defer", "remove it: the content is already in the page"),
2730 ("@log", "use `@warn`"),
2613 ("@log", "use `@warn`"),
2731 (
2614 (
@@ -4461,13 +4344,6 @@ enum JsonValue {
4461 Object(Vec<(String, JsonValue)>),
4344 Object(Vec<(String, JsonValue)>),
4462}
4345}
4463
4346
4464−fn parse_json(input: &str) -> Option<JsonValue> {
4465− let trimmed = input.trim();
4466− let chars: Vec<char> = trimmed.chars().collect();
4467− let (val, _) = parse_json_value(&chars, 0)?;
4468− Some(val)
4469−}
4470−
4471/// Parse JSON with an error message indicating where parsing failed.
4347/// Parse JSON with an error message indicating where parsing failed.
4472fn parse_json_with_error(input: &str) -> Result<JsonValue, String> {
4348fn parse_json_with_error(input: &str) -> Result<JsonValue, String> {
4473 let trimmed = input.trim();
4349 let trimmed = input.trim();
@@ -4809,109 +4685,6 @@ fn json_value_to_string(v: &JsonValue) -> String {
4809 }
4685 }
4810}
4686}
4811
4687
4812−// ---------------------------------------------------------------------------
4813−// HTTP fetch helper (blocking, minimal, no dependencies)
4814−// ---------------------------------------------------------------------------
4815−
4816−fn fetch_url_blocking(url: &str) -> Result<String, String> {
4817− use std::io::{Read, Write};
4818− use std::net::TcpStream;
4819−
4820− let is_https = url.starts_with("https://");
4821− let url_without_scheme = url
4822− .strip_prefix("https://")
4823− .or_else(|| url.strip_prefix("http://"))
4824− .ok_or_else(|| "URL must start with http:// or https://".to_string())?;
4825−
4826− let (host_port, path) = match url_without_scheme.find('/') {
4827− Some(pos) => (&url_without_scheme[..pos], &url_without_scheme[pos..]),
4828− None => (url_without_scheme, "/"),
4829− };
4830−
4831− let (host, port) = match host_port.find(':') {
4832− Some(pos) => (
4833− &host_port[..pos],
4834− host_port[pos + 1..]
4835− .parse::<u16>()
4836− .map_err(|e| format!("invalid port: {}", e))?,
4837− ),
4838− None => (host_port, if is_https { 443 } else { 80 }),
4839− };
4840−
4841− if is_https {
4842− return Err("@fetch does not support https:// (no TLS in std). Use @data with a local JSON file, or set up a build script to fetch data before compilation.".to_string());
4843− }
4844−
4845− let addr = format!("{}:{}", host, port);
4846− let mut stream = TcpStream::connect(&addr).map_err(|e| format!("connection failed: {}", e))?;
4847− stream
4848− .set_read_timeout(Some(std::time::Duration::from_secs(10)))
4849− .ok();
4850−
4851− let request = format!(
4852− "GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept: application/json, text/plain, */*\r\n\r\n",
4853− path, host
4854− );
4855− stream
4856− .write_all(request.as_bytes())
4857− .map_err(|e| format!("write failed: {}", e))?;
4858−
4859− let mut response = Vec::new();
4860− stream
4861− .read_to_end(&mut response)
4862− .map_err(|e| format!("read failed: {}", e))?;
4863−
4864− // Split headers and body (on bytes, so chunk sizes stay byte counts)
4865− if let Some(body_start) = response.windows(4).position(|w| w == b"\r\n\r\n") {
4866− let headers = String::from_utf8_lossy(&response[..body_start]);
4867− let body = &response[body_start + 4..];
4868−
4869− // Check status code
4870− if let Some(first_line) = headers.lines().next()
4871− && let Some(code_str) = first_line.split_whitespace().nth(1)
4872− {
4873− let code: u16 = code_str.parse().unwrap_or(0);
4874− if code >= 400 {
4875− return Err(format!("HTTP {}", code));
4876− }
4877− }
4878−
4879− // Handle chunked transfer encoding
4880− if headers
4881− .to_lowercase()
4882− .contains("transfer-encoding: chunked")
4883− {
4884− return Ok(String::from_utf8_lossy(&decode_chunked(body)).into_owned());
4885− }
4886−
4887− Ok(String::from_utf8_lossy(body).into_owned())
4888− } else {
4889− Err("malformed HTTP response".to_string())
4890− }
4891−}
4892−
4893−fn decode_chunked(body: &[u8]) -> Vec<u8> {
4894− let mut result = Vec::new();
4895− let mut rest = body;
4896− while let Some(size_end) = rest.windows(2).position(|w| w == b"\r\n") {
4897− let size_line = String::from_utf8_lossy(&rest[..size_end]);
4898− // Chunk extensions (`;name=value`) may follow the size.
4899− let size_str = size_line.split(';').next().unwrap_or("").trim();
4900− let size = usize::from_str_radix(size_str, 16).unwrap_or(0);
4901− if size == 0 {
4902− break;
4903− }
4904− let chunk = &rest[size_end + 2..];
4905− if size > chunk.len() {
4906− result.extend_from_slice(chunk);
4907− break;
4908− }
4909− result.extend_from_slice(&chunk[..size]);
4910− rest = chunk[size..].strip_prefix(b"\r\n").unwrap_or(&chunk[size..]);
4911− }
4912− result
4913−}
4914−
4915// ---------------------------------------------------------------------------
4688// ---------------------------------------------------------------------------
4916// SVG attribute injection helper
4689// SVG attribute injection helper
4917// ---------------------------------------------------------------------------
4690// ---------------------------------------------------------------------------
@@ -5330,15 +5103,4 @@ mod tests {
5330 let r = parse("@let x = 1 + 2\n");
5103 let r = parse("@let x = 1 + 2\n");
5331 assert_eq!(r.document.variables.get("x").map(|s| s.as_str()), Some("3"));
5104 assert_eq!(r.document.variables.get("x").map(|s| s.as_str()), Some("3"));
5332 }
5105 }
5333−
5334− #[test]
5335− fn decode_chunked_handles_bytes_and_malformed_input() {
5336− // Chunk sizes are byte counts: "é" is 2 bytes.
5337− let body = "3\r\naé\r\n2\r\nbc\r\n0\r\n\r\n".as_bytes();
5338− assert_eq!(decode_chunked(body), "aébc".as_bytes());
5339− // No CRLF after the size line — must not panic.
5340− assert_eq!(decode_chunked(b"5"), b"");
5341− // Declared size larger than the data — take what's there.
5342− assert_eq!(decode_chunked(b"a\r\nabc"), b"abc");
5343− }
5344}
5106}
src/upgrade.rs+103
@@ -242,6 +242,58 @@ fn rewrite_block(
242 return Some(Block { lines: out, end });
242 return Some(Block { lines: out, end });
243 }
243 }
244
244
245+ // @translations [LOCALE] with `locale:` sections of `key value` lines →
246+ // `@let t.key value` for the active locale
247+ if trimmed == "@translations" || trimmed.starts_with("@translations ") {
248+ let explicit = trimmed["@translations".len()..].trim();
249+ let page_lang = lines.iter().find_map(|l| {
250+ let rest = l.trim().strip_prefix("@page [")?;
251+ let attrs = rest.split(']').next()?;
252+ attrs
253+ .split(',')
254+ .find_map(|a| a.trim().strip_prefix("lang "))
255+ .map(|v| v.trim().to_string())
256+ });
257+ let active = if !explicit.is_empty() {
258+ explicit.to_string()
259+ } else {
260+ page_lang.unwrap_or_else(|| "en".to_string())
261+ };
262+ let mut locale = active.clone();
263+ let mut out = Vec::new();
264+ let mut others = Vec::new();
265+ for l in body {
266+ let t = l.trim();
267+ if t.is_empty() {
268+ continue;
269+ }
270+ if let Some(name) = t.strip_suffix(':').filter(|n| !n.contains(' ')) {
271+ locale = name.to_string();
272+ if locale != active && !others.contains(&locale) {
273+ others.push(locale.clone());
274+ }
275+ continue;
276+ }
277+ if locale == active
278+ && let Some((key, value)) = t.split_once(' ')
279+ {
280+ out.push(format!("{pad}@let t.{} {}", key, value.trim()));
281+ }
282+ }
283+ if !others.is_empty() {
284+ manual.push((
285+ i + 1,
286+ format!(
287+ "kept the '{}' strings; move the other locales ({}) to JSON files and load \
288+ them with `@data $t locales/$lang.json`",
289+ active,
290+ others.join(", ")
291+ ),
292+ ));
293+ }
294+ return Some(Block { lines: out, end });
295+ }
296+
245 // @defer → its body, dedented
297 // @defer → its body, dedented
246 if trimmed == "@defer" || trimmed.starts_with("@defer ") {
298 if trimmed == "@defer" || trimmed.starts_with("@defer ") {
247 return Some(Block {
299 return Some(Block {
@@ -447,6 +499,45 @@ fn rewrite_directive_line(
447 {
499 {
448 return format!("{pad}@let {} = {}", name, value.trim());
500 return format!("{pad}@let {} = {}", name, value.trim());
449 }
501 }
502+ if let Some(rest) = trimmed.strip_prefix("@env ") {
503+ let rest = rest.trim();
504+ let (var, default) = match rest.split_once(char::is_whitespace) {
505+ Some((var, default)) => (var, format!(" {}", default.trim())),
506+ None => (rest, String::new()),
507+ };
508+ let name = var.to_lowercase().replace('-', "_");
509+ return format!("{pad}@data ${} env:{}{}", name, var, default);
510+ }
511+ if let Some(rest) = trimmed.strip_prefix("@collection ") {
512+ let rest = rest.trim();
513+ let parsed = match rest.strip_prefix('$') {
514+ Some(named) => named
515+ .split_once(char::is_whitespace)
516+ .map(|(n, p)| (n.to_string(), p.trim().trim_matches('"').to_string())),
517+ None => rest
518+ .rsplit_once(" as ")
519+ .map(|(p, n)| (n.trim().to_string(), p.trim().trim_matches('"').to_string())),
520+ };
521+ if let Some((name, pattern)) = parsed {
522+ manual.push((
523+ idx + 1,
524+ format!(
525+ "collection values are now `${name}.STEM.key` (they were `${name}_STEM_key`), \
526+ and `${name}` lists the stems separated by commas"
527+ ),
528+ ));
529+ return format!("{pad}@data ${} {}", name, pattern);
530+ }
531+ }
532+ if trimmed.starts_with("@fetch ") {
533+ manual.push((
534+ idx + 1,
535+ "@fetch was removed: download the data before building and load it with \
536+ `@data $name file.json`"
537+ .to_string(),
538+ ));
539+ return line.to_string();
540+ }
450 if let Some(rest) = trimmed.strip_prefix("@import ") {
541 if let Some(rest) = trimmed.strip_prefix("@import ") {
451 return format!("{pad}@include {}", rest);
542 return format!("{pad}@include {}", rest);
452 }
543 }
@@ -992,6 +1083,18 @@ mod tests {
992 assert_eq!(up("@let card $title $tone=primary\n @text $title"), "@let card $title $tone=primary\n @text $title");
1083 assert_eq!(up("@let card $title $tone=primary\n @text $title"), "@let card $title $tone=primary\n @text $title");
993 }
1084 }
994
1085
1086+ #[test]
1087+ fn data_directives() {
1088+ assert_eq!(up("@env API_URL http://x"), "@data $api_url env:API_URL http://x");
1089+ assert_eq!(up("@collection $posts \"posts/*.json\""), "@data $posts posts/*.json");
1090+ assert_eq!(
1091+ up("@page [lang fr] T\n@translations\n en:\n hi Hello\n fr:\n hi Bonjour\n@text $t.hi"),
1092+ "@page [lang fr] T\n@let t.hi Bonjour\n@text $t.hi"
1093+ );
1094+ assert_eq!(upgrade("@translations\n en:\n hi Hello\n fr:\n hi Salut").manual.len(), 1);
1095+ assert_eq!(upgrade("@fetch $d http://x").manual.len(), 1);
1096+ }
1097+
995 #[test]
1098 #[test]
996 fn verbatim_regions_are_untouched() {
1099 fn verbatim_regions_are_untouched() {
997 let src = "@script\n for (x of y) {}\n@raw \"\"\"\n@unless\n\"\"\"\n@style\n .p { }";
1100 let src = "@script\n for (x of y) {}\n@raw \"\"\"\n@unless\n\"\"\"\n@style\n .p { }";
tests/regressions.rs+14
@@ -371,3 +371,17 @@ fn htmlang_attributes_sharing_html_names_do_not_warn() {
371 result.diagnostics
371 result.diagnostics
372 );
372 );
373}
373}
374+
375+#[test]
376+fn data_glob_loads_each_file_under_its_stem() {
377+ let dir = scratch_dir("data_glob");
378+ std::fs::create_dir_all(dir.join("posts")).unwrap();
379+ std::fs::write(dir.join("posts/a.json"), r#"{"title": "First"}"#).unwrap();
380+ std::fs::write(dir.join("posts/b.json"), r#"{"title": "Second"}"#).unwrap();
381+ let out = compile_in(
382+ &dir,
383+ "@data $posts posts/*.json\n@text $posts._count: $posts.a.title, $posts.b.title\n@each $p in $posts\n @text [class=item] $p",
384+ );
385+ assert!(out.contains("2: First, Second"), "{}", out);
386+ assert!(out.contains(">a<") && out.contains(">b<"), "{}", out);
387+}
tests/snapshots.rs+16−38
@@ -4690,13 +4690,12 @@ fn snapshot_responsive_images() {
4690// --- Inline tests for batch 2 ---
4690// --- Inline tests for batch 2 ---
4691
4691
4692#[test]
4692#[test]
4693−fn translations_inject_variables() {
4693+fn translations_point_to_data_files() {
4694− let output =
4695− compile("@translations\n en:\n hello Hi\n world Earth\n@text $t.hello $t.world");
4694+ let diags = parse_diagnostics("@translations\n en:\n hello Hi");
4696− assert!(output.contains("Hi"), "should inject translation for hello");
4697 assert!(
4695 assert!(
4698− output.contains("Earth"),
4696+ diags.iter().any(|d| d.message.contains("@data $t locales/$lang.json")),
4699− "should inject translation for world"
4697+ "{:?}",
4698+ diags
4700 );
4699 );
4701}
4700}
4702
4701
@@ -4771,10 +4770,10 @@ fn repeat_with_index() {
4771#[test]
4770#[test]
4772fn env_directive_with_default() {
4771fn env_directive_with_default() {
4773 let output =
4772 let output =
4774− compile("@env HTMLANG_TEST_NONEXISTENT fallback_value\n@text $htmlang_test_nonexistent");
4773+ compile("@data $fallback env:HTMLANG_TEST_NONEXISTENT fallback_value\n@text $fallback");
4775 assert!(
4774 assert!(
4776 output.contains("fallback_value"),
4775 output.contains("fallback_value"),
4777− "@env should use default when var is not set, got: {}",
4776+ "env: data should use the default when unset, got: {}",
4778 output
4777 output
4779 );
4778 );
4780}
4779}
@@ -4785,10 +4784,10 @@ fn env_directive_from_environment() {
4785 unsafe {
4784 unsafe {
4786 std::env::set_var("HTMLANG_TEST_VAR", "hello_world");
4785 std::env::set_var("HTMLANG_TEST_VAR", "hello_world");
4787 }
4786 }
4788− let output = compile("@env HTMLANG_TEST_VAR\n@text $htmlang_test_var");
4787+ let output = compile("@data $var env:HTMLANG_TEST_VAR\n@text $var");
4789 assert!(
4788 assert!(
4790 output.contains("hello_world"),
4789 output.contains("hello_world"),
4791− "@env should read env var, got: {}",
4790+ "env: data should read the variable, got: {}",
4792 output
4791 output
4793 );
4792 );
4794 unsafe {
4793 unsafe {
@@ -4798,7 +4797,7 @@ fn env_directive_from_environment() {
4798
4797
4799#[test]
4798#[test]
4800fn env_directive_warning_when_missing() {
4799fn env_directive_warning_when_missing() {
4801− let result = htmlang::parser::parse("@env HTMLANG_DEFINITELY_NOT_SET_12345");
4800+ let result = htmlang::parser::parse("@data $x env:HTMLANG_DEFINITELY_NOT_SET_12345");
4802 let warnings: Vec<_> = result
4801 let warnings: Vec<_> = result
4803 .diagnostics
4802 .diagnostics
4804 .iter()
4803 .iter()
@@ -4808,39 +4807,18 @@ fn env_directive_warning_when_missing() {
4808 .collect();
4807 .collect();
4809 assert!(
4808 assert!(
4810 !warnings.is_empty(),
4809 !warnings.is_empty(),
4811− "@env should warn when var is not set, got: {:?}",
4810+ "env: data should warn when unset, got: {:?}",
4812 result.diagnostics
4811 result.diagnostics
4813 );
4812 );
4814}
4813}
4815
4814
4816#[test]
4815#[test]
4817−fn fetch_directive_error_on_https() {
4816+fn fetch_was_removed() {
4818− // @fetch with https should produce an error (no TLS support)
4819− let result = htmlang::parser::parse("@fetch $data https://example.com/api");
4817+ let diags = parse_diagnostics("@fetch $data https://example.com/api");
4820− let errors: Vec<_> = result
4821− .diagnostics
4822− .iter()
4823− .filter(|d| d.severity == htmlang::parser::Severity::Error && d.message.contains("https"))
4824− .collect();
4825 assert!(
4818 assert!(
4826− !errors.is_empty(),
4819+ diags.iter().any(|d| d.message.contains("`@fetch` was removed")),
4827− "@fetch https should error, got: {:?}",
4820+ "{:?}",
4828− result.diagnostics
4821+ diags
4829− );
4830−}
4831−
4832−#[test]
4833−fn fetch_directive_error_on_bad_url() {
4834− let result = htmlang::parser::parse("@fetch $data http://127.0.0.1:1/nonexistent");
4835− let errors: Vec<_> = result
4836− .diagnostics
4837− .iter()
4838− .filter(|d| d.severity == htmlang::parser::Severity::Error)
4839− .collect();
4840− assert!(
4841− !errors.is_empty(),
4842− "@fetch should error on unreachable URL, got: {:?}",
4843− result.diagnostics
4844 );
4822 );
4845}
4823}
4846
4824
tests/snapshots/env_directive.hl+1−1
@@ -1,3 +1,3 @@
1@page Env Test
1@page Env Test
2−@env HTMLANG_TEST_MISSING default_val
2+@data $htmlang_test_missing env:HTMLANG_TEST_MISSING default_val
3@text $htmlang_test_missing
3@text $htmlang_test_missing
tests/snapshots/translations_i18n.hl+2−7
@@ -1,10 +1,5 @@
1−@translations
2− en:
3− greeting Hello
1+@let t.greeting Hello
4− farewell Goodbye
2+@let t.farewell Goodbye
5− fr:
6− greeting Bonjour
7− farewell Au revoir
8
3
9@column [padding 20]
4@column [padding 20]
10 @text $t.greeting
5 @text $t.greeting