@@ -120,10 +120,6 @@ struct ParseContext {
120120 manifest: Option<crate::ast::ManifestConfig>,
121121 /// Track @import paths for circular dependency detection
122122 import_stack: Vec<PathBuf>,
123− /// i18n translations: locale -> (key -> value)
124− translations: HashMap<String, HashMap<String, String>>,
125− /// Current active locale for translations
126− active_locale: Option<String>,
127123}
128124
129125struct Parser {
@@ -222,8 +218,6 @@ pub fn parse_with_base(input: &str, base_path: Option<&Path>) -> ParseResult {
222218 json_ld_blocks: Vec::new(),
223219 manifest: None,
224220 import_stack: Vec::new(),
225− translations: HashMap::new(),
226− active_locale: None,
227221 };
228222 load_prelude(&mut ctx);
229223 let nodes = parser.parse_children(0, &mut ctx);
@@ -1159,113 +1153,6 @@ impl Parser {
11591153 return Ok(Some(included_nodes));
11601154 }
11611155
1162− // --- @env (compile-time environment variables) ---
1163−
1164− if let Some(rest) = content.strip_prefix("@env ") {
1165− let rest = rest.trim();
1166− // @env VAR_NAME default_value OR @env VAR_NAME
1167− let (var_name, default_val) = if let Some((name, default)) = rest.split_once(' ') {
1168− (name.trim(), Some(default.trim()))
1169− } else {
1170− (rest, None)
1171− };
1172− let env_val = std::env::var(var_name)
1173− .ok()
1174− .or_else(|| default_val.map(|d| substitute_vars(d, &ctx.variables)));
1175− match env_val {
1176− Some(val) => {
1177− // Store as variable with lowercase name
1178− let key = var_name.to_lowercase().replace('-', "_");
1179− ctx.variables.insert(key, val);
1180− }
1181− None => {
1182− ctx.diagnostics.push(Diagnostic {
1183− line: line_num,
1184− column: None,
1185− message: format!(
1186− "environment variable '{}' is not set and no default provided",
1187− var_name
1188− ),
1189− severity: Severity::Warning,
1190− source_line: Some(content.clone()),
1191− });
1192− }
1193− }
1194− return Ok(None);
1195− }
1196−
1197− // --- @fetch (compile-time HTTP data fetching) ---
1198−
1199− if let Some(rest) = content.strip_prefix("@fetch ") {
1200− let rest = rest.trim();
1201− // @fetch $prefix https://url OR @fetch https://url
1202− let (prefix, url) = if rest.starts_with('$') {
1203− if let Some((p, u)) = rest.split_once(' ') {
1204− (
1205− p.strip_prefix('$').unwrap_or(p).to_string(),
1206− u.trim().to_string(),
1207− )
1208− } else {
1209− return Err(ParseError {
1210− line: line_num,
1211− message: "@fetch requires: @fetch $prefix url or @fetch url".to_string(),
1212− });
1213− }
1214− } else {
1215− (String::new(), rest.to_string())
1216− };
1217−
1218− let url = substitute_vars(&url, &ctx.variables);
1219−
1220− // Synchronous HTTP GET using std::net
1221− match fetch_url_blocking(&url) {
1222− Ok(body) => {
1223− // Try to parse as JSON
1224− match parse_json(&body) {
1225− Some(json) => {
1226− if prefix.is_empty() {
1227− if let JsonValue::Object(pairs) = &json {
1228− for (key, val) in pairs {
1229− let mut sub = HashMap::new();
1230− flatten_json(key, val, &mut sub);
1231− for (k, v) in sub {
1232− ctx.variables.insert(k, v);
1233− }
1234− }
1235− } else {
1236− // Store the raw body as a single variable
1237− ctx.variables.insert("__fetch_body".to_string(), body);
1238− }
1239− } else {
1240− let mut sub = HashMap::new();
1241− flatten_json(&prefix, &json, &mut sub);
1242− for (k, v) in sub {
1243− ctx.variables.insert(k, v);
1244− }
1245− }
1246− }
1247− None => {
1248− // Not JSON — store raw body
1249− if prefix.is_empty() {
1250− ctx.variables.insert("__fetch_body".to_string(), body);
1251− } else {
1252− ctx.variables.insert(prefix, body);
1253− }
1254− }
1255− }
1256− }
1257− Err(e) => {
1258− ctx.diagnostics.push(Diagnostic {
1259− line: line_num,
1260− column: None,
1261− message: format!("@fetch failed for '{}': {}", url, e),
1262− severity: Severity::Error,
1263− source_line: Some(content.clone()),
1264− });
1265− }
1266− }
1267− return Ok(None);
1268− }
12691156
12701157 // --- @svg (inline SVG from file) ---
12711158
@@ -1345,25 +1232,68 @@ impl Parser {
13451232
13461233 // --- @data (load JSON file into variables) ---
13471234
1235+ // @data file.json top-level keys become variables
1236+ // @data $name file.json values as $name.key
1237+ // @data $name dir/*.json each file as $name.STEM.key; $name
1238+ // lists the stems, $name._count counts them
1239+ // @data $name env:NAME [DEFAULT] an environment variable
13481240 if let Some(rest) = content.strip_prefix("@data ") {
13491241 let rest = rest.trim();
1350− // @data $prefix file.json OR @data file.json (no prefix, top-level keys become vars)
1351− let (prefix, filename) = if rest.starts_with('$') {
1352− if let Some((p, f)) = rest.split_once(' ') {
1353− (
1354− p.strip_prefix('$').unwrap_or(p).to_string(),
1355− f.trim().to_string(),
1356− )
1357− } else {
1242+ let (prefix, filename) = match rest.strip_prefix('$') {
1243+ Some(named) => match named.split_once(char::is_whitespace) {
1244+ Some((name, source)) => (name.to_string(), source.trim().to_string()),
1245+ None => {
1246+ return Err(ParseError {
1247+ line: line_num,
1248+ message: "@data requires: @data $name SOURCE or @data file.json"
1249+ .to_string(),
1250+ });
1251+ }
1252+ },
1253+ None => (String::new(), rest.to_string()),
1254+ };
1255+
1256+ if let Some(env) = filename.strip_prefix("env:") {
1257+ let (var, default) = match env.split_once(char::is_whitespace) {
1258+ Some((var, default)) => (var, Some(default.trim())),
1259+ None => (env, None),
1260+ };
1261+ if prefix.is_empty() {
13581262 return Err(ParseError {
13591263 line: line_num,
1360− message: "@data requires: @data $prefix file.json or @data file.json"
1361− .to_string(),
1264+ message: format!("@data env:{} needs a name: @data $name env:{}", var, var),
13621265 });
13631266 }
1364− } else {
1365− (String::new(), rest.to_string())
1366− };
1267+ let value = std::env::var(var)
1268+ .ok()
1269+ .or_else(|| default.map(|d| substitute_vars(d, &ctx.variables)));
1270+ if value.is_none() {
1271+ ctx.diagnostics.push(Diagnostic {
1272+ line: line_num,
1273+ column: None,
1274+ message: format!(
1275+ "environment variable '{}' is not set and has no default",
1276+ var
1277+ ),
1278+ severity: Severity::Warning,
1279+ source_line: Some(content.clone()),
1280+ });
1281+ }
1282+ ctx.variables.insert(prefix, value.unwrap_or_default());
1283+ return Ok(None);
1284+ }
1285+
1286+ let filename = substitute_vars(&filename, &ctx.variables);
1287+ if filename.contains('*') {
1288+ if prefix.is_empty() {
1289+ return Err(ParseError {
1290+ line: line_num,
1291+ message: format!("@data {} needs a name: @data $name {}", filename, filename),
1292+ });
1293+ }
1294+ self.load_data_glob(&prefix, &filename, line_num, &content, ctx);
1295+ return Ok(None);
1296+ }
13671297
13681298 let filename = substitute_vars(&filename, &ctx.variables);
13691299 let resolved = match &ctx.base_path {
@@ -1431,96 +1361,6 @@ impl Parser {
14311361 }
14321362
14331363
1434− // --- @collection (load multiple JSON files matching a glob pattern) ---
1435−
1436− if let Some(rest) = content.strip_prefix("@collection ") {
1437− let rest = rest.trim();
1438− // @collection $varname "pattern" OR @collection "pattern" as varname
1439− let (var_name, pattern) = if rest.starts_with('$') {
1440− let parts: Vec<&str> = rest.splitn(2, ' ').collect();
1441− if parts.len() == 2 {
1442− let name = parts[0].strip_prefix('$').unwrap_or(parts[0]);
1443− let pat = parts[1].trim().trim_matches('"');
1444− (name.to_string(), pat.to_string())
1445− } else {
1446− return Err(ParseError {
1447− line: line_num,
1448− message: "@collection requires: @collection $var \"pattern\"".to_string(),
1449− });
1450− }
1451− } else {
1452− let pat = rest.trim_matches('"');
1453− ("_items".to_string(), pat.to_string())
1454− };
1455−
1456− let pattern = substitute_vars(&pattern, &ctx.variables);
1457− let base = ctx.base_path.clone().unwrap_or_else(|| PathBuf::from("."));
1458− let glob_pattern = base.join(&pattern);
1459−
1460− // Simple glob: support * in filename part
1461− let parent = glob_pattern.parent().unwrap_or(Path::new("."));
1462− let file_pattern = glob_pattern
1463− .file_name()
1464− .map(|f| f.to_string_lossy().to_string())
1465− .unwrap_or_default();
1466−
1467− let mut items = Vec::new();
1468− if let Ok(entries) = std::fs::read_dir(parent) {
1469− let mut paths: Vec<PathBuf> = entries
1470− .flatten()
1471− .map(|e| e.path())
1472− .filter(|p| {
1473− if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
1474− if file_pattern.contains('*') {
1475− let parts: Vec<&str> = file_pattern.split('*').collect();
1476− if parts.len() == 2 {
1477− name.starts_with(parts[0]) && name.ends_with(parts[1])
1478− } else {
1479− true
1480− }
1481− } else {
1482− name == file_pattern
1483− }
1484− } else {
1485− false
1486− }
1487− })
1488− .collect();
1489− paths.sort();
1490− for path in &paths {
1491− if let Ok(text) = std::fs::read_to_string(path) {
1492− // Store the file content as a JSON string for each item
1493− let stem = path
1494− .file_stem()
1495− .and_then(|s| s.to_str())
1496− .unwrap_or("")
1497− .to_string();
1498− items.push((stem, text));
1499− }
1500− }
1501− }
1502−
1503− // Store item count and serialized data as variables
1504− ctx.variables
1505− .insert(format!("{}_count", var_name), items.len().to_string());
1506− // Store as space-separated list of stems for @each
1507− let stems: Vec<String> = items.iter().map(|(stem, _)| stem.clone()).collect();
1508− ctx.variables.insert(var_name.clone(), stems.join(" "));
1509− // Also parse each JSON file and store keys as prefix_stem_key
1510− for (stem, text) in &items {
1511− if let Some(json) = parse_json(text.trim())
1512− && let JsonValue::Object(ref obj) = json
1513− {
1514− for (key, val) in obj {
1515− let var_key = format!("{}_{}", stem, key);
1516− ctx.variables.insert(var_key, json_value_to_string(val));
1517− }
1518− }
1519− }
1520−
1521− return Ok(None);
1522− }
1523−
15241364 // --- @if / @else ---
15251365
15261366 if let Some(rest) = content.strip_prefix("@if ") {
@@ -2016,58 +1856,6 @@ impl Parser {
20161856 return Ok(None);
20171857 }
20181858
2019− // --- @translations (i18n) ---
2020−
2021− if content.trim() == "@translations" || content.starts_with("@translations ") {
2022− let locale = content
2023− .strip_prefix("@translations")
2024− .unwrap_or("")
2025− .trim()
2026− .to_string();
2027− let active_locale = if locale.is_empty() {
2028− ctx.lang.clone().unwrap_or_else(|| "en".to_string())
2029− } else {
2030− locale
2031− };
2032−
2033− // Collect indented body: key value pairs, grouped by locale headers
2034− let mut current_locale = active_locale.clone();
2035− let mut first_locale_indent: Option<usize> = None;
2036− while self.pos < self.lines.len() && self.lines[self.pos].indent > current_indent {
2037− if let LineContent::Normal(ref s) = self.lines[self.pos].content {
2038− let trimmed = s.trim();
2039− // Check for locale sub-header: e.g., "en:" or "fr:" or "es:"
2040− if trimmed.ends_with(':') && trimmed.len() <= 10 && !trimmed.contains(' ') {
2041− current_locale = trimmed[..trimmed.len() - 1].to_string();
2042− first_locale_indent = Some(self.lines[self.pos].indent);
2043− self.pos += 1;
2044− continue;
2045− }
2046− // key value pair
2047− if let Some((key, value)) = trimmed.split_once(' ') {
2048− let key = key.trim().to_string();
2049− let value = substitute_vars(value.trim(), &ctx.variables);
2050− ctx.translations
2051− .entry(current_locale.clone())
2052− .or_default()
2053− .insert(key, value);
2054− }
2055− }
2056− self.pos += 1;
2057− }
2058− // Set the active locale
2059− if ctx.active_locale.is_none() {
2060− ctx.active_locale = Some(active_locale.clone());
2061− }
2062− let _ = first_locale_indent; // suppress unused
2063− // Inject active locale's translations as $t.key variables
2064− if let Some(strings) = ctx.translations.get(&active_locale) {
2065− for (key, value) in strings {
2066− ctx.variables.insert(format!("t.{}", key), value.clone());
2067− }
2068− }
2069− return Ok(None);
2070− }
20711859
20721860 // --- @deprecated annotation ---
20731861
@@ -2258,6 +2046,90 @@ impl Parser {
22582046 Ok(Some(vec![Node::Text(segments)]))
22592047 }
22602048
2049+ /// `@data $name dir/*.json`: load every matching file as
2050+ /// `$name.STEM.key`, list the stems in `$name` and count them.
2051+ fn load_data_glob(
2052+ &mut self,
2053+ name: &str,
2054+ pattern: &str,
2055+ line_num: usize,
2056+ content: &str,
2057+ ctx: &mut ParseContext,
2058+ ) {
2059+ let base = ctx.base_path.clone().unwrap_or_else(|| PathBuf::from("."));
2060+ let pattern_path = Path::new(pattern);
2061+ let dir = match pattern_path.parent() {
2062+ Some(dir) if !dir.as_os_str().is_empty() => base.join(dir),
2063+ _ => base,
2064+ };
2065+ let file_pattern = pattern_path
2066+ .file_name()
2067+ .unwrap_or_default()
2068+ .to_string_lossy()
2069+ .to_string();
2070+ let mut files: Vec<PathBuf> = match std::fs::read_dir(&dir) {
2071+ Ok(entries) => entries
2072+ .flatten()
2073+ .map(|e| e.path())
2074+ .filter(|p| {
2075+ p.file_name()
2076+ .is_some_and(|n| glob_match(&file_pattern, &n.to_string_lossy()))
2077+ })
2078+ .collect(),
2079+ Err(e) => {
2080+ ctx.diagnostics.push(Diagnostic {
2081+ line: line_num,
2082+ column: None,
2083+ message: format!("cannot read directory for '{}': {}", pattern, e),
2084+ severity: Severity::Error,
2085+ source_line: Some(content.to_string()),
2086+ });
2087+ return;
2088+ }
2089+ };
2090+ files.sort();
2091+ let mut stems = Vec::new();
2092+ for file in files {
2093+ let stem = file
2094+ .file_stem()
2095+ .unwrap_or_default()
2096+ .to_string_lossy()
2097+ .to_string();
2098+ let text = match std::fs::read_to_string(&file) {
2099+ Ok(text) => text,
2100+ Err(e) => {
2101+ ctx.diagnostics.push(Diagnostic {
2102+ line: line_num,
2103+ column: None,
2104+ message: format!("cannot load data '{}': {}", file.display(), e),
2105+ severity: Severity::Error,
2106+ source_line: Some(content.to_string()),
2107+ });
2108+ continue;
2109+ }
2110+ };
2111+ match parse_json_with_error(&text) {
2112+ Ok(json) => {
2113+ let mut vars = HashMap::new();
2114+ flatten_json(&format!("{}.{}", name, stem), &json, &mut vars);
2115+ ctx.variables.extend(vars);
2116+ stems.push(stem);
2117+ }
2118+ Err(detail) => ctx.diagnostics.push(Diagnostic {
2119+ line: line_num,
2120+ column: None,
2121+ message: format!("invalid JSON in '{}': {}", file.display(), detail),
2122+ severity: Severity::Error,
2123+ source_line: Some(content.to_string()),
2124+ }),
2125+ }
2126+ ctx.included_files.push(file);
2127+ }
2128+ ctx.variables
2129+ .insert(format!("{}._count", name), stems.len().to_string());
2130+ ctx.variables.insert(name.to_string(), stems.join(", "));
2131+ }
2132+
22612133 fn expand_fn_call(
22622134 &mut self,
22632135 name: &str,
@@ -2663,9 +2535,7 @@ const KNOWN_DIRECTIVES: &[&str] = &[
26632535 "json-ld",
26642536 "assert",
26652537 "markdown",
2666− "collection",
26672538 "manifest",
2668− "translations",
26692539];
26702540
26712541fn parse_element_kind(s: &str, line_num: usize) -> Result<ElementKind, ParseError> {
@@ -2726,6 +2596,19 @@ const REMOVED_SYNTAX: &[(&str, &str)] = &[
27262596 ("@base", "use `@page [base ...] Title`"),
27272597 ("@og", "use `@meta og:NAME VALUE`"),
27282598 ("@debug", "use `@warn`"),
2599+ (
2600+ "@collection",
2601+ "use `@data $name dir/*.json` (each file becomes `$name.STEM.key`)",
2602+ ),
2603+ ("@env", "use `@data $name env:NAME [default]`"),
2604+ (
2605+ "@fetch",
2606+ "download the data before building and use `@data $name file.json`",
2607+ ),
2608+ (
2609+ "@translations",
2610+ "put each locale's strings in a JSON file and use `@data $t locales/$lang.json`",
2611+ ),
27292612 ("@defer", "remove it: the content is already in the page"),
27302613 ("@log", "use `@warn`"),
27312614 (
@@ -4461,13 +4344,6 @@ enum JsonValue {
44614344 Object(Vec<(String, JsonValue)>),
44624345}
44634346
4464−fn parse_json(input: &str) -> Option<JsonValue> {
4465− let trimmed = input.trim();
4466− let chars: Vec<char> = trimmed.chars().collect();
4467− let (val, _) = parse_json_value(&chars, 0)?;
4468− Some(val)
4469−}
4470−
44714347/// Parse JSON with an error message indicating where parsing failed.
44724348fn parse_json_with_error(input: &str) -> Result<JsonValue, String> {
44734349 let trimmed = input.trim();
@@ -4809,109 +4685,6 @@ fn json_value_to_string(v: &JsonValue) -> String {
48094685 }
48104686}
48114687
4812−// ---------------------------------------------------------------------------
4813−// HTTP fetch helper (blocking, minimal, no dependencies)
4814−// ---------------------------------------------------------------------------
4815−
4816−fn fetch_url_blocking(url: &str) -> Result<String, String> {
4817− use std::io::{Read, Write};
4818− use std::net::TcpStream;
4819−
4820− let is_https = url.starts_with("https://");
4821− let url_without_scheme = url
4822− .strip_prefix("https://")
4823− .or_else(|| url.strip_prefix("http://"))
4824− .ok_or_else(|| "URL must start with http:// or https://".to_string())?;
4825−
4826− let (host_port, path) = match url_without_scheme.find('/') {
4827− Some(pos) => (&url_without_scheme[..pos], &url_without_scheme[pos..]),
4828− None => (url_without_scheme, "/"),
4829− };
4830−
4831− let (host, port) = match host_port.find(':') {
4832− Some(pos) => (
4833− &host_port[..pos],
4834− host_port[pos + 1..]
4835− .parse::<u16>()
4836− .map_err(|e| format!("invalid port: {}", e))?,
4837− ),
4838− None => (host_port, if is_https { 443 } else { 80 }),
4839− };
4840−
4841− if is_https {
4842− return Err("@fetch does not support https:// (no TLS in std). Use @data with a local JSON file, or set up a build script to fetch data before compilation.".to_string());
4843− }
4844−
4845− let addr = format!("{}:{}", host, port);
4846− let mut stream = TcpStream::connect(&addr).map_err(|e| format!("connection failed: {}", e))?;
4847− stream
4848− .set_read_timeout(Some(std::time::Duration::from_secs(10)))
4849− .ok();
4850−
4851− let request = format!(
4852− "GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept: application/json, text/plain, */*\r\n\r\n",
4853− path, host
4854− );
4855− stream
4856− .write_all(request.as_bytes())
4857− .map_err(|e| format!("write failed: {}", e))?;
4858−
4859− let mut response = Vec::new();
4860− stream
4861− .read_to_end(&mut response)
4862− .map_err(|e| format!("read failed: {}", e))?;
4863−
4864− // Split headers and body (on bytes, so chunk sizes stay byte counts)
4865− if let Some(body_start) = response.windows(4).position(|w| w == b"\r\n\r\n") {
4866− let headers = String::from_utf8_lossy(&response[..body_start]);
4867− let body = &response[body_start + 4..];
4868−
4869− // Check status code
4870− if let Some(first_line) = headers.lines().next()
4871− && let Some(code_str) = first_line.split_whitespace().nth(1)
4872− {
4873− let code: u16 = code_str.parse().unwrap_or(0);
4874− if code >= 400 {
4875− return Err(format!("HTTP {}", code));
4876− }
4877− }
4878−
4879− // Handle chunked transfer encoding
4880− if headers
4881− .to_lowercase()
4882− .contains("transfer-encoding: chunked")
4883− {
4884− return Ok(String::from_utf8_lossy(&decode_chunked(body)).into_owned());
4885− }
4886−
4887− Ok(String::from_utf8_lossy(body).into_owned())
4888− } else {
4889− Err("malformed HTTP response".to_string())
4890− }
4891−}
4892−
4893−fn decode_chunked(body: &[u8]) -> Vec<u8> {
4894− let mut result = Vec::new();
4895− let mut rest = body;
4896− while let Some(size_end) = rest.windows(2).position(|w| w == b"\r\n") {
4897− let size_line = String::from_utf8_lossy(&rest[..size_end]);
4898− // Chunk extensions (`;name=value`) may follow the size.
4899− let size_str = size_line.split(';').next().unwrap_or("").trim();
4900− let size = usize::from_str_radix(size_str, 16).unwrap_or(0);
4901− if size == 0 {
4902− break;
4903− }
4904− let chunk = &rest[size_end + 2..];
4905− if size > chunk.len() {
4906− result.extend_from_slice(chunk);
4907− break;
4908− }
4909− result.extend_from_slice(&chunk[..size]);
4910− rest = chunk[size..].strip_prefix(b"\r\n").unwrap_or(&chunk[size..]);
4911− }
4912− result
4913−}
4914−
49154688// ---------------------------------------------------------------------------
49164689// SVG attribute injection helper
49174690// ---------------------------------------------------------------------------
@@ -5330,15 +5103,4 @@ mod tests {
53305103 let r = parse("@let x = 1 + 2\n");
53315104 assert_eq!(r.document.variables.get("x").map(|s| s.as_str()), Some("3"));
53325105 }
5333−
5334− #[test]
5335− fn decode_chunked_handles_bytes_and_malformed_input() {
5336− // Chunk sizes are byte counts: "é" is 2 bytes.
5337− let body = "3\r\naé\r\n2\r\nbc\r\n0\r\n\r\n".as_bytes();
5338− assert_eq!(decode_chunked(body), "aébc".as_bytes());
5339− // No CRLF after the size line — must not panic.
5340− assert_eq!(decode_chunked(b"5"), b"");
5341− // Declared size larger than the data — take what's there.
5342− assert_eq!(decode_chunked(b"a\r\nabc"), b"abc");
5343− }
53445106}