git.iko.soy/j
on master help sign in
j/src/lex.rs
447 lines 14 kB history raw
1//! Lexer for the j language (§3.1).
3use num_bigint::BigInt;
5#[derive(Debug, Clone, PartialEq)]
6pub enum Tok {
7 Ident(String),
8 TypeName(String),
9 Int(BigInt),
10 Text(String),
11 IdLit(String), // existing id, full or unique prefix
12 NewId, // bare `@`
13 LabelLit(String),
14 PathLit(Vec<String>),
15 Selector(String), // `.name` with no whitespace
16 Op(&'static str), // symbolic operator
17 // keywords / punctuation
18 Let,
19 In,
20 If,
21 Then,
22 Else,
23 Or,
24 True,
25 False,
26 LParen,
27 RParen,
28 LBracket,
29 RBracket,
30 LBrace,
31 RBrace,
32 Comma,
33 Semi,
34 Colon,
35 Backslash,
36 Equals,
37 Arrow,
38 Underscore,
39 Newline, // layout marker
40 Eof,
43#[derive(Debug, Clone, PartialEq)]
44pub struct SpTok {
45 pub tok: Tok,
46 pub line: usize, // 1-based
47 pub col: usize, // 1-based
48 /// byte offsets of the token in the source: `start..end`
49 pub start: usize,
50 pub end: usize,
53#[derive(Debug)]
54pub struct LexError {
55 pub msg: String,
56 pub line: usize,
59const SYMBOLIC_OPS: &[&str] = &[
60 "++", "::", "==", "/=", "<=", ">=", "&&", "||", ".", "*", "+", "-", "<", ">",
61];
63fn is_ident_start(c: char) -> bool {
64 c.is_ascii_lowercase() || c == '_'
66fn is_ident_char(c: char) -> bool {
67 c.is_ascii_alphanumeric() || c == '_' || c == '\''
69fn is_type_start(c: char) -> bool {
70 c.is_ascii_uppercase()
72fn is_id_char(c: char) -> bool {
73 ('k'..='z').contains(&c)
76fn valid_label(s: &str) -> bool {
77 !s.is_empty()
78 && s.chars()
79 .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '/' | '-'))
80 && !s.starts_with('.')
81 && !s.starts_with('-')
82 && !s.contains("..")
83 && !s.ends_with('/')
84 && !s.ends_with(".lock")
87pub fn lex(src: &str) -> Result<Vec<SpTok>, LexError> {
88 let mut toks: Vec<SpTok> = Vec::new();
89 let chars: Vec<char> = src.chars().collect();
90 // byte offset of each char, and of the end: tokens carry byte spans so
91 // the parser can keep a lambda's source text (§5.2)
92 let bytes: Vec<usize> = src
93 .char_indices()
94 .map(|(b, _)| b)
95 .chain(std::iter::once(src.len()))
96 .collect();
97 let mut i = 0usize;
98 let mut line = 1usize;
99 let mut col = 1usize; // column of next char
101 macro_rules! err {
102 ($msg:expr) => {
103 return Err(LexError {
104 msg: $msg.into(),
105 line,
106 })
107 };
108 }
110 // push helper; `$s..$e` are the token's char indices
111 macro_rules! push {
112 ($t:expr, $l:expr, $c:expr, $s:expr, $e:expr) => {
113 toks.push(SpTok {
114 tok: $t,
115 line: $l,
116 col: $c,
117 start: bytes[$s],
118 end: bytes[$e],
119 })
120 };
121 }
123 while i < chars.len() {
124 let c = chars[i];
125 if c == '\n' {
126 push!(Tok::Newline, line, col, i, i + 1);
127 i += 1;
128 line += 1;
129 col = 1;
130 continue;
131 }
132 if c == ' ' || c == '\t' || c == '\r' {
133 i += 1;
134 col += 1;
135 continue;
136 }
137 let start_line = line;
138 let start_col = col;
139 let start = i;
140 // comments
141 if c == '-' && i + 1 < chars.len() && chars[i + 1] == '-' {
142 // could be `--` comment, but `->`? no: `->` is dash-gt, distinct.
143 while i < chars.len() && chars[i] != '\n' {
144 i += 1;
145 col += 1;
146 }
147 continue;
148 }
149 if c == '{' && i + 1 < chars.len() && chars[i + 1] == '-' {
150 let mut depth = 1usize;
151 i += 2;
152 col += 2;
153 while i < chars.len() && depth > 0 {
154 if chars[i] == '{' && i + 1 < chars.len() && chars[i + 1] == '-' {
155 depth += 1;
156 i += 2;
157 col += 2;
158 } else if chars[i] == '-' && i + 1 < chars.len() && chars[i + 1] == '}' {
159 depth -= 1;
160 i += 2;
161 col += 2;
162 } else if chars[i] == '\n' {
163 push!(Tok::Newline, line, col, i, i + 1);
164 i += 1;
165 line += 1;
166 col = 1;
167 } else {
168 i += 1;
169 col += 1;
170 }
171 }
172 if depth > 0 {
173 err!("unterminated block comment");
174 }
175 // a comment is whitespace (§3.1): one spanning lines has already
176 // ended the line it started on, with the Newlines pushed above,
177 // and one that does not must leave its line unbroken
178 continue;
179 }
180 // arrow `->` (check before operator `-`)
181 if c == '-' && i + 1 < chars.len() && chars[i + 1] == '>' {
182 push!(Tok::Arrow, start_line, start_col, i, i + 2);
183 i += 2;
184 col += 2;
185 continue;
186 }
187 // path literal: `./` at start of token
188 if c == '.' && i + 1 < chars.len() && chars[i + 1] == '/' {
189 i += 2;
190 col += 2;
191 let mut comp = String::new();
192 let mut comps: Vec<String> = Vec::new();
193 while i < chars.len() {
194 let d = chars[i];
195 if d.is_whitespace() || matches!(d, '(' | ')' | '[' | ']' | '{' | '}' | ',' | ';') {
196 break;
197 }
198 if d == '/' {
199 comps.push(std::mem::take(&mut comp));
200 i += 1;
201 col += 1;
202 // trailing slash: peek—if next is terminator, stop (drop empty comp)
203 if i >= chars.len()
204 || chars[i].is_whitespace()
205 || matches!(chars[i], '(' | ')' | '[' | ']' | '{' | '}' | ',' | ';')
206 {
207 break;
208 }
209 continue;
210 }
211 comp.push(d);
212 i += 1;
213 col += 1;
214 }
215 if !comp.is_empty() {
216 comps.push(comp);
217 }
218 push!(Tok::PathLit(comps), start_line, start_col, start, i);
219 continue;
220 }
221 // selector: `.` immediately followed by identifier-start
222 if c == '.' && i + 1 < chars.len() && is_ident_start(chars[i + 1]) {
223 let mut name = String::new();
224 i += 1;
225 col += 1;
226 while i < chars.len() && is_ident_char(chars[i]) {
227 name.push(chars[i]);
228 i += 1;
229 col += 1;
230 }
231 push!(Tok::Selector(name), start_line, start_col, start, i);
232 continue;
233 }
234 // `@` id literals
235 if c == '@' {
236 if i + 1 >= chars.len() || !is_ident_char(chars[i + 1]) {
237 push!(Tok::NewId, start_line, start_col, i, i + 1);
238 i += 1;
239 col += 1;
240 continue;
241 }
242 if is_id_char(chars[i + 1]) {
243 let mut s = String::new();
244 i += 1;
245 col += 1;
246 while i < chars.len() && is_id_char(chars[i]) {
247 s.push(chars[i]);
248 i += 1;
249 col += 1;
250 }
251 // if the next char is still an identifier char, it's a lexical error
252 if i < chars.len() && is_ident_char(chars[i]) {
253 err!("invalid id literal");
254 }
255 push!(Tok::IdLit(s), start_line, start_col, start, i);
256 continue;
257 }
258 err!("`@` followed by an identifier character outside k-z is not a valid id literal");
259 }
260 // `%` label literals
261 if c == '%' {
262 let mut s = String::new();
263 i += 1;
264 col += 1;
265 while i < chars.len() {
266 let d = chars[i];
267 if d.is_ascii_alphanumeric() || matches!(d, '.' | '_' | '/' | '-') {
268 s.push(d);
269 i += 1;
270 col += 1;
271 } else {
272 break;
273 }
274 }
275 if !valid_label(&s) {
276 err!("invalid label literal");
277 }
278 push!(Tok::LabelLit(s), start_line, start_col, start, i);
279 continue;
280 }
281 // text literal, double- or single-quoted
282 if c == '"' || c == '\'' {
283 let quote = c;
284 i += 1;
285 col += 1;
286 let mut s = String::new();
287 loop {
288 if i >= chars.len() {
289 err!("unterminated text literal");
290 }
291 let d = chars[i];
292 if d == quote {
293 i += 1;
294 col += 1;
295 break;
296 }
297 match d {
298 '\\' => {
299 if i + 1 >= chars.len() {
300 err!("unterminated text literal");
301 }
302 let e = chars[i + 1];
303 let r = match e {
304 '"' => '"',
305 '\'' => '\'',
306 '\\' => '\\',
307 'n' => '\n',
308 't' => '\t',
309 'r' => '\r',
310 _ => err!("unknown escape in text literal"),
311 };
312 s.push(r);
313 i += 2;
314 col += 2;
315 }
316 '\n' => err!("newline in text literal"),
317 _ => {
318 s.push(d);
319 i += 1;
320 col += 1;
321 }
322 }
323 }
324 push!(Tok::Text(s), start_line, start_col, start, i);
325 continue;
326 }
327 // integer
328 if c.is_ascii_digit() {
329 let mut s = String::new();
330 while i < chars.len() && chars[i].is_ascii_digit() {
331 s.push(chars[i]);
332 i += 1;
333 col += 1;
334 }
335 push!(Tok::Int(s.parse::<BigInt>().unwrap()), start_line, start_col, start, i);
336 continue;
337 }
338 // identifier / keyword
339 if is_ident_start(c) {
340 let mut s = String::new();
341 while i < chars.len() && is_ident_char(chars[i]) {
342 s.push(chars[i]);
343 i += 1;
344 col += 1;
345 }
346 let t = match s.as_str() {
347 "let" => Tok::Let,
348 "in" => Tok::In,
349 "if" => Tok::If,
350 "then" => Tok::Then,
351 "else" => Tok::Else,
352 "or" => Tok::Or,
353 "true" => Tok::True,
354 "false" => Tok::False,
355 "_" => {
356 if s == "_" {
357 Tok::Underscore
358 } else {
359 Tok::Ident(s)
360 }
361 }
362 _ => Tok::Ident(s),
363 };
364 push!(t, start_line, start_col, start, i);
365 continue;
366 }
367 // type name
368 if is_type_start(c) {
369 let mut s = String::new();
370 while i < chars.len() {
371 let d = chars[i];
372 if d.is_ascii_alphanumeric() || d == '_' {
373 s.push(d);
374 i += 1;
375 col += 1;
376 } else {
377 break;
378 }
379 }
380 push!(Tok::TypeName(s), start_line, start_col, start, i);
381 continue;
382 }
383 // symbolic operators: longest match
384 let rest: String = chars[i..].iter().take(2).collect();
385 let mut matched: Option<&'static str> = None;
386 for op in SYMBOLIC_OPS {
387 if op.len() == 2 && rest == *op {
388 matched = Some(op);
389 break;
390 }
391 }
392 if matched.is_none() {
393 // compare by character: `rest` is built from chars, so slicing it
394 // by byte would panic on any non-ASCII character reaching here
395 for op in SYMBOLIC_OPS {
396 if op.len() == 1 && op.starts_with(c) {
397 matched = Some(op);
398 break;
399 }
400 }
401 }
402 if let Some(op) = matched {
403 push!(Tok::Op(op), start_line, start_col, i, i + op.len());
404 i += op.len();
405 col += op.len();
406 continue;
407 }
408 // punctuation
409 let t = match c {
410 '(' => Tok::LParen,
411 ')' => Tok::RParen,
412 '[' => Tok::LBracket,
413 ']' => Tok::RBracket,
414 '{' => Tok::LBrace,
415 '}' => Tok::RBrace,
416 ',' => Tok::Comma,
417 ';' => Tok::Semi,
418 ':' => Tok::Colon,
419 '\\' => Tok::Backslash,
420 '=' => Tok::Equals,
421 _ => err!(format!("unexpected character `{}`", c)),
422 };
423 push!(t, start_line, start_col, i, i + 1);
424 i += 1;
425 col += 1;
426 }
427 push!(Tok::Eof, line, col, i, i);
428 Ok(toks)
431impl Tok {
432 pub fn describe(&self) -> String {
433 match self {
434 Tok::Ident(s) => format!("`{}`", s),
435 Tok::TypeName(s) => format!("`{}`", s),
436 Tok::Int(n) => format!("`{}`", n),
437 Tok::Text(_) => "a text literal".into(),
438 Tok::IdLit(s) => format!("`@{}`", s),
439 Tok::NewId => "`@`".into(),
440 Tok::LabelLit(s) => format!("`%{}`", s),
441 Tok::PathLit(_) => "a path literal".into(),
442 Tok::Selector(s) => format!("`.{}`", s),
443 Tok::Op(o) => format!("`{}`", o),
444 other => format!("`{:?}`", other).to_lowercase(),
445 }
446 }