git.iko.soy/parseable-macos
on 2026.03.12.1735 help sign in
parseable-macos/ParseableViewerTests/SQLTokenizerTests.swift
351 lines 13 kB history raw
1import XCTest
2@testable import ParseableViewer
4final class SQLTokenizerTests: XCTestCase {
6 // MARK: - Tokenizer basics
8 func testTokenizesSimpleSelect() {
9 let tokens = SQLTokenizer.tokenize("SELECT * FROM t")
10 let kinds = tokens.map(\.kind)
11 XCTAssertEqual(kinds, [
12 .keyword("SELECT"), .whitespace, .star, .whitespace,
13 .keyword("FROM"), .whitespace, .identifier("t"),
14 ])
15 }
17 func testTokenizesQuotedIdentifier() {
18 let tokens = SQLTokenizer.tokenize(#"SELECT "my col" FROM t"#)
19 let kinds = tokens.map(\.kind)
20 XCTAssertEqual(kinds, [
21 .keyword("SELECT"), .whitespace, .quotedIdentifier("my col"),
22 .whitespace, .keyword("FROM"), .whitespace, .identifier("t"),
23 ])
24 }
26 func testTokenizesDoubledQuoteEscape() {
27 let tokens = SQLTokenizer.tokenize(#"SELECT "a""b" FROM t"#)
28 let ids = tokens.compactMap { token -> String? in
29 if case .quotedIdentifier(let v) = token.kind { return v }
30 return nil
31 }
32 XCTAssertEqual(ids, [#"a"b"#])
33 }
35 func testTokenizesStringLiteral() {
36 let tokens = SQLTokenizer.tokenize("SELECT 'hello' FROM t")
37 let strings = tokens.compactMap { token -> String? in
38 if case .stringLiteral(let v) = token.kind { return v }
39 return nil
40 }
41 XCTAssertEqual(strings, ["hello"])
42 }
44 func testTokenizesEscapedStringLiteral() {
45 let tokens = SQLTokenizer.tokenize("SELECT 'it''s' FROM t")
46 let strings = tokens.compactMap { token -> String? in
47 if case .stringLiteral(let v) = token.kind { return v }
48 return nil
49 }
50 XCTAssertEqual(strings, ["it's"])
51 }
53 func testTokenizesLineComment() {
54 let tokens = SQLTokenizer.tokenize("SELECT * -- get all\nFROM t")
55 let kinds = tokens.map(\.kind)
56 XCTAssertTrue(kinds.contains(.lineComment))
57 // FROM should still be tokenized as a keyword
58 XCTAssertTrue(kinds.contains(.keyword("FROM")))
59 }
61 func testTokenizesBlockComment() {
62 let tokens = SQLTokenizer.tokenize("SELECT /* cols */ * FROM t")
63 let kinds = tokens.map(\.kind)
64 XCTAssertTrue(kinds.contains(.blockComment))
65 XCTAssertTrue(kinds.contains(.star))
66 }
68 func testTokenizesNumber() {
69 let tokens = SQLTokenizer.tokenize("LIMIT 100")
70 let nums = tokens.compactMap { token -> String? in
71 if case .number(let v) = token.kind { return v }
72 return nil
73 }
74 XCTAssertEqual(nums, ["100"])
75 }
77 func testTokenizesNumberWithExponent() {
78 let tokens = SQLTokenizer.tokenize("SELECT 1.5e-3")
79 let nums = tokens.compactMap { token -> String? in
80 if case .number(let v) = token.kind { return v }
81 return nil
82 }
83 XCTAssertEqual(nums, ["1.5e-3"])
84 }
86 func testTokenizesOperators() {
87 let tokens = SQLTokenizer.tokenize("a <> b")
88 let others = tokens.compactMap { token -> String? in
89 if case .other(let v) = token.kind { return v }
90 return nil
91 }
92 XCTAssertEqual(others, ["<", ">"])
93 }
95 func testKeywordsAreCaseInsensitive() {
96 let tokens = SQLTokenizer.tokenize("select distinct from where")
97 let keywords = tokens.compactMap { token -> String? in
98 if case .keyword(let v) = token.kind { return v }
99 return nil
100 }
101 // All stored uppercased
102 XCTAssertEqual(keywords, ["SELECT", "DISTINCT", "FROM", "WHERE"])
103 }
105 func testNonKeywordIdentifier() {
106 let tokens = SQLTokenizer.tokenize("SELECT mycol FROM t")
107 let ids = tokens.compactMap { token -> String? in
108 if case .identifier(let v) = token.kind { return v }
109 return nil
110 }
111 XCTAssertEqual(ids, ["mycol", "t"])
112 }
114 // MARK: - selectColumnListRange
116 func testSimpleSelectStar() {
117 let sql = "SELECT * FROM table1"
118 let range = SQLTokenizer.selectColumnListRange(in: sql)
119 XCTAssertNotNil(range)
120 XCTAssertEqual(String(sql[range!]), "*")
121 }
123 func testSimpleColumnList() {
124 let sql = "SELECT a, b, c FROM table1"
125 let range = SQLTokenizer.selectColumnListRange(in: sql)
126 XCTAssertNotNil(range)
127 XCTAssertEqual(String(sql[range!]), "a, b, c")
128 }
130 func testSelectDistinct() {
131 let sql = "SELECT DISTINCT a, b FROM table1"
132 let range = SQLTokenizer.selectColumnListRange(in: sql)
133 XCTAssertNotNil(range)
134 XCTAssertEqual(String(sql[range!]), "a, b")
135 }
137 func testCaseInsensitiveKeywords() {
138 let sql = "select a, b from table1"
139 let range = SQLTokenizer.selectColumnListRange(in: sql)
140 XCTAssertNotNil(range)
141 XCTAssertEqual(String(sql[range!]), "a, b")
142 }
144 func testQuotedColumnNames() {
145 let sql = #"SELECT "my col", "other" FROM table1"#
146 let range = SQLTokenizer.selectColumnListRange(in: sql)
147 XCTAssertNotNil(range)
148 XCTAssertEqual(String(sql[range!]), #""my col", "other""#)
149 }
151 func testSubqueryInSelect() {
152 let sql = "SELECT (SELECT count(*) FROM other), col FROM main"
153 let range = SQLTokenizer.selectColumnListRange(in: sql)
154 XCTAssertNotNil(range)
155 // Should capture everything up to the top-level FROM
156 XCTAssertEqual(String(sql[range!]), "(SELECT count(*) FROM other), col")
157 }
159 func testStringLiteralContainingFROM() {
160 let sql = "SELECT 'from' AS label FROM table1"
161 let range = SQLTokenizer.selectColumnListRange(in: sql)
162 XCTAssertNotNil(range)
163 XCTAssertEqual(String(sql[range!]), "'from' AS label")
164 }
166 func testFunctionCallsInSelect() {
167 let sql = "SELECT COALESCE(a, b), UPPER(c) FROM table1"
168 let range = SQLTokenizer.selectColumnListRange(in: sql)
169 XCTAssertNotNil(range)
170 XCTAssertEqual(String(sql[range!]), "COALESCE(a, b), UPPER(c)")
171 }
173 func testNestedSubqueries() {
174 let sql = "SELECT (SELECT MAX(x) FROM (SELECT 1 AS x FROM dual)) FROM t"
175 let range = SQLTokenizer.selectColumnListRange(in: sql)
176 XCTAssertNotNil(range)
177 XCTAssertEqual(String(sql[range!]), "(SELECT MAX(x) FROM (SELECT 1 AS x FROM dual))")
178 }
180 func testCommentBeforeFROM() {
181 let sql = "SELECT a, b /* columns */ FROM table1"
182 let range = SQLTokenizer.selectColumnListRange(in: sql)
183 XCTAssertNotNil(range)
184 // Column list should not include the trailing comment
185 XCTAssertEqual(String(sql[range!]), "a, b")
186 }
188 func testLineCommentContainingFROM() {
189 let sql = "SELECT a -- FROM fake\n, b FROM table1"
190 let range = SQLTokenizer.selectColumnListRange(in: sql)
191 XCTAssertNotNil(range)
192 XCTAssertEqual(String(sql[range!]), "a -- FROM fake\n, b")
193 }
195 func testNoFROMReturnsNil() {
196 let sql = "SELECT a, b, c"
197 let range = SQLTokenizer.selectColumnListRange(in: sql)
198 XCTAssertNil(range)
199 }
201 func testNoSELECTReturnsNil() {
202 let sql = "INSERT INTO t VALUES (1, 2)"
203 let range = SQLTokenizer.selectColumnListRange(in: sql)
204 XCTAssertNil(range)
205 }
207 func testEmptyStringReturnsNil() {
208 let range = SQLTokenizer.selectColumnListRange(in: "")
209 XCTAssertNil(range)
210 }
212 func testLeadingWhitespace() {
213 let sql = " SELECT a, b FROM table1"
214 let range = SQLTokenizer.selectColumnListRange(in: sql)
215 XCTAssertNotNil(range)
216 XCTAssertEqual(String(sql[range!]), "a, b")
217 }
219 func testFullQueryWithWhereOrderLimit() {
220 let sql = #"SELECT "p_timestamp", "msg" FROM "logs" WHERE level = 'error' ORDER BY "p_timestamp" DESC LIMIT 1000"#
221 let range = SQLTokenizer.selectColumnListRange(in: sql)
222 XCTAssertNotNil(range)
223 XCTAssertEqual(String(sql[range!]), #""p_timestamp", "msg""#)
224 }
226 func testReplacementRoundTrip() {
227 let sql = "SELECT a, b, c FROM table1 ORDER BY a"
228 guard let range = SQLTokenizer.selectColumnListRange(in: sql) else {
229 XCTFail("Expected range")
230 return
231 }
232 var modified = sql
233 modified.replaceSubrange(range, with: #""x", "y""#)
234 XCTAssertEqual(modified, #"SELECT "x", "y" FROM table1 ORDER BY a"#)
235 }
237 func testReplacementWithStar() {
238 let sql = #"SELECT "a", "b" FROM table1"#
239 guard let range = SQLTokenizer.selectColumnListRange(in: sql) else {
240 XCTFail("Expected range")
241 return
242 }
243 var modified = sql
244 modified.replaceSubrange(range, with: "*")
245 XCTAssertEqual(modified, "SELECT * FROM table1")
246 }
248 func testQuotedIdentifierContainingFROM() {
249 let sql = #"SELECT "from" FROM table1"#
250 let range = SQLTokenizer.selectColumnListRange(in: sql)
251 XCTAssertNotNil(range)
252 XCTAssertEqual(String(sql[range!]), #""from""#)
253 }
255 // MARK: - SQLErrorPosition.parse
257 func testParseDataFusionError() {
258 let msg = "Expected: an expression, found: FROM at Line: 1, Column 15"
259 let pos = SQLErrorPosition.parse(from: msg)
260 XCTAssertEqual(pos, SQLErrorPosition(line: 1, column: 15))
261 }
263 func testParseDataFusionErrorWithColonAfterColumn() {
264 let msg = "Error at Line: 2, Column: 10"
265 let pos = SQLErrorPosition.parse(from: msg)
266 XCTAssertEqual(pos, SQLErrorPosition(line: 2, column: 10))
267 }
269 func testParseReturnsNilForNoPosition() {
270 let msg = "Some generic error without position info"
271 XCTAssertNil(SQLErrorPosition.parse(from: msg))
272 }
274 func testParseReturnsNilForEmptyString() {
275 XCTAssertNil(SQLErrorPosition.parse(from: ""))
276 }
278 // MARK: - characterOffset
280 func testCharacterOffsetSingleLine() {
281 let sql = "SELECT * FROM t"
282 // Line 1, Column 8 → offset 7 (the '*')
283 XCTAssertEqual(SQLTokenizer.characterOffset(line: 1, column: 8, in: sql), 7)
284 }
286 func testCharacterOffsetMultiLine() {
287 let sql = "SELECT *\nFROM t"
288 // Line 2, Column 1 → offset 9 (the 'F' of FROM)
289 XCTAssertEqual(SQLTokenizer.characterOffset(line: 2, column: 1, in: sql), 9)
290 }
292 func testCharacterOffsetLineBeyondEnd() {
293 let sql = "SELECT *"
294 XCTAssertNil(SQLTokenizer.characterOffset(line: 3, column: 1, in: sql))
295 }
297 func testCharacterOffsetColumnBeyondEnd() {
298 let sql = "SELECT *"
299 // Column 100 on a short string
300 XCTAssertNil(SQLTokenizer.characterOffset(line: 1, column: 100, in: sql))
301 }
303 // MARK: - tokenRange
305 func testTokenRangeAtKeyword() {
306 let sql = "SELEC * FROM t"
307 // Offset 0 is at the start of "SELEC" (which is an identifier, not a keyword)
308 let range = SQLTokenizer.tokenRange(atOffset: 0, in: sql)
309 XCTAssertNotNil(range)
310 XCTAssertEqual((sql as NSString).substring(with: range!), "SELEC")
311 }
313 func testTokenRangeAtWhitespaceSnapsToNextToken() {
314 let sql = "SELECT * FROM t"
315 // Offset 6 is inside the whitespace between SELECT and *
316 let range = SQLTokenizer.tokenRange(atOffset: 7, in: sql)
317 XCTAssertNotNil(range)
318 XCTAssertEqual((sql as NSString).substring(with: range!), "*")
319 }
321 func testTokenRangeAtIdentifier() {
322 let sql = "SELECT * FROM mytable"
323 // Offset 14 is at 'm' of "mytable"
324 let range = SQLTokenizer.tokenRange(atOffset: 14, in: sql)
325 XCTAssertNotNil(range)
326 XCTAssertEqual((sql as NSString).substring(with: range!), "mytable")
327 }
329 // MARK: - errorHighlightRange (end-to-end)
331 func testErrorHighlightRangeTypoInKeyword() {
332 let sql = "SELEC * FROM t"
333 // DataFusion would report Line: 1, Column 1 for SELEC
334 let range = SQLTokenizer.errorHighlightRange(line: 1, column: 1, in: sql)
335 XCTAssertNotNil(range)
336 XCTAssertEqual((sql as NSString).substring(with: range!), "SELEC")
337 }
339 func testErrorHighlightRangeMultiLine() {
340 let sql = "SELECT *\nFRO t"
341 // Line 2, Column 1 → "FRO"
342 let range = SQLTokenizer.errorHighlightRange(line: 2, column: 1, in: sql)
343 XCTAssertNotNil(range)
344 XCTAssertEqual((sql as NSString).substring(with: range!), "FRO")
345 }
347 func testErrorHighlightRangeInvalidPosition() {
348 let sql = "SELECT * FROM t"
349 XCTAssertNil(SQLTokenizer.errorHighlightRange(line: 99, column: 1, in: sql))
350 }