@@ -0,0 +1,254 @@
1+import XCTest
2+@testable import ParseableViewer
3+
4+final class SQLTokenizerTests: XCTestCase {
5+
6+ // MARK: - Tokenizer basics
7+
8+ func testTokenizesSimpleSelect() {
9+ let tokens = SQLTokenizer.tokenize("SELECT * FROM t")
10+ let kinds = tokens.map(\.kind)
11+ XCTAssertEqual(kinds, [
12+ .keyword("SELECT"), .whitespace, .star, .whitespace,
13+ .keyword("FROM"), .whitespace, .identifier("t"),
14+ ])
15+ }
16+
17+ func testTokenizesQuotedIdentifier() {
18+ let tokens = SQLTokenizer.tokenize(#"SELECT "my col" FROM t"#)
19+ let kinds = tokens.map(\.kind)
20+ XCTAssertEqual(kinds, [
21+ .keyword("SELECT"), .whitespace, .quotedIdentifier("my col"),
22+ .whitespace, .keyword("FROM"), .whitespace, .identifier("t"),
23+ ])
24+ }
25+
26+ func testTokenizesDoubledQuoteEscape() {
27+ let tokens = SQLTokenizer.tokenize(#"SELECT "a""b" FROM t"#)
28+ let ids = tokens.compactMap { token -> String? in
29+ if case .quotedIdentifier(let v) = token.kind { return v }
30+ return nil
31+ }
32+ XCTAssertEqual(ids, [#"a"b"#])
33+ }
34+
35+ func testTokenizesStringLiteral() {
36+ let tokens = SQLTokenizer.tokenize("SELECT 'hello' FROM t")
37+ let strings = tokens.compactMap { token -> String? in
38+ if case .stringLiteral(let v) = token.kind { return v }
39+ return nil
40+ }
41+ XCTAssertEqual(strings, ["hello"])
42+ }
43+
44+ func testTokenizesEscapedStringLiteral() {
45+ let tokens = SQLTokenizer.tokenize("SELECT 'it''s' FROM t")
46+ let strings = tokens.compactMap { token -> String? in
47+ if case .stringLiteral(let v) = token.kind { return v }
48+ return nil
49+ }
50+ XCTAssertEqual(strings, ["it's"])
51+ }
52+
53+ func testTokenizesLineComment() {
54+ let tokens = SQLTokenizer.tokenize("SELECT * -- get all\nFROM t")
55+ let kinds = tokens.map(\.kind)
56+ XCTAssertTrue(kinds.contains(.lineComment))
57+ // FROM should still be tokenized as a keyword
58+ XCTAssertTrue(kinds.contains(.keyword("FROM")))
59+ }
60+
61+ func testTokenizesBlockComment() {
62+ let tokens = SQLTokenizer.tokenize("SELECT /* cols */ * FROM t")
63+ let kinds = tokens.map(\.kind)
64+ XCTAssertTrue(kinds.contains(.blockComment))
65+ XCTAssertTrue(kinds.contains(.star))
66+ }
67+
68+ func testTokenizesNumber() {
69+ let tokens = SQLTokenizer.tokenize("LIMIT 100")
70+ let nums = tokens.compactMap { token -> String? in
71+ if case .number(let v) = token.kind { return v }
72+ return nil
73+ }
74+ XCTAssertEqual(nums, ["100"])
75+ }
76+
77+ func testTokenizesNumberWithExponent() {
78+ let tokens = SQLTokenizer.tokenize("SELECT 1.5e-3")
79+ let nums = tokens.compactMap { token -> String? in
80+ if case .number(let v) = token.kind { return v }
81+ return nil
82+ }
83+ XCTAssertEqual(nums, ["1.5e-3"])
84+ }
85+
86+ func testTokenizesOperators() {
87+ let tokens = SQLTokenizer.tokenize("a <> b")
88+ let others = tokens.compactMap { token -> String? in
89+ if case .other(let v) = token.kind { return v }
90+ return nil
91+ }
92+ XCTAssertEqual(others, ["<", ">"])
93+ }
94+
95+ func testKeywordsAreCaseInsensitive() {
96+ let tokens = SQLTokenizer.tokenize("select distinct from where")
97+ let keywords = tokens.compactMap { token -> String? in
98+ if case .keyword(let v) = token.kind { return v }
99+ return nil
100+ }
101+ // All stored uppercased
102+ XCTAssertEqual(keywords, ["SELECT", "DISTINCT", "FROM", "WHERE"])
103+ }
104+
105+ func testNonKeywordIdentifier() {
106+ let tokens = SQLTokenizer.tokenize("SELECT mycol FROM t")
107+ let ids = tokens.compactMap { token -> String? in
108+ if case .identifier(let v) = token.kind { return v }
109+ return nil
110+ }
111+ XCTAssertEqual(ids, ["mycol", "t"])
112+ }
113+
114+ // MARK: - selectColumnListRange
115+
116+ func testSimpleSelectStar() {
117+ let sql = "SELECT * FROM table1"
118+ let range = SQLTokenizer.selectColumnListRange(in: sql)
119+ XCTAssertNotNil(range)
120+ XCTAssertEqual(String(sql[range!]), "*")
121+ }
122+
123+ func testSimpleColumnList() {
124+ let sql = "SELECT a, b, c FROM table1"
125+ let range = SQLTokenizer.selectColumnListRange(in: sql)
126+ XCTAssertNotNil(range)
127+ XCTAssertEqual(String(sql[range!]), "a, b, c")
128+ }
129+
130+ func testSelectDistinct() {
131+ let sql = "SELECT DISTINCT a, b FROM table1"
132+ let range = SQLTokenizer.selectColumnListRange(in: sql)
133+ XCTAssertNotNil(range)
134+ XCTAssertEqual(String(sql[range!]), "a, b")
135+ }
136+
137+ func testCaseInsensitiveKeywords() {
138+ let sql = "select a, b from table1"
139+ let range = SQLTokenizer.selectColumnListRange(in: sql)
140+ XCTAssertNotNil(range)
141+ XCTAssertEqual(String(sql[range!]), "a, b")
142+ }
143+
144+ func testQuotedColumnNames() {
145+ let sql = #"SELECT "my col", "other" FROM table1"#
146+ let range = SQLTokenizer.selectColumnListRange(in: sql)
147+ XCTAssertNotNil(range)
148+ XCTAssertEqual(String(sql[range!]), #""my col", "other""#)
149+ }
150+
151+ func testSubqueryInSelect() {
152+ let sql = "SELECT (SELECT count(*) FROM other), col FROM main"
153+ let range = SQLTokenizer.selectColumnListRange(in: sql)
154+ XCTAssertNotNil(range)
155+ // Should capture everything up to the top-level FROM
156+ XCTAssertEqual(String(sql[range!]), "(SELECT count(*) FROM other), col")
157+ }
158+
159+ func testStringLiteralContainingFROM() {
160+ let sql = "SELECT 'from' AS label FROM table1"
161+ let range = SQLTokenizer.selectColumnListRange(in: sql)
162+ XCTAssertNotNil(range)
163+ XCTAssertEqual(String(sql[range!]), "'from' AS label")
164+ }
165+
166+ func testFunctionCallsInSelect() {
167+ let sql = "SELECT COALESCE(a, b), UPPER(c) FROM table1"
168+ let range = SQLTokenizer.selectColumnListRange(in: sql)
169+ XCTAssertNotNil(range)
170+ XCTAssertEqual(String(sql[range!]), "COALESCE(a, b), UPPER(c)")
171+ }
172+
173+ func testNestedSubqueries() {
174+ let sql = "SELECT (SELECT MAX(x) FROM (SELECT 1 AS x FROM dual)) FROM t"
175+ let range = SQLTokenizer.selectColumnListRange(in: sql)
176+ XCTAssertNotNil(range)
177+ XCTAssertEqual(String(sql[range!]), "(SELECT MAX(x) FROM (SELECT 1 AS x FROM dual))")
178+ }
179+
180+ func testCommentBeforeFROM() {
181+ let sql = "SELECT a, b /* columns */ FROM table1"
182+ let range = SQLTokenizer.selectColumnListRange(in: sql)
183+ XCTAssertNotNil(range)
184+ // Column list should not include the trailing comment
185+ XCTAssertEqual(String(sql[range!]), "a, b")
186+ }
187+
188+ func testLineCommentContainingFROM() {
189+ let sql = "SELECT a -- FROM fake\n, b FROM table1"
190+ let range = SQLTokenizer.selectColumnListRange(in: sql)
191+ XCTAssertNotNil(range)
192+ XCTAssertEqual(String(sql[range!]), "a -- FROM fake\n, b")
193+ }
194+
195+ func testNoFROMReturnsNil() {
196+ let sql = "SELECT a, b, c"
197+ let range = SQLTokenizer.selectColumnListRange(in: sql)
198+ XCTAssertNil(range)
199+ }
200+
201+ func testNoSELECTReturnsNil() {
202+ let sql = "INSERT INTO t VALUES (1, 2)"
203+ let range = SQLTokenizer.selectColumnListRange(in: sql)
204+ XCTAssertNil(range)
205+ }
206+
207+ func testEmptyStringReturnsNil() {
208+ let range = SQLTokenizer.selectColumnListRange(in: "")
209+ XCTAssertNil(range)
210+ }
211+
212+ func testLeadingWhitespace() {
213+ let sql = " SELECT a, b FROM table1"
214+ let range = SQLTokenizer.selectColumnListRange(in: sql)
215+ XCTAssertNotNil(range)
216+ XCTAssertEqual(String(sql[range!]), "a, b")
217+ }
218+
219+ func testFullQueryWithWhereOrderLimit() {
220+ let sql = #"SELECT "p_timestamp", "msg" FROM "logs" WHERE level = 'error' ORDER BY "p_timestamp" DESC LIMIT 1000"#
221+ let range = SQLTokenizer.selectColumnListRange(in: sql)
222+ XCTAssertNotNil(range)
223+ XCTAssertEqual(String(sql[range!]), #""p_timestamp", "msg""#)
224+ }
225+
226+ func testReplacementRoundTrip() {
227+ let sql = "SELECT a, b, c FROM table1 ORDER BY a"
228+ guard let range = SQLTokenizer.selectColumnListRange(in: sql) else {
229+ XCTFail("Expected range")
230+ return
231+ }
232+ var modified = sql
233+ modified.replaceSubrange(range, with: #""x", "y""#)
234+ XCTAssertEqual(modified, #"SELECT "x", "y" FROM table1 ORDER BY a"#)
235+ }
236+
237+ func testReplacementWithStar() {
238+ let sql = #"SELECT "a", "b" FROM table1"#
239+ guard let range = SQLTokenizer.selectColumnListRange(in: sql) else {
240+ XCTFail("Expected range")
241+ return
242+ }
243+ var modified = sql
244+ modified.replaceSubrange(range, with: "*")
245+ XCTAssertEqual(modified, "SELECT * FROM table1")
246+ }
247+
248+ func testQuotedIdentifierContainingFROM() {
249+ let sql = #"SELECT "from" FROM table1"#
250+ let range = SQLTokenizer.selectColumnListRange(in: sql)
251+ XCTAssertNotNil(range)
252+ XCTAssertEqual(String(sql[range!]), #""from""#)
253+ }
254+}