From 6a8cc166173b0a34cae75b0bf7c8b33776bec098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Wed, 8 Mar 2023 16:05:38 -0300 Subject: [PATCH 01/17] Adding highlight files for Clojure --- .../grammars/tree-sitter-clojure.cson | 43 +++++++++++++++++ .../language-clojure/grammars/ts/folds.scm | 0 .../language-clojure/grammars/ts/grammar.wasm | Bin 0 -> 89688 bytes .../grammars/ts/highlights.scm | 45 ++++++++++++++++++ .../language-clojure/grammars/ts/indents.scm | 0 .../language-clojure/spec/fixtures/tokens.clj | 36 ++++++++++++++ .../language-clojure/spec/tokenizer-spec.js | 18 +++++++ 7 files changed, 142 insertions(+) create mode 100644 packages/language-clojure/grammars/tree-sitter-clojure.cson create mode 100644 packages/language-clojure/grammars/ts/folds.scm create mode 100755 packages/language-clojure/grammars/ts/grammar.wasm create mode 100644 packages/language-clojure/grammars/ts/highlights.scm create mode 100644 packages/language-clojure/grammars/ts/indents.scm create mode 100644 packages/language-clojure/spec/fixtures/tokens.clj create mode 100644 packages/language-clojure/spec/tokenizer-spec.js diff --git a/packages/language-clojure/grammars/tree-sitter-clojure.cson b/packages/language-clojure/grammars/tree-sitter-clojure.cson new file mode 100644 index 000000000..e6bb9771b --- /dev/null +++ b/packages/language-clojure/grammars/tree-sitter-clojure.cson @@ -0,0 +1,43 @@ +'name': 'Clojure' +'scopeName': 'source.clojure' +'type': 'tree-sitter-2' +'fileTypes': [ + 'boot' + 'clj' + 'clj.hl' + 'cljc' + 'cljs' + 'cljs.hl' + 'cljx' + 'clojure' + 'edn' + 'org' + 'bb' + 'joke' + 'joker' +] +'firstLineMatch': '''(?x) + # Hashbang + ^\\#!.*(?:\\s|\\/) + boot + (?:$|\\s) + | + # Modeline + (?i: + # Emacs + -\\*-(?:\\s*(?=[^:;\\s]+\\s*-\\*-)|(?:.*?[;\\s]|(?<=-\\*-))mode\\s*:\\s*) + clojure(script)? + (?=[\\s;]|(?]?\\d+|m)?|\\sex)(?=:(?=\\s*set?\\s[^\\n:]+:)|:(?!\\s*set?\\s))(?:(?:\\s|\\s*:\\s*)\\w*(?:\\s*=(?:[^\\n\\\\\\s]|\\\\.)*)?)*[\\s:](?:filetype|ft|syntax)\\s*= + clojure + (?=\\s|:|$) + ) +''' +treeSitter: + grammar: 'ts/grammar.wasm' + syntaxQuery: 'ts/highlights.scm' + # localsQuery: 'ts/locals.scm' + foldsQuery: 'ts/folds.scm' + indentsQuery: 'ts/indents.scm' diff --git a/packages/language-clojure/grammars/ts/folds.scm b/packages/language-clojure/grammars/ts/folds.scm new file mode 100644 index 000000000..e69de29bb diff --git a/packages/language-clojure/grammars/ts/grammar.wasm b/packages/language-clojure/grammars/ts/grammar.wasm new file mode 100755 index 0000000000000000000000000000000000000000..e50647edd97d894898f29d82b832cefa381a3215 GIT binary patch literal 89688 zcmeHw378ed)pk|iS%B#~42#Onun35Vh=_bsu_3dH$4(9uw|H(9`Z=X6`MEG8k6G(LCW%jOIgzjIJ6ny1H`slp!O} zs~SFK^n@x-BPR-(YFO%dsRC*Lii!XvBT9;YQP60}kf{JXWcbi=UUjALfMUet2}4E=tr{_|GUYYX35QOcIPStBBZf{HI;6UC z+=Stig{)sJMF&|TQw=`8QkE8oGFft?FFJ_Kob(u+*S{pk^!A9Kr}(XD%`Q;vg8Pj4 z2E{+PE~Vv*Ir?KpzEttcerxz;#IxoVoYY9}R_uZzt!R~E*V}{-Dt^Pm1zP?H%kMY* zYQ=AT!l-)^yyT3w0u4&?REbWyjrnI9XnqItJBl^GllhlT{F#rd^gHTl`E2H2_ccF< z`4uMqJm%LK{R^1?(8zCKev9E3Gyj~?x0Ly3O@7OmzpB2@Zw2!+4SzTDe>C~6V*X~M z|3T*eZ1_i*UvK!;%)e#wf0Fq(48Ml?zZ?5J$NZOuU#sIAee0Ni-SF#~|C`abf%#2F z{xonSaU1w=w^g;de0qxhc<1=I=A| znX6U(e`)MHoB7ubKZp5m4L^_h?@awHVE!&spEoeS)$og%|BXq%l=+p0U&j2MhF`(_ z0+ar3=I0uI74sVn{~+^k8vYUHZ#DW>Gr!pIPcna-iNA*Vn+*RP^IHtRmid{6U&s7h zqi;R)ml}Qp^Zz#d+swaY(r;w`USq$_%wK2tEjqs8KVkj>ll}|l-!k&8%s*}TZOs49 z@H?2_V)&iR-(vWg=!DRJJZk)DHuEbCKZp4rjK9od{$s;0V1B-dzk>N2jr?xrHyV4a zV*Vq;Kgj&^CjKML-);ES%s*oICz*fN@N1a=+UR?Z`F|LHU(5W}CjL6+uT^|WJsJfz z`(sM(8Z8TFl*@LhjA$oQ{mMlbPqgzgsdk0sf|zEFq-a+JVS(50zsG+qP&W&dr!qhO zn3`02rxemFEGjPX8sTCR|H$UEg-@@{#LTq*CBtK@z1e))iWP(CCdmcNmY z$VcU4@^Se)`J{YWu945kXXPK{bMlY!dHE-~R=yx#lrPD3@@4soywxiyFG}C$1ua0-IcYYo!gsICqeo2fq#nEOI@JB5@L^iO*}%Ev`R28ZY} z;Swb({$lLaLB2_B`)K4=@%IK$TTy z|BCd%D$DdOUb)vF+V-ia#<|DXOoM^=-6+Opyn)r#>6^XGl}J`D+Z7=A-zf2Ip1%~z zl&Ouei{+F>+SwVWKm-zrO5c;k-HZAenZ%q4wOzY#)zw$D&ykJV^(wQ^Yq$Z`8c>b>d zjUv+pB5oFA+k2Td$3aIqih$UjiaoPvQh$W`H>p_dD^L%gQ!11UsWu|>KO<^?$^ZrR zZ`LM1k}Bjz*h7`01B`tOjO!iHlgm<(K9CYE_Iw$rkAtP`OQlt5RE;e3q`a1WiTvSO zm1+I4nZkkTMW_y{1wT*=2x&DI1ui=wBe=Zpt|^$*lIrWD#C*>6mym?~|N zt~4)m*Ba&9TzhlYpjJ~Sp-%n9D$93~JCff`;izrOoMX=1jEdIQyn(9U%TQralb$EK zqmq{+P%D8erO0RiT{18|pYVT=YI;qo@+yz)Ip3S_9W>wTt;}?2MY;zfQOW0e{xzsD zav5XM!bKkLJ+EWykUO4`UZ5H~lJwMQ~%JOEO1orCcEFJPCX7uNtszi^`K ztrMxP0tiv zf~B~Y^pLxku)klgG9l1p4V0mq!U;6m*ND*P+C zGeIP6x(;Nz56RPSP}w8f{UD{*b`^Oou3@r$*kspwnU4?^VJ?(;StxU0GUPPRnyg@8 z`kIi*V8BSzyUz3Hs}vom>8&R~*AZV<*grjoQUh35{1C7!fXVbgsK^DT-1}3~b(9qK z#YmO+mKF6!lOjZ!Qj;$89uxuQ)d>88%EW&DK0>eY{J99-!v=nUzJEi{}0^8caZ3z6A z^mibzl@0tS0>793w+P(R27Zgc@1(ySfgN;DvG3T@)OS+gUrA6{|4d8GTAZk8MOZ*^oswK+FRaJ0pR_w5&H$w9*w=tiM>tw|3K`H zIU|4LME*wlTM@Za&d6Uok-wJy-x2wxD2^6D{f}Vm76ZZiUm^AuYZGc}Tc!Uc0_RW{ z&us^Rf0zCj2>cQ;J?Cbn+{2`1Y-!k9<3h>Z;@bA)V(xeIUZ6a_K6@%fwdi5KZV8}AM!_&d~4NCmR z>BjU|+Bh(TZ;_45-h=Z&IW?G4{L*x=QD!EMOJvmR=zLz2L$Nke5PH?WNHx#%-#{waPL^Y$ai}aP7j*OLHxh!n zUgsjO6#Nm_PL1+vpN;1Ktn{C*l>wfWYGgIGMqYuO@6iT8 zyQPeu)t1(UXH!2T{iot``cu%<&xF1C-cU|=8B=Gy`EnXuo2-is?N+3H1M14jOJpn7LhnSZCv ztjkSiJ}LdU^qg;IJy-2F*cp$$PC#4^d{=+C)A%^UsG^!N3BX)tBm2;x9!t{O8 ze^6yvQZ+HDdAe zq~nkXb09oG`*AaF` zR0T5)|1k3x%*pEYPv0*yb2T!=*hIGt_sfcVbCi`8-0Uj2kDaTxYA^3%y1i2R_vK7p zR=vh{LpclbATOb6Tfo(Z5ONVz**(%0e9#&K!zS-6j3gm%H}ALz<>d8n|gM zm;N2OHTeD07pq%%f~j-mG`CBCMQoajOdjDh%cTG79BH&(^L(od-H|jqE`4FS%oO7> z2z1bm$@Mdr)x-1R3;kvET-P>^lG`O87u_x|!e6$D;O6v%^qKwz{4AOZ;4iOz?&opt zGeVi4=ie5}yx}q|uh20S|MXu3z?2d+Gw#v++j9UwMWy?vCsAEjq%XnG7id(MNj&b< ziBRAit(y`}4KafsG< z06bJhqgz0FbU&V=p|xt5x7I4hF`VN?;T*^Kmm+l_$59+j*S*PclpoWC$d_IJ83`5e zpJ|~Q9O*C7%L4yI*VjOvOMgow{w?8(Lx&6ZD?B0n3x7ljR-<&`Zq8A-7{49L`Cbsp zcc{NO{rw;@I(xo*Yn|;P^HzVCPn2vCxse2q9>(1=L~(qsSP?uD=v5HGPpkxFmCsDg}>$ zYACxAtCVys2B|^N8O+Y~r{zHPk+udGw5*s$$wdl&y)gVwLlHB52VijZZ_E)BjSf$! z$7Mc?B)CrcH^8myW2W#kf>J-6gBeXVl*072JYY9K{0u#FHx2DvuvmNIiHv#s4N(S12C;BebZrJg&aLz<%zXG;Gv zrLi;=k;*_*&u36>|GW~Lj>md*|9@bEWG4qR^n@g$2Wv4<^n*5_>d9%1IRtmQ>Xk(% zk{IhKNcmlkz@JTcGLpV0+WV(9JCT zJWQZ+)CoNQm8#~ZqRn9=#i%0R(<#yz^v}SP@O{F>yNS~V_^<)R^ zj*FIp?1P(FgI?(){8hygn{-0>iv#!z3bAwN7LD^}`W;)6xs86ulw=mu@8}YBkLiyp z$>288KbLQ4*dfa5rHA_?qi75YAkS`6QMTKM^KMs>KAnnHfqeyCsDR3tol>1kG6X`E z5*ZQ@b&XsJjJh7FfT+vLXx0el846`5hw;UtO{ zZ65ZKn{-6d4zz19K%ULd9-N4g8O2$c07mqoW5vqDVVN9i2m3>4owv|0Ey=XRPnuG< z45Nu;at2M$S|N4`#cs8W*r)kt(wcXnU!P*P-bL(R_@|d-+8}l@Ej1VVMU<%RE)t#M z|Dq&Qj@ShynRfWWeio5wA5OF)YSWYaQ%W)d@iAxf+7%*EdHC6m-?QYXOLxwk`R-;> z?%g-%D0$h*=`Q{O(ZEiAj}miNQh?i=uKvCuzLVeCx?OSo0VSDk@a=&mnC;7obZfuA zzH06tO;-h2kE98tk~Z*R^ zow*qUR3HCPqy1nQ*xU->2B|r3gTj6JZ8JCTg?&)z-hu3C2awMG13)GCf$V$xhvdrs zAocu1$5q)Mdv?mg&79nze4DXstcZotmh;iF}{BF76bg{XT zB68a^tu&`%Z!E_pUMfPQHTbT4V1>_IqP|vTYfi1`Tb{yzkidh1ko6s~%(9 z7vV{HwO*{&@^|$PSiRy7XF}^@Z|h$TH)BKXN1Fv{CmbHkgnY7y8etbzT-vy$`YL7C zlhWuKX{>(-O|iVNG}T3Rp|)&VsOfbOVk}K10P7aDfh`q;%F-QCJTrYOC`X}AN_X%( zMN7Q@SnNwoSY8DIb%?xT%gfbE<;~WkEiV-emM<(V6rS+9U@z9&)Nd-w#P?2Aq#R|h zAE{<4XtwO;S!#QKZ%Ur#x}lA(=nh(@G4|-?k@~v2u_VyVllY_kocNgwW#+2>5i12eo|g9W>#NL z(`m}YSDnqFq!I&A+kvz#)-eGEnHzA_XQq{k#0B~{c-rY)t?X3GayI^u)^saCLzdr2n|l|g4ff(KapwS4ZmY4TrMRlqw4i$nfbyI3 z&N0l`l!1M}gx29OB~y!felwa0r7*sz1xJ6+619Hm@4>N@$#>B{FdssmJSV_vQ9BG! z#y7$vP!qo_7xu>5b_#Z5zhQ}5^-^1ncagK&SgLb|lg8& zF|yVjm+^o?obayqIEiQM)ZrI+bO*&_}nTT0nscDk+DVC&5nGs)fEd(;c8O zj}^3qt6rl*;foScG8655210*Sd$2@iU)qX=DLenDrNoAms@*Z_r&|TCVeV?EB}0nL zm21R_1*BmUZ!H(PQW2uNaFuZ78o?TrOF{=4OEn5zQt}lcK;I~^o1b&-9Kujfas8E+ zQr6L)P_#QH3!>{Wcnv`Cih|(1%_7sSLZvyBf7YOr5%fw&reI(;AyA;5t5R^LKX&8a zof-=^94jq9YWU=8K+iktmE>xaOzE%&z$O!PW}Gfq+;-?1>c=#?In``<_TXK_SxOL_5Y`9JraG%*(H*KObiCWrV%JaWxj%KA4Hp0ZR-4`ZCEq{PTm9cH`8-Vf!{czxy_I&?cE@0| zFrJQ4+wFP`PF;>^Y5Sva*QL8}+hHpou6dk_YXn$^`Dt4~$)<-3B;MvO!PicN`jbN9 z-Ytc50jLmH^>F1QtQ1rRYK-tEpr#0K26;=wDi=kf6Zo!>cSG3zpq`+Ez#oj?zKC-q zevd+&3dB7gzbAtRfli0)EXd9Q4afB;Q7k6lcM9T7gM2!|FGbiKgk1w#2%}JjF+tM0!a&CB-BCBr|y#ImeHN zQ#{yC;VQ;2=ZpJc=;FNjn(mQ{kRREbaZtX59lbKbG!~L;T%N2Cu0-<<_&Ovf8ONvk zW82h27^lgGfma9PNdX60BW3esJt>`UO6SWw=gV;$KuZZykv6s$>tdTx{A`}}1nE)n zL*YgT@(+bmeCQ5^QC@_Z(~yT2gyis5okuv_#1GiQA^93Mq zAV=UcMbGP>$w+6Xv_-EuBzPuNwfm>=W>3?_CE_A6vAS~P=nKT~3FF6CR!tGrmFHGo zFl5~5DFRj zFRTjjl}Kjdj;N$!>V&KYUrCxcZuD>yICkm?24n)OD~FC4GJHZ62eUQKuNr^iCoLD(@$^j}16vFZ$ z8`w}&ARE9qvMf8tkSPjbU zQuG0Z=!5HHoJ(VzYbKgwRBI(#i?*V@7?f%Z-P9u&px-IM7}yA7U=xggdx$;7UZRC) zBg#cP$Oos2p(U%My*M+~T&GPTbs=)dt$}lEgZx<2IT1}~=c#E#MANXmG^NBSSgTNs ziRc@dY8EJCZA(}c(K9YDT%M?oXquR(rt>43rsSzs=res z`Ud4yf3qWcX61#;6LTV(uEC0!ddOm{dxx8?B;>C!jwRviKC8B9vo|;~ZXj-4ArZ*#+ zHsq=4ortEl^VGB{qG@BEn%<9S+MK7Rk0P43K8AQ`7emO*`_`^g~3`&O9}F1@8TVD9G=Ap(vuMAWuyVBAV*usi`!g$hZ(bP0gO)Vmtn&qjfO+-_xJT>LIH_OkT+D9;z=ZUFPL{rB+HFb?>>XN6X?h#Gh z{y);xGg7)9d6uqsL{qOkHT8{X>XWCYBO;m(%Tv=)5l#K_)Kn4CbabAYj*VzKCQnT# zMl>Ctr>0XPnoiDB)2R_ngYwjLW<=BJd1^X4qG@oRnubL*os*}g-G^yp1k;E-G5ufE zG$vBIQF)fGDxzszo|>v7nkMF{>HLVMDS2v|7SVJ;o|>jdG+mshrrlS%SrJS#^Tc#T zMAPg%HO-A^nv^%nifSg-H@lIB@s=F3!165mL%&Xh2l2sVn~Ul z1+>QQiN079?kM&a{l$ui{^bRYBP%gYv5wP1ulSfXn8w1#+G?R_jB^9g0_Ub;8P4^@ za-8dnyH$=WaV{0Ba4r+~oUd4% zzr#+0&R8nj2TNuLBmEvIM;X?ZNMkdkX{-^n!u6hl_G}c2M{q6`kKx=*tX8=`fpZJ- zB+f0xQ#iL4YjAEOp2fMXcn;@s@jTA$#9Ex&i(8pKfZibIF6Yt~PQf$Gwjra)Xw&D|<%f)9n zw-aCBoZJt?xSFHaYK%On$83%Bp5iNLEfiaEE)`$n+)Ql4xw+Vma|^Kp=a%AooLh^X zIJXf$;M^9QvEky&I+Xx@$wxSg0a#4nJJJA^D{Cy$VU+pfNG%e(L z$1%3uhR2Uwea7y}drzJBp*HW`*=IC^t(%D!IAezj&TT{+oUuy<=W@{==XRna&dK`> z>?18Gz$}9H+O!vC1+=4-_B`?)nNq#W#>~uGBWKP@(pxkA!Rqze$FL|2^a zi*7hK5Z!Su6+Lh+6FqTmBzoc8RQwug_Yqrh?yl!F??c1B;scz!i4SqE5S_F|zmXp1 zWQT}#*0P1-5NJt>-a5vLf|SO8l8v*xpcLrh@u$RL&_VlXvZWu19Y$Pw+C3AW))ONF zsfML$YctQut+wfmt$79-wRP0W{ZO_B;wWY7KPy|G51Yo@`Uq{wkF~8|vUc>uvOrmH zO;FaQfpkl1ovvn+CrEcl^(k%+4yCFfkV*}}qNsz80knZSOr%QTD!jds6d|m zYMrMiW(RVcmB43?2;_HIg8b$L^1C8IetiS^^+}N5yg+_)6Xe%BkYBF^`7H?KH$Ops zJp=jmsC9m3oXKzga^bN&1@4F8E$0CTrmWB$*lyCF~~3lr2y*FgEY zBq(2}K)Q|z(zOqyD^HMadLZ4!3DUI*q-&KR-NHb+1+`9Rd^NxM-ydr+Cpy-d6TKGj z)fNHF&1#L=6MF{IHBFE%-!Zi`ke{C*zxjc5^Ae8<(Jbiv#&BN|4|Afpk+6r28R|ZfAmYa|7j@lc0Q81k%k; zkZx8W-OL2(z7LddM}qQg52V{x>vVBW>ObZQdhxnfK~|y&g+ouSWvu9!`+%vp~8}5~O=Dkna8j=~e~O ztxS;a?m)UbYn{%tx48St=(AdWBHmuLv;%5I)A~|w8^>lcM1G_ z_yU#lv>J2%LgoCW%6W{=`NcZt{93}C$LX9m)H&y0B+R)==lpt|bKW;$&J%Uce~RRs z)Wc?}oNMo4)jH?rY|be$1^qd_sa;>K5Scf$&HLH(RS)_$3B8xiD@OcAc0F+c(v^ve zaBdX4N<7{6UgsD3YVF2oX^9nSv+mR|wC)t#@xv`^7V=1m**G^B9|4b9^9b(#*$6+b z5Hof8N>RRj#D3WC^NuZPqr|yTtJ_C3$9X@sN9f=jyN)>DD|Eh3+uS@c*Oq-wVIyTl zT7ipSf!cjmn5WVF6INp@JRZQcz~Y);D_owq!KPSPs}v3O{x0sd;`VWIt(*V&d3@cvdA&M1BHyKY zO0I$Ey7Y35;ys(UuYH)km$e4hyk`e$G^lSY!Yf2BeZ00Fhp$^=d^PK9yPNf;W6I0x z`pL^XVmITvWjE`4Yd7ngzMJ(;O{lLB`*mqQ$6abHU7`IwyGwR?K|PF4**y|XG1l&? zO-)INVOMSHoERVDHnmb?{$rqx*4zfR)rfDE*0;9S`cAD;U-CAv-)`3Tr-b?n1dqY@ zYfHRkEpc{XacHj69Q(RC?t?n++cxfq*tljkmL1myC5U@+B(9mO4ocBC)jaHmp?PUj z%w3Dr+?BpER0_W>5RWP^e^_J9=D!?!)`N4f=J@6i&mV%b&fp&K)oRvBJIZp+QZ}jH zzC`bs3hD^%%X~uV;2oCHIe^i@`{08*qBwXbd7O@7@g#C86HnvZD92otb2`7UP}zWH z;>}?-7Hr9w7v7)#J1!C0;?U4Rsyyux)>qa!{8_f&jk#e7F?P|AcC=)Ptv;sYhh(_Pf_q&Noo@zaB^ zmE>zkv7a0AtYn*#?5Wc12uZdp$sS5lthI5!$ob4u5|a;0cBnKLDoJ4o0jDVmrD1F= zX^=zmz0&p%h5ARx2{q@lQ%NpRX$nHveo&HMD9O%H8c{^nnX4p4p)`!(bR`Lwhb3pk zN*079!SZ~dP_s6CuT$k+h;w6IGoC00huIa*#i-p9#icdJp$%1-C)gXpvPN3QF`Fn( ziaqe#5IOIqf1Bf%zNOa^ztn%X*5Pe6-%jy}qW`+!_XJQi=vSbpKp%kWLCt=kQ$SNe zw}Sov`UI55HwAlwP6J&Gx*ha9=yMQ#JMbXTnV=b<6`&VDUxD_(2Tc2dhJa>+?g705 z`Ucbtp9DP|Gz@elXcg$spzWZR_*P**P$g(C=mF3M(08CV_z>#PK%+p{fF1_D1=kQp!!XP*c+4qodvoObR*~?(957tL4|w3cAz6czXVMMT@QKy^b+V}P-;&h+JX)R zT?o1n^bqJ}(5Ilny`TehIOq(}d7$e+uYx`Y72_jO9Y9BdehHcix*qfZ=q1p{Ao}P_ zThO7P(?F9!*MjZ^{R#8|=toey7VrVk5YR=SUx9uDdKL6JsJJEa0v!qZC1@(>de8%) zFF_@(Q0JhZfrfxC1>FLA8uVAtx1c7iQ7@q5K<9#1fZhYOX#<_0#h^Dp{{ZcUS8I+1 zT?$$PdKvT)=toe`av?^8mVw>^Z3T5`C&VG3p`fck4}tyylI?}q2lPLn3qiMoUI2Xn zD(N7^fuLW4E(6^QdL8sB=ts~V9r4Xi(2<~DfTn_O0{ss32IynZ_n`FNLX?9J1XX~B zfT}^4gBFAC1w8?J8MGPn4XCISd=S(V)E{&@Xe?+JXd&o!(0!mMKz{;l0KEtL3iK~f zz0SY~Y7g2UbQowL=v2^f&_vKBpesT1K{tYyf_@EJ33>?hDCoDK--Dh3{Sov6DERkO zA^aS)BL)2Uy`VsdpBJI-aD6=JL(mq`(Fi*Vva3KdLH!YSGl=@1^`O%r{{`sp(DNne z4TQZ8ItyW&@cRzv3|tQey@%_Kpfhp(5`NzTeU3Eu;deaf=b)*e?~qSt&_vLI;Jbo4 zfy%%)01XD63TlHe>KohQx;bbB2!kTt-GP6AE&&|}{$9{^pqD`VgWdu?4f-8u3+NWm zLBP}z^aIj-3@XL7`*$P42ZKg}o(J6xdIa<=s2cPSl;vvtUJIgehTbTld3_PaG`jPm zudnj=SsUv0g6wx(=}i>(E#UT|gW5aUNxz-j74M34!;{(Wcu%*7H~?>;9f&t*4-&oc zcMp z?SFb3NZ1)&JbGlN;B_#}4G;3cYm?@`zge69U$6xYyZO{pK51%GUN7Kt!T2<_2=c*e zw?8OGTsMq(N)rq>JZ|4G)7U+KxUIvSB}!cpMnr%6fau!%QW*Ww(}nD@D?S@De8swU z&zYVF7GfqwYdG{*g0Jm21lI@$ZvODqlf5tp!yP9GH!pbOw;kqRdt>gk59VF_Va{~` z=39-tCWvj0Kgc`f-LeVxTr?3MgFXR$YNfa{%6VWGP1S|JgK+f0tnCQJWiVSC0Pa{E zCm`fx@pEyiI0OHU!WrUh&`>c#oGZqN@nWKwEY24fii^bz#H0WJ`l95w*2< zVry-8diF-Ku~vSxsYV-Ng?2ai5gvTuZ!?ZvwYB$m+19B4GOam(&&H!O{B6OpD@5c+ zAJu51rg%DEcSLyDhQCj6)CLjx(Py!KREGQQx+B8lM*P**v%jd(M!VCqzY<^V(nec% z=|{9;R(BiW5l8|Xef=M^(KfMdmo_5L-d-c3B0S5fyN&qKQf>W+JbOp1XYWowstuy= zcj-sF6VXnwvsShCLyfgoinYeN*BTZhFpc$aV1-Hs=a8N(z>%FFnb}C(95Otf|EI?O zA{jrc-MwRfJy{PnY9M_z%Po~^ zciBVsM0j7>SI*Z5+i1R=1O7_T)u1!AY(C{}myO~ORp8m~V95s~8Vp2(foL!g4F)13#gl7$oxQ@khA@aumL~(zDIw23MGnG| z#IpzB4raSP7}0jj7PeznyB#C>cFbyx6zi8c4uSM$R(q;E6^Kp`AvzspIx`WXf%t#l zzSRdK`WA@31)^_(=vyE%Qmkg>IOzY4GlC}z(MjSY@x4A6(f2^~JrI2lMBf9Ekz%bb$AOh-i>OLg z0g-8I+#kiawYdK1Y4NmeLyRbCTYFlB`=g1W)<02J<4Dr_ssFl4AB^ZKwEnBm`maLk zzY6WdNU?rrud`Rd{%DGv0vnwlLUg{o07nu;_(GewL-N6h?f{}Yfane&x&w%e6f64n zI(voD!1hPnXHN?unkFyCkpvOh=w5v=qI-epULd*`i0%aS|QD;0qGE%I|>UhCl(l9VI(k=+BUFIxJ+PM?yAwL_UTi z7oxbapFEpd109U$Vj#L0h%N@Ai-E{Uu@lK&XRo3*S{*{PT0VgzDWXcr2P3KkqDmmD z1fohHGE(eUve((GD557rqsWu;DIB>xJNL6%($`m-KSm_s*%F>D;n@(%EtXzvDmyKRjezZ>c(JOMjd`)hUZ=(L!;TS9VU_@hKqp`5j zSlDPRY-FU^H)5}|SES+ikRQDrszQ|lYB6u(}3tSAUX|*P6Hw%#cmIK zoxO_IniUff;zrutF|L`BcB9;gTH6$|(I&YWM-m&Ai*j+6CHU*efd5RZ3!XTB0s{B1W`MzPSD(s4n{N#h=u{tFd!NRL`I5z zgZ4UmC6L}A`b2&LL}u*gT8kh1<7%xd)`84O&WMtZ{au~;(PtqWeI~!ak;F#iN5l2O zh=#*P!(pT0u+ebX$Vjob%3f!$f}Z`A{0cVO8bY*HevKmuqKoh!pO2P&Frv{wG#ZFT z1JP(8GE(e>v)9=x?BTPuwoPsWBGVu7J#Bn{6xaGYU`Aki7e=pLYSw!J^Hn;xxd3Idur?zHh&5S7dJlpiy^}Kqp zQG<|;z~M+@qkS;TohtcYL{nj-sj$&h*k~$jWTe=2Xs@$Zw5Dx($neacZ4^KAugOMc z9?yu9&-{&zO1)CpsLTsKhb;3N<47_;nv8Yi`jQVuR3G!B`j{Wp$NZ>1=0`?~T?IN` z@E7}zg_$3*jhcpPt*N&s9cTgWQ@^V)-FP=U4&XQQuKxPI(rrD zvl&q{jfiJ#&Ab*kav_SFu@R!_`d~!UfoM7qO$VasKxCvsn5*C~Aqvmf%-s?DQT*Le zO@3tVbr@0dyCdUAVH(N8^zzR)np@c_r-{k->-#j)GFlJ zt-Llka`{nBHku*%U_>+EM>F6@GvG%v;73M^w?pi8_A2Vx<_?l=6n_U)8o;cd;>{Zl89YZ~2N3Ro(Bt0ax zeqSGq$VU(9qlfg-L;C0;ja1kx6GFd)DBMGK3E8NN*A+)n8(k{-U__U~Mwh}ym%>Ju z!bV1lw=C>+_9|*4bKlQ?6o21elOLJ8ct(``zCY|o#iwvUduOBsMx6?+LAxd@!Pwu+d7`XeDg45;ihYyjNzgvsZX~(Xr8>kc|d; zr{YLzqsDlqN(Uoq3>!6ujT*y7jbS4r752)6&~MO2r+cTvMrVd>bfz~LM-m$yiT9{h zVGV!|MzjhxS_K=ef{j+eMn;Nva_x2Y3U3%XHaa_GqqDtpa3r-+6Uhf7Y62TIfsLBL zMonNNBgNih9WVGBw9znc7;H2mWTO$@NE}IQl)-yu_enk&(S5MdeX!Aeu+e?6k&)s( zKYN|M!dr;eMx(q@Kr|+VXpA=wM-oJP;{SZ3T?=$DqCr432#5v&(I6l)Qeoqn5c(BJ zAMPQ|%%5i?@iYILW+P@E&xn%G{LO46e80ws;_ugLLR8oLwJ;)c-_MBR@B3>)WbWeG zM#=B{4WcTq3Vvjs`SbiJ{+WN={HPb+tu#;Y8Bx+_{=Mw;?1|n)ATrPV8BzQ*|2Raw z@usJFg3pMOKJ)Kw5mkHDKr|)PA5HPj$C0Ezx>{T<_L6)sqP@@`?S=kmFZ4%yp+7QG zy!ox;1%L6rtbKlTfp-BAnWuj2NAXYn;{51BycJ=d;W475PyH?w#*e0X(}3vWkc}?( zrsGIrqwBR4wVRgLVFW!Tl9dwFRQKK-3n9+5(Z0V&AEb z7yQLH4lJU@-eMqHq7m`FktNK}`$lf{ZUrKzF7-mL`m0u3?J@KykkS%YLm z@oSKAh%OVC)o~5-GK+}kM|Xxi`%dp}97#NzoMXlwhSt?h@lwjbJ>k>V{( z8#8-F^K9eUE4`Jl(W(%lRo?wLk{}u`hKugl8%+lz>JCKRfv7tWbq69N#gif(FZhcU zZrj!#^d1BvvzE=Z7QdDqS8H>`9J@x%h?1^l&v9zatoQT$D1N;^Zhl0yW>)VRQS$Zv z@Dr_vL$&s>_Xv(8wMK}F^udUVP-{h~wIbA75o*my@#PpBGkX>EBlc`#Bd)c08^zVy z?y!-WA2FiGLNBjSiH2u#FCcjShs34up*kgpG_8UxU%{g1=E4JrP3mg!d$l zq=nu9k5tI=n z9ceq;v7c*gO~^)Tyk~JFwb4P64@PtlY;+K8bP#NG5Nu?mc*|MG3;qUe#E6~?A$rbx z9!F9{y(J%vs5cPx2BO|T)EkJ5RM;yMLcdW&YeR_EdN1Ngis)d;2O~NdhzH<+;* zDZXZv;}A%1X0=>vX2p;DqxcoSn))NNe#eNCulSk%=r!-Pz_`YvNc^}KH;U|my$WW> z6(dSIuI*v(ve`x(LVmQtdlN^lT8o=!Q=i>OAB?CE{HPE7s1N+85B$hT@pRwD%w7fk zh!MRV@}sxCcW~rF6z4~TsDnNjQ3oLE07M;tr~?ogDZb!lV`i@ik@2IA-bU2grVye{ z-ew$05Z#7%@H^ryGCCMhM+;sBM=!WzVoHy1%L6yLC3S-57pZH-WD9W5Y<#` z-6S83s2dP<1EOv~)D4J?RM;yMLchUU`^ftUHZt>U_U!n1cARJT#TUZrw%4z(_3Tf) zPk_ig(c;!0|3oXU^)D0)>-a=#p`Am1=6yzfBzb2YBZ`0O7l-HuaYG%S`rTj=>9!U^ z6u(Xzhltu*UDs(%56OtkdOzDJe!agYM0H*74W~r_mkUN@^?ke zGygBVFW^UJ6@qOPzX}m&qsPQ!c6EUfC0&Jh%<&_$BZ_Slzay$98`X73l(ErQ-dEJt zLOo=BpB>jjjuxYBUz}f`J=)&WhQ~F2q7^@`#Ua`q08$FIMBbjwzMwE2r_;K4qe(!w`L}tC8 z5yh|f$03@C?`oOVdq$LWy?>%bwA0%OL}oRMYb|~?E3Vf1#u=6p3G{i5o@6;2Ujb z<%Rty=}OHQXIwM)BHTm9-;2cckYposZ^4L?-;0?0HGC#0m0e$B8^y1$#o1__7-!eb z7*W#owQ<%)UdjU^vj)kC;@2SK5RJ!o^~{MZF4mfCWOk)8qU5_+!!|OjS!|>D)vTIqWLBpbQS#NSu#L=aX!fJ{-Ox4J z$n0)rM9FtUhizo`urs3gJ?u3hGJDb)QSv?PVMON514b18=0Qz}%$o#^DEXTQ22q1l z1B|rh{eGVL$G_hn*F!E7%j)=k|1x`b#I4`Fo5ePYe>bZp8<}^l*hb0U%`!G(L}o3U z+gkiuc1>-~tW7hbR130^{?-^H z8YAPrSQ&?C5&k~_^Ce0~l=O?0i)^3WD%A>z%vU`bQT$gu;}9(ti|w~M8Bx-&dM>tz z+N9b5k@@l|BZ~j>X&j;@Vu}5pDI-ey<%&sv;6u)aM4$-aR zR=XFB5hdLhI z)MuNoVKbu3aW9hW``Kn(@XC3F+cUsTxp6U)n<_?m-$P|AE8TUn| z7sZQpyn}oZ7iZ|;xhkVo48n=G4_n2)IzxO!KM%32o zy7$=JuZ3s+j3|EQUlXFb&ioCc-l^WGH8aw3t;LVDakWPOYf{}t+Ksl>`lR{*kr`reV5VzWhbSn4n!GW`)F zitmr&5FLX54Y+Ro(IFPm5ve19$n-~yD84_6L$pC`sAGS$!6NFH>IXz-e#C7pets0! z)<%kvb(|lKv^^xZe)FuBZ502kHqJ)WAJz3)tr^#jN*x6onP>itDE^s$9HQsN^L2dY z|Gcd=wvkztRIZ`4J_L}rI3BZ}XlSrekV?$8V)GW)X`QT+bwnh@1>e|8v=`HCJR zivNmUO^E9H6}>Pb^X35~ihuKEye33-eVaav z$UHw{8^u3AstHkDpC1`S$EJ>jADQ>p7*YKDYjKEP5HHm6{k0byKQd3Wcvc(#L@RDq zOMX<>CtBtX^7z#8uu%igSIO z@mV~PaMyp0e%F6Zxa+?roc%Xy2iIRseAi!1xa+Sb-1S!z?)s|#ru<^;Z+_`s=Sk{w2gBhxZ+SywQ<2a^zn)d{c-2(&5V-{ws$c>Ey>F zZMJ=kapdhBzR1xx&XE^8e1XHO|93smzUzhLQo7?j+9Cc_Bz2{E8&DWZPhVM4@vWTx zKXIhD{3*;UU34l@T~Jo;^8e$+*ZOSwlO6qOCCTb*=`cn=b&*ATN^sRUD zf7aoDGUM?N-JGn?%=KO;M2THl`H=34!-|UF|+nM+2K!g_~RY^ zSmxE4PRB5B@m4teU=(=50ARqD#JLi0ADjT8S4C%)!w{Cgd_ z=B@l%N3MA*KiA2xvlHKU{7>^L6`kDpCvtkL|6xb}Lk|C-!$08g_cO1~bh?jutN$7Y zujZ|MmLo59%AhpcJja6;jeJ`I~@LQ=GB=__b{K;@8G@I$-mN(Yu={k zmTpGAI+;&baLbWoYUJ3fA8pDHpIVj_;bc@jE&3CpqyoZ{zRd#GmTK*Sw8C*@<7@iLZGZ{~|}Oc`Kjp$Te@} zGaR|*t^5Q>u6Zlx=Qozc|58Q;$hMzLo%EWw3A#Jwz0QfRc^iMJBiFo@H+1BhxAHzt zek+{#nz!-G9J%JL{7^^VJx+Yh+xU$fx#q3>a7W)tC%)!w{Kk%4^HzSOqi>ZHU-LG8 z6GyIjE6+Ik?sMX6-p1e4i9g7RuX!84mlMCY!(Z+2dpY_vZ}na1#J|qrTRQ1AZ_{7z z#Bb%q*Sw8?qZ7Zi6JPT-{;!<)ZJhX;xAAXs;C3*s@j~-fey}6gyp{KJ^81E)b*7W%ZG3vh8SG#CIQlhj<9Bf6nz!=X9DN;~_?oxz zyE$^rTX|nc-$IAK!Qmfs_|Xo3n8P38@GBhtD2IRC;U_wLe}^CA@Z%hQyu(*He6_=$ z=kUuMex<|DaQIUkezn8@+~Ef~{5uZ6$l(_|{1S)1+2NNu{H+fEw!=T=@S`05PKSTd z;g>u7RSy5A!*6u>O%A`=;SX{64Gurj;h%T-HynPg!@uP4FF5>qhkw=KUvc=CL%j5; zzp7`SN4Y$nS95#S?Ng?xeeRWwZ+IgwI2iJdS-IhjTz297)tUMs&09Imck!rR^H{gy z`y(4aESGXWND8z08Q$vK-;rzH$`5elnz!-;AukH(*SwK?H2*Ko$_=mO_`d)dzl~4x z)%t<>nz!;Ej$HFr-qVq5-pXmdQxedxc`H8%az7x~yp>b^1?8Hza+?1J<(jv0n$HI9 zp?NE({x~Styp_}ZH7M7-mD7AQDA&A|_jBZ$xAKf5*SwWe{~y$^c`K*+Oi-?QD<}U< z2kfhPEyo(8;}7J&rGfaGxBAFmgL2JVIr(c)u6Zjbe{C4huX!U+k^eTz$_;Pj)IT&1 z$Te@}RK6wwx#q2${JUvDu6Zjb{|?GEZ{&qEUU$i+H@uOj$p3@sHE-o~Kd@&ueM<9I zPWqb#aZ2j!YK z@*>jLHJje>My~pceX?@HTRGwF9FS|?$P36HgYy~98+nSx&!GQm-pEycodWs^FL)y_ zAb$v!PxDqz{uGpJ-pVtMT=Q1m&yj21%BejB^=sb9Grc)*JN-JGfIQj=LuVT^ZVMl(Yl4SADcKAyj zex}2p&%8R*X%h1)7rN}{-`+`>m6l{bHt{Ypt9w+n&mt{@^7?|3rtc=iqzUkvDbpz3Rwc zb9msyUltGW1$gOo1y^9(i{(`+q*L;gBL}bLLOk#2Z|LB=J|vft@(buQyw$fbNBjUj z6(78f|3WCf5HC9X28Vy$;s50D&oNI%Qhzsx`_C@^Q{ifzkhT`#X{1nN9^Ks4V_%hYak!#-Sdn&BY zBY7}?&D;3D5672e7qyiWzvP#ixAEUoSJC!{aUYUsIL3d3XYKPxl|IX_b@;a&{%yra z^`c%NiH6_qAroN!Zq;10z%;lPh@Q+_G+^Ho(U3U{Bk zeyi}PlZx}XI-B%(w(0a&KX81TV28sOJMjx0{zoT%fg|7P$cvObYnNkmNq`G64NmEC zxZ$y`=I9ya@b5Y0bj$sy6aOy`|E|Liarh-6UKIj4X7QDW { + + beforeEach(async () => { + await atom.packages.activatePackage('language-clojure'); + }); + + it('tokenizes the editor using TextMate parser', async () => { + atom.config.set('core.languageParser', 'textmate'); + await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/) + }); + + it('tokenizes the editor using node tree-sitter parser', async () => { + atom.config.set('core.languageParser', 'wasm-tree-sitter'); + await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/) + }); +}); From 1d755afb457e9ea30c9791e04896aa70b2f33402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Thu, 9 Mar 2023 22:34:47 -0300 Subject: [PATCH 02/17] Anon tokenizer --- packages/language-clojure/grammars/ts/highlights.scm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/language-clojure/grammars/ts/highlights.scm b/packages/language-clojure/grammars/ts/highlights.scm index 41c7bfaba..671f6e7de 100644 --- a/packages/language-clojure/grammars/ts/highlights.scm +++ b/packages/language-clojure/grammars/ts/highlights.scm @@ -1,4 +1,10 @@ ;; Collections +(anon_fn_lit + "(" @punctuation.section.expression.begin + . + (sym_lit) @entity.name.function + ")" @punctuation.section.expression.end) + (list_lit "(" @punctuation.section.expression.begin . From 4ba385dd75db2829b1ded250744669c4b9bc0d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Thu, 9 Mar 2023 22:48:01 -0300 Subject: [PATCH 03/17] Comment forms --- packages/language-clojure/grammars/ts/highlights.scm | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/language-clojure/grammars/ts/highlights.scm b/packages/language-clojure/grammars/ts/highlights.scm index 671f6e7de..8e3a86c87 100644 --- a/packages/language-clojure/grammars/ts/highlights.scm +++ b/packages/language-clojure/grammars/ts/highlights.scm @@ -49,3 +49,4 @@ (str_lit) @string.quoted.double (num_lit) @constant.numeric (comment) @comment.line.semicolon +(dis_expr) @comment.block.clojure From 8abf3b6b065b1424757cbbdcf44fce487634184f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Thu, 9 Mar 2023 22:53:46 -0300 Subject: [PATCH 04/17] Specific tree-sitter tokens --- packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj diff --git a/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj b/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj new file mode 100644 index 000000000..dd62c84b9 --- /dev/null +++ b/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj @@ -0,0 +1,3 @@ +#_ +(+ 1 2) +; <- comment.block From 84d4e9b8b34b55875ba6eb2378a1c5b03931d97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Thu, 9 Mar 2023 23:22:25 -0300 Subject: [PATCH 05/17] Back to using languageMode.ready Just to document this: I tried a HUGE amount of ideas for tests to run reliably. They need to wait for the editor when it's ready to be tokenized, but our callbacks are not that reliable - meaning, for the first tokenization on tree-sitter, they are called _after_ we register the callback, but for the second time, they are called _before_ we register them. There's no reliable way to _actually listen_ to changes, and none of these even were _called_ by TextMate grammars. So, this `.ready` will stay here so we know when TreeSitter is ready to tokenize. --- src/wasm-tree-sitter-language-mode.js | 10 ++++++++-- vendor/jasmine.js | 3 +-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/wasm-tree-sitter-language-mode.js b/src/wasm-tree-sitter-language-mode.js index 393a9c0c2..3651d4e67 100644 --- a/src/wasm-tree-sitter-language-mode.js +++ b/src/wasm-tree-sitter-language-mode.js @@ -79,6 +79,9 @@ class WASMTreeSitterLanguageMode { this.parsersByLanguage = new Map(); + let resolve + this.ready = new Promise(r => resolve = r) + this.grammar.getLanguage().then(lang => { this.rootLanguage = lang; this.rootLanguageLayer = new LanguageLayer(null, this, grammar, 0); @@ -86,7 +89,8 @@ class WASMTreeSitterLanguageMode { }).then(() => { this.rootLanguageLayer .update(null) - .then(() => this.emitter.emit('did-tokenize')); + .then(() => this.emitter.emit('did-tokenize')) + .then(() => resolve(true)); }); this.rootScopeDescriptor = new ScopeDescriptor({ @@ -128,6 +132,7 @@ class WASMTreeSitterLanguageMode { } bufferDidChange(change) { + if(!this.rootLanguageLayer) return; // if (!this.tree) { return; } let { oldRange, newRange, oldText, newText } = change; @@ -161,6 +166,7 @@ class WASMTreeSitterLanguageMode { } bufferDidFinishTransaction({ changes }) { + if(!this.rootLanguageLayer) return; for (let i = 0, { length } = changes; i < length; i++) { const { oldRange, newRange } = changes[i]; spliceArray( @@ -443,10 +449,10 @@ class WASMTreeSitterLanguageMode { } getFoldableRangesAtIndentLevel(level) { + if (!this.tree) return []; const tabLength = this.buffer.displayLayers[0]?.tabLength || 2 const minCol = (level-1) * tabLength const maxCol = (level) * tabLength - if (!this.tree) return []; return this.foldsQuery .captures(this.tree.rootNode) .filter(fold => { diff --git a/vendor/jasmine.js b/vendor/jasmine.js index 23f172e07..c902852ab 100644 --- a/vendor/jasmine.js +++ b/vendor/jasmine.js @@ -2757,8 +2757,7 @@ if (isCommonJS) exports.normalizeTreeSitterTextData = normalizeTreeSitterTextDat async function openDocument(fullPath) { const editor = await atom.workspace.open(fullPath); - const mode = editor.languageMode; - await mode.ready; + await editor.languageMode.ready; return editor; } From 4d0992051b5bc4a18672f043381d6e0cb4cff1ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Thu, 9 Mar 2023 23:25:26 -0300 Subject: [PATCH 06/17] Tokenizing #_ spec --- packages/language-clojure/spec/tokenizer-spec.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/language-clojure/spec/tokenizer-spec.js b/packages/language-clojure/spec/tokenizer-spec.js index b697a1769..040034b79 100644 --- a/packages/language-clojure/spec/tokenizer-spec.js +++ b/packages/language-clojure/spec/tokenizer-spec.js @@ -11,8 +11,13 @@ describe('Clojure grammars', () => { await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/) }); - it('tokenizes the editor using node tree-sitter parser', async () => { + it('tokenizes the editor using node tree-sitter parser the same as TextMate', async () => { atom.config.set('core.languageParser', 'wasm-tree-sitter'); await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/) }); + + it('tokenizes the editor using node tree-sitter parser (specific rules)', async () => { + atom.config.set('core.languageParser', 'wasm-tree-sitter'); + await runGrammarTests(path.join(__dirname, 'fixtures', 'tree-sitter-tokens.clj'), /;/) + }); }); From 7223a568f4bcedb0896031af2c3d0273902065f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Fri, 10 Mar 2023 21:58:42 -0300 Subject: [PATCH 07/17] Moving Jasmine tests to core --- .../spec/wasm-tree-sitter-spec.js | 84 +------------------ vendor/jasmine.js | 55 +++++++++++- 2 files changed, 56 insertions(+), 83 deletions(-) diff --git a/packages/language-ruby/spec/wasm-tree-sitter-spec.js b/packages/language-ruby/spec/wasm-tree-sitter-spec.js index 3ecfc52b1..58a982071 100644 --- a/packages/language-ruby/spec/wasm-tree-sitter-spec.js +++ b/packages/language-ruby/spec/wasm-tree-sitter-spec.js @@ -23,55 +23,8 @@ describe('WASM Tree-sitter Ruby grammar', () => { }); it('folds code', async () => { - const editor = await openDocument('folds.rb'); - let grouped = {} - normalized = normalizeTreeSitterTextData(editor, /#/).forEach(test => { - const [kind, id] = test.expected.split('.') - if(!kind || !id) { - throw new Error(dedent`Folds must be in the format fold_end.some-id - at ${test.testPosition.row+1}:${test.testPosition.column+1}`) - } - grouped[id] ||= {} - grouped[id][kind] = test - }) - for(const k in grouped) { - const v = grouped[k] - const keys = Object.keys(v) - if(keys.indexOf('fold_begin') === -1) - throw new Error(`Fold ${k} must contain fold_begin`) - if(keys.indexOf('fold_end') === -1) - throw new Error(`Fold ${k} must contain fold_end`) - if(keys.indexOf('fold_new_position') === -1) - throw new Error(`Fold ${k} must contain fold_new_position`) - } - - for(const k in grouped) { - const fold = grouped[k] - const begin = fold['fold_begin'] - const end = fold['fold_end'] - const newPos = fold['fold_new_position'] - - expect(editor.isFoldableAtBufferRow(begin.editorPosition.row)) - .toSatisfy((foldable, reason) => { - reason(dedent`Editor is not foldable at row ${begin.editorPosition.row+1} - at fixtures/folds.rb:${begin.testPosition.row+1}:${begin.testPosition.column+1}`) - return foldable - }) - editor.foldBufferRow(begin.editorPosition.row) - - expect(editor.screenPositionForBufferPosition(end.editorPosition)) - .toSatisfy((screenPosition, reason) => { - const {row,column} = newPos.editorPosition - reason(`At row ${begin.editorPosition.row+1}, editor should fold ` + - `up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` + - ` into the new position ${row+1}:${column+1}\n`+ - ` but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+ - ` at fixtures/folds.rb:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` + - ` at fixtures/folds.rb:${end.testPosition.row+1}:${end.testPosition.column+1}`) - return row === screenPosition.row && column === screenPosition.column - }) - editor.unfoldAll() - } + const fullPath = path.join(__dirname, 'fixtures', 'folds.rb') + await runFoldsTests(fullPath, /#/) }); }); @@ -81,36 +34,3 @@ async function openDocument(fileName) { await editor.languageMode.ready return editor } - -// function normalizeTestData(editor, commentRegex) { -// let allMatches = [], lastNonComment = 0 -// editor.getBuffer().getLines().forEach((row, i) => { -// const m = row.match(commentRegex) -// if(m) { -// const scope = editor.scopeDescriptorForBufferPosition([i, m.index]) -// if(scope.scopes.find(s => s.match(/comment/))) { -// allMatches.push({row: lastNonComment, text: row, col: m.index, testRow: i}) -// return -// } -// } -// lastNonComment = i -// }) -// return allMatches.map(({text, row, col, testRow}) => { -// const exactPos = text.match(/\^\s+(.*)/) -// if(exactPos) { -// const expected = exactPos[1] -// return { -// expected, -// editorPosition: {row, column: exactPos.index}, -// testPosition: {row: testRow, column: col} -// } -// } else { -// const pos = text.match(/\<-\s+(.*)/) -// return { -// expected: pos[1], -// editorPosition: {row, column: col}, -// testPosition: {row: testRow, column: col} -// } -// } -// }) -// } diff --git a/vendor/jasmine.js b/vendor/jasmine.js index c902852ab..7679f2977 100644 --- a/vendor/jasmine.js +++ b/vendor/jasmine.js @@ -2718,7 +2718,7 @@ jasmine.Matchers.prototype.toSatisfy = function(fn) { // to construct an error showing where EXACTLY was the assertion that failed function normalizeTreeSitterTextData(editor, commentRegex) { let allMatches = [], lastNonComment = 0 - const checkAssert = new RegExp('^' + commentRegex.source + '\\s*[\\<\\-|\\^]') + const checkAssert = new RegExp('^\\s*' + commentRegex.source + '\\s*[\\<\\-|\\^]') editor.getBuffer().getLines().forEach((row, i) => { const m = row.trim().match(commentRegex) if(m) { @@ -2781,3 +2781,56 @@ async function runGrammarTests(fullPath, commentRegex) { }) } if (isCommonJS) exports.runGrammarTests = runGrammarTests; + +async function runFoldsTests(fullPath, commentRegex) { + const editor = await openDocument(fullPath); + let grouped = {} + const normalized = normalizeTreeSitterTextData(editor, commentRegex).forEach(test => { + const [kind, id] = test.expected.split('.') + if(!kind || !id) { + throw new Error(`Folds must be in the format fold_end.some-id\n` + + ` at ${test.testPosition.row+1}:${test.testPosition.column+1}`) + } + grouped[id] ||= {} + grouped[id][kind] = test + }) + for(const k in grouped) { + const v = grouped[k] + const keys = Object.keys(v) + if(keys.indexOf('fold_begin') === -1) + throw new Error(`Fold ${k} must contain fold_begin`) + if(keys.indexOf('fold_end') === -1) + throw new Error(`Fold ${k} must contain fold_end`) + if(keys.indexOf('fold_new_position') === -1) + throw new Error(`Fold ${k} must contain fold_new_position`) + } + + for(const k in grouped) { + const fold = grouped[k] + const begin = fold['fold_begin'] + const end = fold['fold_end'] + const newPos = fold['fold_new_position'] + + expect(editor.isFoldableAtBufferRow(begin.editorPosition.row)) + .toSatisfy((foldable, reason) => { + reason(`Editor is not foldable at row ${begin.editorPosition.row+1}\n` + + ` at ${fullPath}:${begin.testPosition.row+1}:${begin.testPosition.column+1}`) + return foldable + }) + editor.foldBufferRow(begin.editorPosition.row) + + expect(editor.screenPositionForBufferPosition(end.editorPosition)) + .toSatisfy((screenPosition, reason) => { + const {row,column} = newPos.editorPosition + reason(`At row ${begin.editorPosition.row+1}, editor should fold ` + + `up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` + + ` into the new position ${row+1}:${column+1}\n`+ + ` but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+ + ` at ${fullPath}:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` + + ` at ${fullPath}:${end.testPosition.row+1}:${end.testPosition.column+1}`) + return row === screenPosition.row && column === screenPosition.column + }) + editor.unfoldAll() + } +} +if (isCommonJS) exports.runFoldsTests = runFoldsTests; From 85f1b2cd35d039cda2fc5f57fd83b6d359fbccb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Fri, 10 Mar 2023 22:11:05 -0300 Subject: [PATCH 08/17] Clojure folds --- .../language-clojure/grammars/ts/folds.scm | 7 ++++ .../spec/fixtures/tree-sitter-folds.clj | 34 +++++++++++++++++++ .../language-clojure/spec/tokenizer-spec.js | 5 +++ 3 files changed, 46 insertions(+) create mode 100644 packages/language-clojure/spec/fixtures/tree-sitter-folds.clj diff --git a/packages/language-clojure/grammars/ts/folds.scm b/packages/language-clojure/grammars/ts/folds.scm index e69de29bb..bd70d34b4 100644 --- a/packages/language-clojure/grammars/ts/folds.scm +++ b/packages/language-clojure/grammars/ts/folds.scm @@ -0,0 +1,7 @@ +[ + (list_lit) + (vec_lit) + (map_lit) + (anon_fn_lit) + (set_lit) +] @fold diff --git a/packages/language-clojure/spec/fixtures/tree-sitter-folds.clj b/packages/language-clojure/spec/fixtures/tree-sitter-folds.clj new file mode 100644 index 000000000..83db030b0 --- /dev/null +++ b/packages/language-clojure/spec/fixtures/tree-sitter-folds.clj @@ -0,0 +1,34 @@ +(defn a [a b] +; <- fold_begin.paren +; ^ fold_new_position.paren + (+ a b) + [1 + ; <- fold_begin.vector + ; ^ fold_new_position.vector + 2 + 3] + ; ^ fold_end.vector + {:a 10 + ; <- fold_begin.map + ; ^ fold_new_position.map + :b 20 + :c [1 + ; ^ fold_begin.inner_vector + ; ^ fold_new_position.inner_vector + 2 + 3]}) +; ^ fold_end.paren +; ^ fold_end.map +; ^ fold_end.inner_vector + +#(inner +; <- fold_begin.anon + ; ^ fold_new_position.anon + "function" + #{:with + ; <- fold_begin.set + ; ^ fold_new_position.set + :inner + :set}) +; ^ fold_end.anon +; ^ fold_end.set diff --git a/packages/language-clojure/spec/tokenizer-spec.js b/packages/language-clojure/spec/tokenizer-spec.js index 040034b79..b342d5277 100644 --- a/packages/language-clojure/spec/tokenizer-spec.js +++ b/packages/language-clojure/spec/tokenizer-spec.js @@ -20,4 +20,9 @@ describe('Clojure grammars', () => { atom.config.set('core.languageParser', 'wasm-tree-sitter'); await runGrammarTests(path.join(__dirname, 'fixtures', 'tree-sitter-tokens.clj'), /;/) }); + + fit('folds Clojure code', async () => { + atom.config.set('core.languageParser', 'wasm-tree-sitter'); + await runFoldsTests(path.join(__dirname, 'fixtures', 'tree-sitter-folds.clj'), /;/) + }); }); From 3d6a93ed593db1673d2bfb11b98f73e384e3dd72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Fri, 10 Mar 2023 22:11:44 -0300 Subject: [PATCH 09/17] Remove focus --- packages/language-clojure/spec/tokenizer-spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/language-clojure/spec/tokenizer-spec.js b/packages/language-clojure/spec/tokenizer-spec.js index b342d5277..4c3c845c8 100644 --- a/packages/language-clojure/spec/tokenizer-spec.js +++ b/packages/language-clojure/spec/tokenizer-spec.js @@ -21,7 +21,7 @@ describe('Clojure grammars', () => { await runGrammarTests(path.join(__dirname, 'fixtures', 'tree-sitter-tokens.clj'), /;/) }); - fit('folds Clojure code', async () => { + it('folds Clojure code', async () => { atom.config.set('core.languageParser', 'wasm-tree-sitter'); await runFoldsTests(path.join(__dirname, 'fixtures', 'tree-sitter-folds.clj'), /;/) }); From c3d41f2f292070e61e98a6bcd955482a79318385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Fri, 10 Mar 2023 23:59:00 -0300 Subject: [PATCH 10/17] Removed double resolve --- src/wasm-tree-sitter-language-mode.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/wasm-tree-sitter-language-mode.js b/src/wasm-tree-sitter-language-mode.js index bf01bf3e8..e156490dc 100644 --- a/src/wasm-tree-sitter-language-mode.js +++ b/src/wasm-tree-sitter-language-mode.js @@ -98,9 +98,6 @@ class WASMTreeSitterLanguageMode { this.parsersByLanguage = new Map(); - let resolve - this.ready = new Promise(r => resolve = r) - this.grammar.getLanguage().then(lang => { this.rootLanguage = lang; this.rootLanguageLayer = new LanguageLayer(null, this, grammar, 0); From a5ab58d903cbcd59f43c18cc6aa78460b7ea0419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Sat, 11 Mar 2023 00:08:45 -0300 Subject: [PATCH 11/17] Fixed meta.expression --- packages/language-clojure/grammars/ts/highlights.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/language-clojure/grammars/ts/highlights.scm b/packages/language-clojure/grammars/ts/highlights.scm index 8e3a86c87..dcde1edb2 100644 --- a/packages/language-clojure/grammars/ts/highlights.scm +++ b/packages/language-clojure/grammars/ts/highlights.scm @@ -2,13 +2,13 @@ (anon_fn_lit "(" @punctuation.section.expression.begin . - (sym_lit) @entity.name.function + (sym_lit) @entity.name.function @meta.expression ")" @punctuation.section.expression.end) (list_lit "(" @punctuation.section.expression.begin . - (sym_lit) @entity.name.function + (sym_lit) @entity.name.function @meta.expression ")" @punctuation.section.expression.end) (vec_lit From 2bfc7e72aae08b4fdf1a0bc8ff1cede8b00a570a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Sat, 11 Mar 2023 00:08:52 -0300 Subject: [PATCH 12/17] Removed punctuation for strings --- packages/language-clojure/spec/fixtures/tokens.clj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/language-clojure/spec/fixtures/tokens.clj b/packages/language-clojure/spec/fixtures/tokens.clj index cf8d51778..e9f69a385 100644 --- a/packages/language-clojure/spec/fixtures/tokens.clj +++ b/packages/language-clojure/spec/fixtures/tokens.clj @@ -15,14 +15,12 @@ (+ a b 10 20)) ;^ meta.expression ;^ entity.name.function - ; ^ constant.numeric.long + ; ^ constant.numeric (def a "A STRING") ; <- keyword.control ; ^ entity.global - ; ^ punctuation.definition.string.begin ; ^ string.quoted.double - ; ^ punctuation.definition.string.end #{'asd} ; <- punctuation.section.set.begin From bce60e749c21a36424ec0a7c3328f374bf01789d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Sat, 11 Mar 2023 00:09:06 -0300 Subject: [PATCH 13/17] Fixed matching of `<-` tests --- vendor/jasmine.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/jasmine.js b/vendor/jasmine.js index 7679f2977..f4ff41dc6 100644 --- a/vendor/jasmine.js +++ b/vendor/jasmine.js @@ -2720,7 +2720,7 @@ function normalizeTreeSitterTextData(editor, commentRegex) { let allMatches = [], lastNonComment = 0 const checkAssert = new RegExp('^\\s*' + commentRegex.source + '\\s*[\\<\\-|\\^]') editor.getBuffer().getLines().forEach((row, i) => { - const m = row.trim().match(commentRegex) + const m = row.match(commentRegex) if(m) { // const scope = editor.scopeDescriptorForBufferPosition([i, m.index]) // FIXME: use editor.scopeDescriptorForBufferPosition when it works From 0b405deac7ade4ff8f87bd979d7eaa88d48bd025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Sat, 11 Mar 2023 00:17:34 -0300 Subject: [PATCH 14/17] Not tokenizing quoted lists as function calls --- packages/language-clojure/grammars/ts/highlights.scm | 6 ++++++ .../language-clojure/spec/fixtures/tree-sitter-tokens.clj | 3 +++ 2 files changed, 9 insertions(+) diff --git a/packages/language-clojure/grammars/ts/highlights.scm b/packages/language-clojure/grammars/ts/highlights.scm index dcde1edb2..ac54f2f42 100644 --- a/packages/language-clojure/grammars/ts/highlights.scm +++ b/packages/language-clojure/grammars/ts/highlights.scm @@ -1,3 +1,9 @@ +(quoting_lit + value: (list_lit + "(" @punctuation.section.expression.begin + . + (sym_lit) @meta.symbol (#set! final "true"))) + ;; Collections (anon_fn_lit "(" @punctuation.section.expression.begin diff --git a/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj b/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj index dd62c84b9..771e37c1f 100644 --- a/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj +++ b/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj @@ -1,3 +1,6 @@ #_ (+ 1 2) ; <- comment.block + +'(a b 1 2) +; ^ !entity.name.function From 2502f4d87ddf00913d9ea5940d38b219227db4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Sat, 11 Mar 2023 00:17:50 -0300 Subject: [PATCH 15/17] Allow a non-match for scope :) --- vendor/jasmine.js | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/vendor/jasmine.js b/vendor/jasmine.js index f4ff41dc6..d69b33d2d 100644 --- a/vendor/jasmine.js +++ b/vendor/jasmine.js @@ -2771,12 +2771,22 @@ async function runGrammarTests(fullPath, commentRegex) { }) normalized.forEach(({expected, editorPosition, testPosition}) => { expect(editor.scopeDescriptorForBufferPosition(editorPosition).scopes).toSatisfy((scopes, reason) => { - reason(`Expected to find scope "${expected}" but found "${scopes}"\n` + - ` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}` - ); - const normalized = expected.replace(/([\.\-])/g, '\\$1') - const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$') - return scopes.find(e => e.match(scopeRegex)) !== undefined; + const dontFindScope = expected.startsWith("!"); + expected = expected.replace(/^!/, "") + if(dontFindScope) { + reason(`Expected to NOT find scope "${expected}" but found it\n` + + ` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}` + ); + } else { + reason(`Expected to find scope "${expected}" but found "${scopes}"\n` + + ` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}` + ); + } + const normalized = expected.replace(/([\.\-])/g, '\\$1'); + const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$'); + let result = scopes.find(e => e.match(scopeRegex)) !== undefined; + if(dontFindScope) result = !result; + return result }) }) } From 1d1bb5a9b99a6c5f3a133a81031384e7d74a0945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Sat, 11 Mar 2023 00:23:48 -0300 Subject: [PATCH 16/17] Removed external promise to `languageMode.ready` --- src/wasm-tree-sitter-language-mode.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/wasm-tree-sitter-language-mode.js b/src/wasm-tree-sitter-language-mode.js index f6e0f79c6..f02fd2fb9 100644 --- a/src/wasm-tree-sitter-language-mode.js +++ b/src/wasm-tree-sitter-language-mode.js @@ -77,11 +77,6 @@ class WASMTreeSitterLanguageMode { this.rootScopeId = this.getOrCreateScopeId(this.grammar.scopeName); this.ignoreScopeId = this.getOrCreateScopeId('ignore'); - let resolveReady; - this.ready = new Promise((resolve) => { - resolveReady = resolve; - }); - this.tokenized = false; this.subscriptions = new CompositeDisposable; @@ -98,15 +93,14 @@ class WASMTreeSitterLanguageMode { this.parsersByLanguage = new Map(); - this.grammar.getLanguage().then(lang => { + this.ready = this.grammar.getLanguage().then(lang => { this.rootLanguage = lang; this.rootLanguageLayer = new LanguageLayer(null, this, grammar, 0); return this.getOrCreateParserForLanguage(lang); }).then(() => { - this.rootLanguageLayer + return this.rootLanguageLayer .update(null) .then(() => this.emitter.emit('did-tokenize')) - .then(() => resolveReady(true)); }); this.rootScopeDescriptor = new ScopeDescriptor({ From db0a25bee13556d016adb3ef3cc354d9efb9993d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maur=C3=ADcio=20Szabo?= Date: Sat, 11 Mar 2023 00:31:44 -0300 Subject: [PATCH 17/17] Second argument is a grammar --- src/wasm-tree-sitter-language-mode.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wasm-tree-sitter-language-mode.js b/src/wasm-tree-sitter-language-mode.js index f02fd2fb9..77e74c210 100644 --- a/src/wasm-tree-sitter-language-mode.js +++ b/src/wasm-tree-sitter-language-mode.js @@ -518,7 +518,7 @@ class WASMTreeSitterLanguageMode { let root = layer.tree.rootNode; let index = this.buffer.characterIndexForPosition(position); let node = root.descendantForIndex(index); - while (node && !where(node)) { + while (node && !where(node, this.grammar)) { node = node.parent; } return node;