Set invisible values for spaces and tabs when initial tokenization occurs.

Also break whitespace into its own token just like tabs.

spec/app/editor-spec.coffee

@@ -1828,9 +1828,10 @@ describe "Editor", ->
       expect(editor.getCursor().getScreenPosition().row).toBe(0)
       expect(editor.getFirstVisibleScreenRow()).toBe(0)
 
-  describe "when showInvisibles is enabled on the editSession", ->
+  describe "when editSession.showInvisibles is true", ->
     beforeEach ->
-      editor.activeEditSession.showInvisibles = true
+      project.setShowInvisibles(true)
+      rootView.open()
       editor.attachToDom(5)
 
     it "displays spaces as •, tabs as ▸ and newlines as ¬", ->

spec/app/token-spec.coffee

@@ -1,4 +1,5 @@
 _ = require 'underscore'
+Token = require 'token'
 Buffer = require 'buffer'
 TokenizedBuffer = require 'tokenized-buffer'
 
@@ -13,3 +14,48 @@ describe "Token", ->
 
   afterEach ->
     editSession.destroy()
+
+  describe ".breakOutWhitespaceCharacters(tabLength, showInvisbles)", ->
+    describe "when showInvisbles is false", ->
+      it "replaces spaces and tabs with their own tokens", ->
+        value = " spaces   tabs\t\t\t  "
+        scopes = ["whatever"]
+        isAtomic = false
+        bufferDelta = value.length
+        token = new Token({value, scopes, isAtomic, bufferDelta})
+        tokens = token.breakOutWhitespaceCharacters(4, false)
+
+        expect(tokens.length).toBe(8)
+        expect(tokens[0].value).toBe(" ")
+        expect(tokens[0].scopes).not.toContain("invisible")
+        expect(tokens[1].value).toBe("spaces")
+        expect(tokens[1].scopes).not.toContain("invisible")
+        expect(tokens[2].value).toBe("   ")
+        expect(tokens[3].value).toBe("tabs")
+        expect(tokens[4].value).toBe("    ")
+        expect(tokens[4].scopes).not.toContain("invisible")
+        expect(tokens[5].value).toBe("    ")
+        expect(tokens[6].value).toBe("    ")
+        expect(tokens[7].value).toBe("  ")
+
+    describe "when showInvisbles is true", ->
+      it "replaces spaces and tabs with their own tokens", ->
+        value = " spaces   tabs\t\t\t  "
+        scopes = ["whatever"]
+        isAtomic = false
+        bufferDelta = value.length
+        token = new Token({value, scopes, isAtomic, bufferDelta})
+        tokens = token.breakOutWhitespaceCharacters(4, true)
+
+        expect(tokens.length).toBe(8)
+        expect(tokens[0].value).toBe("•")
+        expect(tokens[0].scopes).toContain("invisible")
+        expect(tokens[1].value).toBe("spaces")
+        expect(tokens[1].scopes).not.toContain("invisible")
+        expect(tokens[2].value).toBe("•••")
+        expect(tokens[3].value).toBe("tabs")
+        expect(tokens[4].value).toBe("▸   ")
+        expect(tokens[4].scopes).toContain("invisible")
+        expect(tokens[5].value).toBe("▸   ")
+        expect(tokens[6].value).toBe("▸   ")
+        expect(tokens[7].value).toBe("••")

spec/app/tokenized-buffer-spec.coffee

@@ -81,12 +81,12 @@ describe "TokenizedBuffer", ->
 
           # previous line 3 should be combined with input to form line 1
           expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
+          expect(tokenizedBuffer.lineForScreenRow(1).tokens[8]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
 
-          # lines below deleted regions should be shifted upward
+          # # lines below deleted regions should be shifted upward
           expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[1]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[1]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
+          expect(tokenizedBuffer.lineForScreenRow(3).tokens[3]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
+          expect(tokenizedBuffer.lineForScreenRow(4).tokens[3]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
@@ -125,7 +125,7 @@ describe "TokenizedBuffer", ->
           expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
 
           # previous line 3 is pushed down to become line 5
-          expect(tokenizedBuffer.lineForScreenRow(5).tokens[3]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
+          expect(tokenizedBuffer.lineForScreenRow(5).tokens[5]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
@@ -165,11 +165,12 @@ describe "TokenizedBuffer", ->
         screenLine0 = tokenizedBuffer.lineForScreenRow(0)
         expect(screenLine0.text).toBe "# Econ 101#{editSession2.getTabText()}"
         { tokens } = screenLine0
-        expect(tokens.length).toBe 3
+        expect(tokens.length).toBe 6
         expect(tokens[0].value).toBe "#"
-        expect(tokens[1].value).toBe " Econ 101"
-        expect(tokens[2].value).toBe editSession2.getTabText()
-        expect(tokens[2].scopes).toEqual tokens[1].scopes
-        expect(tokens[2].isAtomic).toBeTruthy()
+        expect(tokens[2].value).toBe "Econ"
+        expect(tokens[4].value).toBe "101"
+        expect(tokens[5].value).toBe editSession2.getTabText()
+        expect(tokens[5].scopes).toEqual tokens[1].scopes
+        expect(tokens[5].isAtomic).toBeTruthy()
 
         expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{editSession2.getTabText()} buy()#{editSession2.getTabText()}while supply > demand"

src/app/edit-session.coffee

@@ -44,7 +44,7 @@ class EditSession
     @id = @constructor.idCounter++
     @softTabs ?= true
     @languageMode = new LanguageMode(this, @buffer.getExtension())
-    @displayBuffer = new DisplayBuffer(@buffer, { @languageMode, @tabLength })
+    @displayBuffer = new DisplayBuffer(@buffer, { @languageMode, @tabLength, @showInvisibles })
     @tokenizedBuffer = @displayBuffer.tokenizedBuffer
     @anchors = []
     @anchorRanges = []

src/app/token.coffee

@@ -5,7 +5,6 @@ class Token
   value: null
   scopes: null
   isAtomic: null
-  isTab: null
 
   constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @fold, @isTab}) ->
     @screenDelta = @value.length
@@ -22,13 +21,19 @@ class Token
     value2 = @value.substring(splitIndex)
     [new Token(value: value1, scopes: @scopes), new Token(value: value2, scopes: @scopes)]
 
-  breakOutTabCharacters: (tabLength) ->
-    return [this] unless /\t/.test(@value)
+  breakOutWhitespaceCharacters: (tabLength, showInvisibles) ->
+    return [this] unless /\t| /.test(@value)
 
-    tabText = new Array(tabLength + 1).join(" ")
-    for substring in @value.match(/([^\t]+|\t)/g)
-      if substring == '\t'
-        new Token(value: tabText, scopes: @scopes, bufferDelta: 1, isAtomic: true, isTab: true)
+    for substring in @value.match(/([^\t ]+|(\t| +))/g)
+      scopesForInvisibles = if showInvisibles then @scopes.concat("invisible") else @scopes
+
+      if substring == "\t"
+        value = new Array(tabLength + 1).join(" ")
+        value = "▸" + value[1..] if showInvisibles
+        new Token(value: value, scopes: scopesForInvisibles, bufferDelta: 1, isAtomic: true)
+      else if /^ +$/.test(substring)
+        value = if showInvisibles then substring.replace(/[ ]/g, "•") else substring
+        new Token(value: value, scopes: scopesForInvisibles)
       else
         new Token(value: substring, scopes: @scopes)
 
@@ -45,10 +50,4 @@ class Token
       .replace(/</g, '&lt;')
       .replace(/>/g, '&gt;')
 
-    if showInvisibles
-      if @isTab
-        value = "▸" + value[1..]
-      else
-        value = value.replace(/[ ]+/g, "•")
-
     value

src/app/tokenized-buffer.coffee

@@ -11,11 +11,12 @@ class TokenizedBuffer
 
   languageMode: null
   tabLength: null
+  showInvisibles: null
   buffer: null
   aceAdaptor: null
   screenLines: null
 
-  constructor: (@buffer, { @languageMode, @tabLength }) ->
+  constructor: (@buffer, { @languageMode, @tabLength, @showInvisibles }) ->
     @tabLength ?= 2
     @languageMode.tokenizedBuffer = this
     @id = @constructor.idCounter++
@@ -66,7 +67,7 @@ class TokenizedBuffer
     tokenObjects = []
     for tokenProperties in tokens
       token = new Token(tokenProperties)
-      tokenObjects.push(token.breakOutTabCharacters(@tabLength)...)
+      tokenObjects.push(token.breakOutWhitespaceCharacters(@tabLength, @showInvisibles)...)
     text = _.pluck(tokenObjects, 'value').join('')
     new ScreenLine(tokenObjects, text, [1, 0], [1, 0], { stack })