From 0bf0c0527f70731da154dc6027001bf0b6ac34c3 Mon Sep 17 00:00:00 2001
From: Nathan Sobo <nathan@github.com>
Date: Tue, 24 Feb 2015 12:10:25 -0700
Subject: [PATCH] =?UTF-8?q?Don=E2=80=99t=20tokenize=20spaces=20preceding?=
 =?UTF-8?q?=20combining=20chars=20as=20leading=20whitespace?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #5349
---
 spec/tokenized-buffer-spec.coffee | 12 ++++++++++++
 src/tokenized-line.coffee         |  3 +++
 2 files changed, 15 insertions(+)

diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee
index 41a63a562..0adcd2756 100644
--- a/spec/tokenized-buffer-spec.coffee
+++ b/spec/tokenized-buffer-spec.coffee
@@ -318,6 +318,18 @@ describe "TokenizedBuffer", ->
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].isAtomic).toBeFalsy()
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].value).toBe "  current "
 
+      it "does not tokenize whitespaces followed by combining characters as leading whitespace", ->
+        buffer.setText("    \u030b")
+        fullyTokenize(tokenizedBuffer)
+
+        {tokens} = tokenizedBuffer.tokenizedLineForRow(0)
+        expect(tokens[0].value).toBe "  "
+        expect(tokens[0].hasLeadingWhitespace()).toBe true
+        expect(tokens[1].value).toBe " "
+        expect(tokens[1].hasLeadingWhitespace()).toBe true
+        expect(tokens[2].value).toBe " \u030b"
+        expect(tokens[2].hasLeadingWhitespace()).toBe false
+
   describe "when the buffer contains hard-tabs", ->
     beforeEach ->
       waitsForPromise ->
diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee
index d1576ab9a..903a03300 100644
--- a/src/tokenized-line.coffee
+++ b/src/tokenized-line.coffee
@@ -1,4 +1,5 @@
 _ = require 'underscore-plus'
+{isPairedCharacter} = require './text-utils'
 
 NonWhitespaceRegex = /\S/
 LeadingWhitespaceRegex = /^\s*/
@@ -147,6 +148,8 @@ class TokenizedLine
 
   markLeadingAndTrailingWhitespaceTokens: ->
     firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex)
+    if firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, firstNonWhitespaceIndex - 1)
+      firstNonWhitespaceIndex--
     firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex)
     @lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
     index = 0