From 9962ce98593120af25a3c7ab04ad32c19def53a8 Mon Sep 17 00:00:00 2001
From: Kevin Sawicki <kevin@github.com>
Date: Tue, 31 Dec 2013 13:42:35 -0800
Subject: [PATCH] Implement createToken in Syntax

---
 package.json                | 2 +-
 src/syntax.coffee           | 3 +++
 src/tokenized-buffer.coffee | 1 -
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 70791dc4d..b1db26b84 100644
--- a/package.json
+++ b/package.json
@@ -25,7 +25,7 @@
     "coffeestack": "0.6.0",
     "diff": "git://github.com/benogle/jsdiff.git",
     "emissary": "0.19.0",
-    "first-mate": "0.7.0",
+    "first-mate": "0.9.0",
     "fs-plus": "0.13.0",
     "fuzzaldrin": "0.1.0",
     "git-utils": "0.29.0",
diff --git a/src/syntax.coffee b/src/syntax.coffee
index 554c02ba7..20d69a9d8 100644
--- a/src/syntax.coffee
+++ b/src/syntax.coffee
@@ -4,6 +4,7 @@ _ = require 'underscore-plus'
 {GrammarRegistry, ScopeSelector} = require 'first-mate'
 
 {$, $$} = require './space-pen-extensions'
+Token = require './token'
 
 ### Public ###
 module.exports =
@@ -26,6 +27,8 @@ class Syntax extends GrammarRegistry
   serialize: ->
     {deserializer: @constructor.name, @grammarOverridesByPath}
 
+  createToken: (value, scopes) -> new Token({value, scopes})
+
   addProperties: (args...) ->
     name = args.shift() if args.length > 2
     [selector, properties] = args
diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee
index e747d3062..ac3cf01b9 100644
--- a/src/tokenized-buffer.coffee
+++ b/src/tokenized-buffer.coffee
@@ -203,7 +203,6 @@ class TokenizedBuffer extends Model
     lineEnding = @buffer.lineEndingForRow(row)
     tabLength = @getTabLength()
     { tokens, ruleStack } = @grammar.tokenizeLine(line, ruleStack, row is 0)
-    tokens = (new Token(token) for token in tokens)
     new TokenizedLine({tokens, ruleStack, tabLength, lineEnding})
 
   # FIXME: benogle says: These are actually buffer rows as all buffer rows are