From 8687f612d4834b68bdb6242089ad58b3bc79cd8f Mon Sep 17 00:00:00 2001
From: Felix Angell <felixangell@members.fsf.org>
Date: Sat, 2 Mar 2019 20:54:01 +0000
Subject: [PATCH] command palette is now has a hand written lexer

this means that we can 'type check' command palette arguments, e.g. if something should be a string or not, etc.
it uses the same lexer which is used for the syntax highlighting ... let's see how long that lasts.
---
 "\"this is a test.c\"" |   0
 buff/action.go         |  36 +++++++++----
 buff/buffer.go         |   2 +-
 buff/close_buffer.go   |   4 +-
 buff/delete_line.go    |   4 +-
 buff/exit_action.go    |   8 +--
 buff/lex.go            |   4 ++
 buff/palette.go        |  35 ++++++------
 buff/shortcuts.go      |  11 ++--
 buff/view.go           |   3 +-
 foo                    |   0
 foo bar baz            |   0
 lex/lex.go             | 120 +++++++++++++++++++++++++++++++++++------
 lex/token.go           |  23 ++++++--
 14 files changed, 187 insertions(+), 63 deletions(-)
 create mode 100644 "\"this is a test.c\""
 create mode 100644 buff/lex.go
 create mode 100644 foo
 create mode 100644 foo bar baz

diff --git "a/\"this is a test.c\"" "b/\"this is a test.c\""
new file mode 100644
index 0000000..e69de29
diff --git a/buff/action.go b/buff/action.go
index 41d3709..950829e 100644
--- a/buff/action.go
+++ b/buff/action.go
@@ -4,15 +4,17 @@ import (
 	"log"
 	"strconv"
 	"strings"
+
+	"github.com/felixangell/phi/lex"
 )
 
 type BufferAction struct {
 	name          string
-	proc          func(*BufferView, []string) bool
+	proc          func(*BufferView, []*lex.Token) bool
 	showInPalette bool
 }
 
-func NewBufferAction(name string, proc func(*BufferView, []string) bool) BufferAction {
+func NewBufferAction(name string, proc func(*BufferView, []*lex.Token) bool) BufferAction {
 	return BufferAction{
 		name:          name,
 		proc:          proc,
@@ -20,7 +22,7 @@ func NewBufferAction(name string, proc func(*BufferView, []string) bool) BufferA
 	}
 }
 
-func OpenFile(v *BufferView, commands []string) bool {
+func OpenFile(v *BufferView, commands []*lex.Token) bool {
 	path := ""
 	if path == "" {
 		panic("unimplemented")
@@ -42,12 +44,20 @@ func OpenFile(v *BufferView, commands []string) bool {
 	return false
 }
 
-func NewFile(v *BufferView, commands []string) bool {
+func NewFile(v *BufferView, commands []*lex.Token) bool {
 	// TODO some nice error stuff
 	// have an error roll thing in the view?
 
+	if !commands[0].IsType(lex.String) {
+		return false
+	}
+
+	fileName := commands[0].Lexeme
+	// strip out the quotes (1...n-1)
+	fileName = fileName[1 : len(fileName)-1]
+
 	buff := v.AddBuffer()
-	buff.OpenFile(commands[0])
+	buff.OpenFile(fileName)
 
 	buff.SetFocus(true)
 	v.focusedBuff = buff.index
@@ -55,12 +65,16 @@ func NewFile(v *BufferView, commands []string) bool {
 	return false
 }
 
-func GotoLine(v *BufferView, commands []string) bool {
+func GotoLine(v *BufferView, commands []*lex.Token) bool {
 	if len(commands) == 0 {
 		return false
 	}
 
-	lineNum, err := strconv.ParseInt(commands[0], 10, 64)
+	if !commands[0].IsType(lex.Number) {
+		return false
+	}
+
+	lineNum, err := strconv.ParseInt(commands[0].Lexeme, 10, 64)
 	if err != nil {
 		log.Println("goto line invalid argument ", err.Error())
 		return false
@@ -75,7 +89,7 @@ func GotoLine(v *BufferView, commands []string) bool {
 	return false
 }
 
-func focusLeft(v *BufferView, commands []string) bool {
+func focusLeft(v *BufferView, commands []*lex.Token) bool {
 	if v == nil {
 		return false
 	}
@@ -83,7 +97,7 @@ func focusLeft(v *BufferView, commands []string) bool {
 	return false
 }
 
-func focusRight(v *BufferView, commands []string) bool {
+func focusRight(v *BufferView, commands []*lex.Token) bool {
 	if v == nil {
 		return false
 	}
@@ -91,7 +105,7 @@ func focusRight(v *BufferView, commands []string) bool {
 	return false
 }
 
-func pageDown(v *BufferView, commands []string) bool {
+func pageDown(v *BufferView, commands []*lex.Token) bool {
 	if v == nil {
 		return false
 	}
@@ -107,7 +121,7 @@ func pageDown(v *BufferView, commands []string) bool {
 	return false
 }
 
-func pageUp(v *BufferView, commands []string) bool {
+func pageUp(v *BufferView, commands []*lex.Token) bool {
 	if v == nil {
 		return false
 	}
diff --git a/buff/buffer.go b/buff/buffer.go
index 824ef57..16d99ea 100644
--- a/buff/buffer.go
+++ b/buff/buffer.go
@@ -476,7 +476,7 @@ func (b *Buffer) processTextInput(r rune) bool {
 		actionName, actionExists := source[key]
 		if actionExists {
 			if action, ok := register[actionName]; ok {
-				return action.proc(b.parent, []string{})
+				return action.proc(b.parent, []*lex.Token{})
 			}
 		} else {
 			log.Println("warning, unimplemented shortcut", shortcutName, "+", unicode.ToLower(r), "#", int(r), actionName)
diff --git a/buff/close_buffer.go b/buff/close_buffer.go
index 2c0997b..5503ac5 100644
--- a/buff/close_buffer.go
+++ b/buff/close_buffer.go
@@ -2,9 +2,11 @@ package buff
 
 import (
 	"fmt"
+
+	"github.com/felixangell/phi/lex"
 )
 
-func CloseBuffer(v *BufferView, commands []string) bool {
+func CloseBuffer(v *BufferView, commands []*lex.Token) bool {
 	b := v.getCurrentBuff()
 	if b == nil {
 		return false
diff --git a/buff/delete_line.go b/buff/delete_line.go
index 0a9cc90..63c71bb 100644
--- a/buff/delete_line.go
+++ b/buff/delete_line.go
@@ -1,6 +1,8 @@
 package buff
 
-func DeleteLine(v *BufferView, commands []string) bool {
+import "github.com/felixangell/phi/lex"
+
+func DeleteLine(v *BufferView, commands []*lex.Token) bool {
 	b := v.getCurrentBuff()
 	if b == nil {
 		return false
diff --git a/buff/exit_action.go b/buff/exit_action.go
index e57b4c1..32056d6 100644
--- a/buff/exit_action.go
+++ b/buff/exit_action.go
@@ -3,15 +3,17 @@ package buff
 import (
 	"log"
 	"os"
+
+	"github.com/felixangell/phi/lex"
 )
 
-func ExitPhi(v *BufferView, commands []string) bool {
+func ExitPhi(v *BufferView, commands []*lex.Token) bool {
 	// todo this probably wont work...
 	// would also be nice to have a thing
 	// that asks if we want to save all buffers
 	// rather than going thru each one specifically?
-	for i, _ := range v.buffers {
-		CloseBuffer(v, []string{})
+	for i := range v.buffers {
+		CloseBuffer(v, []*lex.Token{})
 		log.Println("Closing buffer ", i)
 	}
 
diff --git a/buff/lex.go b/buff/lex.go
new file mode 100644
index 0000000..d989520
--- /dev/null
+++ b/buff/lex.go
@@ -0,0 +1,4 @@
+package buff
+
+type lexer struct {
+}
diff --git a/buff/palette.go b/buff/palette.go
index cd9351a..bddf1b7 100644
--- a/buff/palette.go
+++ b/buff/palette.go
@@ -1,12 +1,13 @@
 package buff
 
 import (
-	"log"
+	"fmt"
 	"strings"
 
 	"github.com/felixangell/fuzzysearch/fuzzy"
 	"github.com/felixangell/phi/cfg"
 	"github.com/felixangell/phi/gui"
+	"github.com/felixangell/phi/lex"
 	"github.com/felixangell/strife"
 	"github.com/veandco/go-sdl2/sdl"
 )
@@ -140,33 +141,27 @@ func NewCommandPalette(conf cfg.TomlConfig, view *BufferView) *CommandPalette {
 
 func (b *CommandPalette) processCommand() {
 	input := b.buff.table.Lines[0].String()
-	input = strings.TrimSpace(input)
-	tokenizedLine := strings.Split(input, " ")
+	fmt.Println("raw comand palette input is ", input)
 
-	// command
-	if strings.Compare(tokenizedLine[0], "!") == 0 {
+	tokens := lex.New(input).Tokenize()
+	fmt.Println("tokenized to", tokens)
 
-		// no commands to process, just the
-		// bang.
-		if len(tokenizedLine) == 1 {
-			return
-		}
-
-		// slice off the command token
-		tokenizedLine := strings.Split(input, " ")[1:]
-
-		log.Println("Command Entered: '", input, "', ", tokenizedLine)
-		command := tokenizedLine[0]
-
-		log.Println("command palette: ", tokenizedLine)
+	// FIXME
+	if len(tokens) <= 1 {
+		return
+	}
 
+	if tokens[0].Equals("!") && tokens[1].IsType(lex.Word) {
+		command := tokens[1].Lexeme
 		action, exists := register[command]
 		if !exists {
+			fmt.Println("No such action ", command)
 			return
 		}
 
-		action.proc(b.parent, tokenizedLine[1:])
-		return
+		args := tokens[2:]
+		fmt.Println("executing action", command, "with arguments", args)
+		action.proc(b.parent, args)
 	}
 
 	if index, ok := b.pathToIndex[input]; ok {
diff --git a/buff/shortcuts.go b/buff/shortcuts.go
index bb464fe..d52bf4a 100644
--- a/buff/shortcuts.go
+++ b/buff/shortcuts.go
@@ -10,16 +10,17 @@ import (
 	"path/filepath"
 
 	"github.com/atotto/clipboard"
+	"github.com/felixangell/phi/lex"
 )
 
-func ShowPalette(v *BufferView, commands []string) bool {
+func ShowPalette(v *BufferView, commands []*lex.Token) bool {
 	b := v.getCurrentBuff()
 	v.UnfocusBuffers()
 	v.focusPalette(b)
 	return true
 }
 
-func Paste(v *BufferView, commands []string) bool {
+func Paste(v *BufferView, commands []*lex.Token) bool {
 	b := v.getCurrentBuff()
 	if b == nil {
 		return false
@@ -37,7 +38,7 @@ func Paste(v *BufferView, commands []string) bool {
 	return false
 }
 
-func Undo(v *BufferView, commands []string) bool {
+func Undo(v *BufferView, commands []*lex.Token) bool {
 	b := v.getCurrentBuff()
 	if b == nil {
 		return false
@@ -46,7 +47,7 @@ func Undo(v *BufferView, commands []string) bool {
 	return false
 }
 
-func Redo(v *BufferView, commands []string) bool {
+func Redo(v *BufferView, commands []*lex.Token) bool {
 	b := v.getCurrentBuff()
 	if b == nil {
 		return false
@@ -67,7 +68,7 @@ func genFileName(dir, prefix, suffix string) string {
 // if the buffer is modified it will be
 // re-rendered.
 
-func Save(v *BufferView, commands []string) bool {
+func Save(v *BufferView, commands []*lex.Token) bool {
 	// TODO Config option for this.
 	atomicFileSave := true
 
diff --git a/buff/view.go b/buff/view.go
index b527ca0..0f0570d 100644
--- a/buff/view.go
+++ b/buff/view.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/felixangell/phi/cfg"
 	"github.com/felixangell/phi/gui"
+	"github.com/felixangell/phi/lex"
 	"github.com/felixangell/strife"
 	"github.com/fsnotify/fsnotify"
 	"github.com/veandco/go-sdl2/sdl"
@@ -308,7 +309,7 @@ func (n *BufferView) OnUpdate() bool {
 		if actionExists {
 			if action, ok := register[actionName]; ok {
 				log.Println("Executing action '" + actionName + "'")
-				return action.proc(n, []string{})
+				return action.proc(n, []*lex.Token{})
 			}
 		} else {
 			log.Println("view: unimplemented shortcut", shortcutName, "+", string(unicode.ToLower(r)), "#", int(r), actionName, key)
diff --git a/foo b/foo
new file mode 100644
index 0000000..e69de29
diff --git a/foo bar baz b/foo bar baz
new file mode 100644
index 0000000..e69de29
diff --git a/lex/lex.go b/lex/lex.go
index 28ba425..ec2b4fc 100644
--- a/lex/lex.go
+++ b/lex/lex.go
@@ -1,14 +1,21 @@
 package lex
 
+import (
+	"fmt"
+	"unicode"
+)
+
 type Lexer struct {
-	pos   int
-	input []rune
+	startingPos int
+	pos         int
+	input       []rune
 }
 
 func New(input string) *Lexer {
 	return &Lexer{
-		pos:   0,
-		input: []rune(input),
+		startingPos: 0,
+		pos:         0,
+		input:       []rune(input),
 	}
 }
 
@@ -18,6 +25,17 @@ func (l *Lexer) consume() rune {
 	return consumed
 }
 
+func (l *Lexer) expect(c rune) (rune, bool) {
+	if l.hasNext() && l.peek() == c {
+		return l.consume(), true
+	}
+	if !l.hasNext() {
+		return rune(0), false
+	}
+	// TODO, fail?
+	return l.consume(), true
+}
+
 func (l *Lexer) next(offs int) rune {
 	return l.input[l.pos+offs]
 }
@@ -30,6 +48,64 @@ func (l *Lexer) hasNext() bool {
 	return l.pos < len(l.input)
 }
 
+func (l *Lexer) recognizeString() *Token {
+	l.expect('"')
+	for l.hasNext() && l.peek() != '"' {
+		l.consume()
+	}
+	l.expect('"')
+	return NewToken(l.captureLexeme(), String, l.startingPos)
+}
+
+func (l *Lexer) recognizeCharacter() *Token {
+	l.expect('\'')
+	for l.hasNext() && l.peek() != '\'' {
+		l.consume()
+	}
+	l.expect('\'')
+	return NewToken(l.captureLexeme(), Character, l.startingPos)
+}
+
+func (l *Lexer) recognizeNumber() *Token {
+	for l.hasNext() && unicode.IsDigit(l.peek()) {
+		l.consume()
+	}
+	if l.hasNext() && l.peek() == '.' {
+		l.consume()
+		for l.hasNext() && unicode.IsDigit(l.peek()) {
+			l.consume()
+		}
+	}
+	return NewToken(l.captureLexeme(), Number, l.startingPos)
+}
+
+func (l *Lexer) recognizeSymbol() *Token {
+	l.consume()
+	return NewToken(l.captureLexeme(), Symbol, l.startingPos)
+}
+
+func (l *Lexer) recognizeWord() *Token {
+	for l.hasNext() && (unicode.IsLetter(l.peek()) || unicode.IsDigit(l.peek())) {
+		l.consume()
+	}
+
+	if l.hasNext() {
+		curr := l.peek()
+		if curr == '_' || curr == '-' {
+			l.consume()
+			for l.hasNext() && (unicode.IsLetter(l.peek()) || unicode.IsDigit(l.peek())) {
+				l.consume()
+			}
+		}
+	}
+
+	return NewToken(l.captureLexeme(), Word, l.startingPos)
+}
+
+func (l *Lexer) captureLexeme() string {
+	return string(l.input[l.startingPos:l.pos])
+}
+
 func (l *Lexer) Tokenize() []*Token {
 	var result []*Token
 	for l.hasNext() {
@@ -46,22 +122,34 @@ func (l *Lexer) Tokenize() []*Token {
 			l.consume()
 		}
 
-		startPos := l.pos
-		for l.hasNext() {
-			// we run into a layout character
-			if l.peek() <= ' ' {
-				break
+		l.startingPos = l.pos
+
+		if token := func() *Token {
+			if !l.hasNext() {
+				return nil
 			}
 
-			l.consume()
+			curr := l.peek()
+			switch {
+			case curr == '"':
+				return l.recognizeString()
+			case curr == '\'':
+				return l.recognizeCharacter()
+			case unicode.IsLetter(curr):
+				return l.recognizeWord()
+			case unicode.IsDigit(curr):
+				return l.recognizeNumber()
+			case unicode.IsGraphic(curr):
+				return l.recognizeSymbol()
+			case curr == ' ':
+				return nil
+			}
+
+			panic(fmt.Sprintln("unhandled input! ", string(curr)))
+		}(); token != nil {
+			result = append(result, token)
 		}
 
-		// this should be a recognized
-		// token i think?
-
-		lexeme := string(l.input[startPos:l.pos])
-		tok := NewToken(lexeme, Word, startPos)
-		result = append(result, tok)
 	}
 	return result
 }
diff --git a/lex/token.go b/lex/token.go
index 28ad7de..b32f83b 100644
--- a/lex/token.go
+++ b/lex/token.go
@@ -1,11 +1,18 @@
 package lex
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
-type TokenType uint
+type TokenType string
 
 const (
-	Word TokenType = iota
+	Word      TokenType = "word"
+	Symbol              = "sym"
+	Character           = "char"
+	String              = "string"
+	Number              = "num"
 )
 
 type Token struct {
@@ -14,10 +21,18 @@ type Token struct {
 	Start  int
 }
 
+func (t *Token) Equals(str string) bool {
+	return strings.Compare(str, t.Lexeme) == 0
+}
+
+func (t *Token) IsType(typ TokenType) bool {
+	return t.Type == typ
+}
+
 func NewToken(lexeme string, kind TokenType, start int) *Token {
 	return &Token{lexeme, kind, start}
 }
 
 func (t *Token) String() string {
-	return fmt.Sprintf("lexeme: %s, type %s, at pos %d", t.Lexeme, string(t.Type), t.Start)
+	return fmt.Sprintf("{ %s = %s }", t.Lexeme, string(t.Type))
 }