feat: draw insert marked text without accutally into vim

this commit fixes following issues:

1. use `-`, `=`, `arrow key` to select candidate. which shouldn't be
   handle by vim
2. use `left`, `right` to move in marked text by word. which shouldn't
   be handle by vim directly
3. though inserting marked text directly into UGrid cells, bypass vim
   key mapping system, only the finall insertText be inserted into vim.
   so vim map like jk to esc won't affect input method.
   and when input i in normal mode when input method, I can press <CR>
   to confirm it and only i will send to vim.

 this is also the behaviour of mac terminal.

NvimView/Sources/NvimView/CellAttributesCollection.swift

@@ -8,6 +8,7 @@ import Foundation
 final class CellAttributesCollection {
   static let defaultAttributesId = 0
   static let reversedDefaultAttributesId = Int.max
+  static let markedAttributesId = Int.max - 1
 
   private(set) var defaultAttributes = CellAttributes(
     fontTrait: [],
@@ -24,7 +25,12 @@ final class CellAttributesCollection {
   }
 
   func attributes(of id: Int, withDefaults defaults: CellAttributes) -> CellAttributes? {
-    if id == Int.max { return self.defaultAttributes.reversed }
+    if id == Self.markedAttributesId {
+      var attr = defaultAttributes
+      attr.fontTrait.formUnion(.underline)
+      return attr
+    }
+    if id == Self.reversedDefaultAttributesId { return self.defaultAttributes.reversed }
 
     let absId = abs(id)
     guard let attrs = self.attributes[absId] else { return nil }

NvimView/Sources/NvimView/KeyUtils.swift

@@ -37,6 +37,14 @@ class KeyUtils {
 
     return key
   }
+  static func isHalfWidth(char: Character) -> Bool {
+    // https://stackoverflow.com/questions/13505075/analyzing-full-width-or-half-width-character-in-java?noredirect=1&lq=1 // swiftlint:disable:this all
+    switch char {
+    case "\u{00}"..."\u{FF}", "\u{FF61}"..."\u{FFDC}", "\u{FFE8}"..."\u{FFEE}":
+      return true
+    default: return false
+    }
+  }
 }
 
 private let specialKeys = [

NvimView/Sources/NvimView/NvimView+Draw.swift

@@ -68,10 +68,10 @@ extension NvimView {
     // the corresponding cell...
     if self.mode == .termFocus { return }
 
-    let cursorPosition = self.ugrid.cursorPosition
+    let cursorPosition = self.ugrid.cursorPositionWithMarkedInfo()
     let defaultAttrs = self.cellAttributesCollection.defaultAttributes
 
-    let cursorRegion = self.cursorRegion(for: self.ugrid.cursorPosition)
+    let cursorRegion = self.cursorRegion(for: cursorPosition)
     if cursorRegion.top < 0
       || cursorRegion.bottom > self.ugrid.size.height - 1
       || cursorRegion.left < 0

NvimView/Sources/NvimView/NvimView+Key.swift

@@ -19,6 +19,7 @@ public extension NvimView {
 
     if !isMeta {
       let cocoaHandledEvent = NSTextInputContext.current?.handleEvent(event) ?? false
+      if self.hasMarkedText() { self.keyDownDone = true } // mark state ignore Down,Up,Left,Right,=,- etc keys
       if self.keyDownDone, cocoaHandledEvent { return }
     }
 
@@ -54,9 +55,8 @@ public extension NvimView {
     default: return
     }
 
-    let length = self.markedText?.count ?? 0
     try? self.bridge
-      .deleteCharacters(length, andInputEscapedString: self.vimPlainString(text))
+      .deleteCharacters(0, andInputEscapedString: self.vimPlainString(text))
       .wait()
 
     if self.hasMarkedText() { _unmarkText() }
@@ -150,10 +150,6 @@ public extension NvimView {
 
     defer { self.keyDownDone = true }
 
-    if self.markedText == nil { self.markedPosition = self.ugrid.cursorPosition }
-
-    let oldMarkedTextLength = self.markedText?.count ?? 0
-
     switch object {
     case let string as String: self.markedText = string
     case let attributedString as NSAttributedString: self.markedText = attributedString.string
@@ -161,25 +157,23 @@ public extension NvimView {
     }
 
     if replacementRange != .notFound {
-      guard let newMarkedPosition = self.ugrid.firstPosition(
-        fromFlatCharIndex: replacementRange.location
-      ) else { return }
-
-      self.markedPosition = newMarkedPosition
-
-      self.log.debug("Deleting \(replacementRange.length) and inputting \(self.markedText!)")
-      try? self.bridge.deleteCharacters(
-        replacementRange.length,
-        andInputEscapedString: self.vimPlainString(self.markedText!)
-      ).wait()
-    } else {
-      self.log.debug("Deleting \(oldMarkedTextLength) and inputting \(self.markedText!)")
-      try? self.bridge.deleteCharacters(
-        oldMarkedTextLength,
-        andInputEscapedString: self.vimPlainString(self.markedText!)
-      ).wait()
+      guard self.ugrid.firstPosition(fromFlatCharIndex: replacementRange.location) != nil else { return }
+      // FIXME: here not validate location, only delete by length.
+      // after delete, cusor should be the location
+    }
+    if replacementRange.length > 0 {
+      try? self.bridge
+      .deleteCharacters(replacementRange.length, andInputEscapedString: "")
+      .wait()
     }
 
+    // delay to wait async gui update handled.
+    // this avoid insert and then delete flicker
+    // the markedPosition is not needed since marked Text should always following cursor..
+    DispatchQueue.main.async { [self, markedText] in
+        ugrid.updateMark(markedText: markedText!, selectedRange: selectedRange)
+        markForRender(region: regionForRow(at: ugrid.cursorPosition))
+    }
     self.keyDownDone = true
   }
 
@@ -189,16 +183,15 @@ public extension NvimView {
   }
 
   func _unmarkText() {
-    let position = self.markedPosition
-    self.ugrid.unmarkCell(at: position)
-    self.markForRender(position: position)
-    if self.ugrid.isNextCellEmpty(position) {
-      self.ugrid.unmarkCell(at: position.advancing(row: 0, column: 1))
-      self.markForRender(position: position.advancing(row: 0, column: 1))
+    guard hasMarkedText() else { return }
+    // wait inserted text gui update event, so hanji in korean get right previous string and can popup candidate window
+    DispatchQueue.main.async { [self] in
+      if let markedInfo = self.ugrid.markedInfo {
+        self.ugrid.markedInfo = nil
+        self.markForRender(region: regionForRow(at: markedInfo.position))
+      }
     }
-
     self.markedText = nil
-    self.markedPosition = .null
   }
 
   /**
@@ -217,7 +210,7 @@ public extension NvimView {
 
     let result: NSRange
     result = NSRange(
-      location: self.ugrid.flatCharIndex(forPosition: self.ugrid.cursorPosition),
+      location: self.ugrid.flatCharIndex(forPosition: self.ugrid.cursorPositionWithMarkedInfo(allowOverflow: true)),
       length: 0
     )
 
@@ -232,7 +225,7 @@ public extension NvimView {
     }
 
     let result = NSRange(
-      location: self.ugrid.flatCharIndex(forPosition: self.markedPosition),
+      location: self.ugrid.flatCharIndex(forPosition: self.ugrid.cursorPosition),
       length: marked.count
     )
 

NvimView/Sources/NvimView/NvimView+UiBridge.swift

@@ -83,7 +83,7 @@ extension NvimView {
   final func flush(_ renderData: [MessagePackValue]) {
     self.bridgeLogger.trace("# of render data: \(renderData.count)")
 
-    gui.async {
+    gui.async { [self] in
       var (recompute, rowStart) = (false, Int.max)
       renderData.forEach { value in
         guard let renderEntry = value.arrayValue else { return }
@@ -109,10 +109,13 @@ extension NvimView {
                 let textPositionRow = innerArray[2].uint64Value,
                 let textPositionCol = innerArray[3].uint64Value else { return }
 
-          self.doGoto(
+          if let possibleNewRowStart = self.doGoto(
             position: Position(row: Int(row), column: Int(col)),
             textPosition: Position(row: Int(textPositionRow), column: Int(textPositionCol))
-          )
+          ) {
+            rowStart = min(rowStart, possibleNewRowStart)
+            recompute = true
+          }
 
         case .scroll:
           let values = innerArray.compactMap(\.intValue)
@@ -131,10 +134,7 @@ extension NvimView {
       }
 
       guard recompute else { return }
-      self.ugrid.recomputeFlatIndices(
-        rowStart: rowStart,
-        rowEndInclusive: self.ugrid.size.height - 1
-      )
+      self.ugrid.recomputeFlatIndices(rowStart: rowStart)
     }
   }
 
@@ -298,7 +298,12 @@ extension NvimView {
     )
 
     if count > 0 {
-      if self.usesLigatures {
+      if row == self.ugrid.markedInfo?.position.row  {
+        self.markForRender(region: Region(
+          top: row, bottom: row,
+          left: startCol, right: self.ugrid.size.width
+          ))
+      } else if self.usesLigatures {
         let leftBoundary = self.ugrid.leftBoundaryOfWord(
           at: Position(row: row, column: startCol)
         )
@@ -321,30 +326,47 @@ extension NvimView {
       ))
     }
 
-    if row == self.markedPosition.row,
-       startCol <= self.markedPosition.column,
-       self.markedPosition.column <= endCol
-    {
-      self.ugrid.markCell(at: self.markedPosition)
-    }
-
     return row
   }
 
-  private func doGoto(position: Position, textPosition: Position) {
+  func regionForRow(at: Position) -> Region {
+    return Region(
+      top: at.row,
+      bottom: at.row,
+      left: at.column,
+      right: ugrid.size.width
+      )
+  }
+
+  private func doGoto(position: Position, textPosition: Position) -> Int? {
     self.bridgeLogger.debug(position)
 
-    // Re-render the old cursor position.
-    self.markForRender(
-      region: self.cursorRegion(for: self.ugrid.cursorPosition)
-    )
+    var rowStart: Int?
+    if var markedInfo = self.ugrid.popMarkedInfo() {
+      rowStart = min(markedInfo.position.row, position.row)
+      self.markForRender(
+        region: self.regionForRow(at: self.ugrid.cursorPosition)
+      )
+      self.ugrid.goto(position)
+      markedInfo.position = position
+      self.ugrid.updateMarkedInfo(newValue: markedInfo)
+      self.markForRender(
+        region: self.regionForRow(at: self.ugrid.cursorPosition)
+      )
+    } else {
+      // Re-render the old cursor position.
+      self.markForRender(
+        region: self.cursorRegion(for: self.ugrid.cursorPosition)
+      )
 
-    self.ugrid.goto(position)
-    self.markForRender(
-      region: self.cursorRegion(for: self.ugrid.cursorPosition)
-    )
+      self.ugrid.goto(position)
+      self.markForRender(
+        region: self.cursorRegion(for: self.ugrid.cursorPosition)
+      )
+    }
 
     self.eventsSubject.onNext(.cursor(textPosition))
+    return rowStart
   }
 
   private func doScroll(_ array: [Int]) -> Int {

NvimView/Sources/NvimView/NvimView.swift

@@ -285,8 +285,6 @@ public class NvimView: NSView,
   let disposeBag = DisposeBag()
 
   var markedText: String?
-  var markedPosition = Position.null
-
   let bridgeLogger = OSLog(subsystem: Defs.loggerSubsystem, category: Defs.LoggerCategory.bridge)
   let log = OSLog(subsystem: Defs.loggerSubsystem, category: Defs.LoggerCategory.view)
 

NvimView/Sources/NvimView/UGrid.swift

@@ -71,61 +71,6 @@ final class UGrid: CustomStringConvertible, Codable {
 
   var hasData: Bool { !self.cells.isEmpty }
 
-  func unmarkCell(at position: Position) {
-    let attrId = self.cells[position.row][position.column].attrId
-
-    guard attrId < CellAttributesCollection.defaultAttributesId
-      || attrId == CellAttributesCollection.reversedDefaultAttributesId
-    else { return }
-
-    let newAttrsId: Int
-    if attrId == CellAttributesCollection.reversedDefaultAttributesId {
-      newAttrsId = CellAttributesCollection.defaultAttributesId
-    } else {
-      newAttrsId = abs(attrId)
-    }
-    self.cells[position.row][position.column].attrId = newAttrsId
-
-    if self.isNextCellEmpty(position) {
-      self.cells[position.row][position.column + 1].attrId = newAttrsId
-    }
-  }
-
-  func markCell(at position: Position) {
-    let attrId = self.cells[position.row][position.column].attrId
-
-    guard attrId >= CellAttributesCollection.defaultAttributesId,
-          attrId != CellAttributesCollection.reversedDefaultAttributesId else { return }
-
-    let newAttrsId: Int
-    if attrId == CellAttributesCollection.defaultAttributesId {
-      newAttrsId = CellAttributesCollection.reversedDefaultAttributesId
-    } else {
-      newAttrsId = (-1) * attrId
-    }
-    self.cells[position.row][position.column].attrId = newAttrsId
-
-    if self.isNextCellEmpty(position) {
-      self.cells[position.row][position.column + 1].attrId = newAttrsId
-    }
-  }
-
-  func position(fromOneDimCellIndex flattenedIndex: Int) -> Position {
-    let row = min(
-      self.size.height - 1,
-      max(0, Int(floor(Double(flattenedIndex) / Double(self.size.width))))
-    )
-    let col = min(self.size.width - 1, max(0, flattenedIndex % self.size.width))
-
-    return Position(row: row, column: col)
-  }
-
-  func oneDimCellIndex(forRow row: Int, column: Int) -> Int { row * self.size.width + column }
-
-  func oneDimCellIndex(forPosition position: Position) -> Int {
-    position.row * self.size.width + position.column
-  }
-
   func flatCharIndex(forPosition position: Position) -> Int {
     self.cells[position.row][position.column].flatCharIndex
   }
@@ -191,6 +136,16 @@ final class UGrid: CustomStringConvertible, Codable {
       stop = region.top - rows - 1
       step = -1
     }
+    var oldMarkedInfo: MarkedInfo?
+    if let row = self.markedInfo?.position.row, region.top <= row && row <= region.bottom  {
+      oldMarkedInfo = popMarkedInfo()
+    }
+    defer {
+      // keep markedInfo position not changed. markedInfo only following cursor position change
+      if let oldMarkedInfo = oldMarkedInfo {
+        updateMarkedInfo(newValue: oldMarkedInfo)
+      }
+    }
 
     // copy cell data
     let rangeWithinRow = region.left...region.right
@@ -241,6 +196,7 @@ final class UGrid: CustomStringConvertible, Codable {
       repeating: UCell(string: clearString, attrId: CellAttributesCollection.defaultAttributesId),
       count: self.size.width
     )
+    updateMarkedInfo(newValue: nil) // everything need to be reset
     self.cells = Array(repeating: emptyRow, count: self.size.height)
   }
 
@@ -277,6 +233,16 @@ final class UGrid: CustomStringConvertible, Codable {
     chunk: [String],
     attrIds: [Int]
   ) {
+    // remove marked patch and recover after modified from vim
+    var oldMarkedInfo: MarkedInfo?
+    if row == self.markedInfo?.position.row {
+      oldMarkedInfo = popMarkedInfo()
+    }
+    defer {
+      if let oldMarkedInfo = oldMarkedInfo {
+        updateMarkedInfo(newValue: oldMarkedInfo)
+      }
+    }
     for column in startCol..<endCol {
       self.cells[row][column].string = chunk[column - startCol]
       self.cells[row][column].attrId = attrIds[column - startCol]
@@ -292,21 +258,90 @@ final class UGrid: CustomStringConvertible, Codable {
       )
     }
   }
+  struct MarkedInfo {
+      var position: Position
+      var markedCell: [UCell]
+      var selectedRange: NSRange // begin from markedCell and calculate by ucell count
+  }
+  var _markedInfo: MarkedInfo?
+  func popMarkedInfo() -> MarkedInfo? {
+      if let markedInfo = _markedInfo {
+          // true clear or just popup
+          updateMarkedInfo(newValue: nil)
+          return markedInfo
+      }
+      return nil
+  }
+  // return changedRowStart. Int.max if no change
+  @discardableResult
+  func updateMarkedInfo(newValue: MarkedInfo?) -> Int {
+    assert(Thread.isMainThread, "should occur on main thread!")
+    var changedRowStart = Int.max
+    if let old = _markedInfo {
+      self.cells[old.position.row].removeSubrange(old.position.column..<(old.position.column+old.markedCell.count))
+      changedRowStart = old.position.row
+    }
+    _markedInfo = newValue
+    if let new = newValue {
+      self.cells[new.position.row].insert(contentsOf: new.markedCell, at: new.position.column)
+      changedRowStart = min(changedRowStart, new.position.row)
+    }
+    return changedRowStart
+  }
+  var markedInfo: MarkedInfo? {
+      get { _markedInfo }
+      set {
+          let changedRowStart = updateMarkedInfo(newValue: newValue)
+          if changedRowStart < self.size.height {
+            recomputeFlatIndices(rowStart: changedRowStart)
+          }
+      }
+  }
+  func cursorPositionWithMarkedInfo(allowOverflow: Bool = false) -> Position {
+    var position: Position = cursorPosition
+    if let markedInfo = markedInfo { position.column += markedInfo.selectedRange.location }
+    if !allowOverflow, position.column >= size.width { position.column = size.width - 1 }
+    return position
+  }
 
-  func recomputeFlatIndices(rowStart: Int, rowEndInclusive: Int) {
+  // marked text insert into cell directly
+  // marked text always following cursor position
+  func updateMark(
+      markedText: String,
+      selectedRange: NSRange
+  ) {
+      assert(Thread.isMainThread, "should occur on main thread!")
+      var selectedRangeByCell = selectedRange
+      let markedTextArray: [String] = markedText.enumerated().reduce(into: []) { (array, pair) in
+          array.append(String(pair.element))
+          if !KeyUtils.isHalfWidth(char: pair.element) {
+              array.append("")
+              if pair.offset < selectedRange.location { selectedRangeByCell.location += 1 }
+              else { selectedRangeByCell.length += 1 }
+          }
+      }
+      let cells = markedTextArray.map {
+          UCell(string: $0, attrId: CellAttributesCollection.markedAttributesId)
+      }
+      self.markedInfo = MarkedInfo(position: cursorPosition, markedCell: cells, selectedRange: selectedRangeByCell)
+
+  }
+
+  func recomputeFlatIndices(rowStart: Int) {
     self.log.debug("Recomputing flat indices from row \(rowStart)")
 
-    var delta = 0
+    var counter = 0
     if rowStart > 0 {
-      delta = self.cells[rowStart - 1][self.size.width - 1].flatCharIndex
-        - self.oneDimCellIndex(forRow: rowStart - 1, column: self.size.width - 1)
+      counter = self.cells[rowStart - 1].last!.flatCharIndex + 1
     }
 
-    for row in rowStart...rowEndInclusive {
-      for column in 0..<self.size.width {
-        if self.cells[row][column].string.isEmpty { delta -= 1 }
-        self.cells[row][column].flatCharIndex = self
-          .oneDimCellIndex(forRow: row, column: column) + delta
+    // should update following char too since previous line is change
+    for row in rowStart...(size.height - 1) {
+      // marked text may overflow size, counter it too
+      for column in self.cells[row].indices {
+        if self.cells[row][column].string.isEmpty { counter -= 1 }
+        self.cells[row][column].flatCharIndex = counter
+        counter += 1
       }
     }
   }

NvimView/Tests/NvimViewTests/UGridTest.swift

@@ -161,48 +161,6 @@ class UGridTest: XCTestCase {
       .to(equal(CellAttributesCollection.reversedDefaultAttributesId))
   }
 
-  func testFlattenedIndex() {
-    self.ugrid.resize(Size(width: 20, height: 10))
-    expect(
-      self.ugrid.oneDimCellIndex(forPosition: Position(row: 0, column: 0))
-    ).to(equal(0))
-    expect(
-      self.ugrid.oneDimCellIndex(forPosition: Position(row: 0, column: 5))
-    ).to(equal(5))
-    expect(
-      self.ugrid.oneDimCellIndex(forPosition: Position(row: 1, column: 0))
-    ).to(equal(20))
-    expect(
-      self.ugrid.oneDimCellIndex(forPosition: Position(row: 1, column: 5))
-    ).to(equal(25))
-    expect(
-      self.ugrid.oneDimCellIndex(forPosition: Position(row: 9, column: 0))
-    ).to(equal(180))
-    expect(
-      self.ugrid.oneDimCellIndex(forPosition: Position(row: 9, column: 19))
-    ).to(equal(199))
-  }
-
-  func testPositionFromFlattenedIndex() {
-    self.ugrid.resize(Size(width: 20, height: 10))
-    expect(self.ugrid.position(fromOneDimCellIndex: 0))
-      .to(equal(Position(row: 0, column: 0)))
-    expect(self.ugrid.position(fromOneDimCellIndex: 5))
-      .to(equal(Position(row: 0, column: 5)))
-    expect(self.ugrid.position(fromOneDimCellIndex: 20))
-      .to(equal(Position(row: 1, column: 0)))
-    expect(self.ugrid.position(fromOneDimCellIndex: 25))
-      .to(equal(Position(row: 1, column: 5)))
-    expect(self.ugrid.position(fromOneDimCellIndex: 180))
-      .to(equal(Position(row: 9, column: 0)))
-    expect(self.ugrid.position(fromOneDimCellIndex: 199))
-      .to(equal(Position(row: 9, column: 19)))
-    expect(self.ugrid.position(fromOneDimCellIndex: 418))
-      .to(equal(Position(row: 9, column: 18)))
-    expect(self.ugrid.position(fromOneDimCellIndex: 419))
-      .to(equal(Position(row: 9, column: 19)))
-  }
-
   func testLeftBoundaryOfWord() {
     self.ugrid.resize(Size(width: 10, height: 2))
     self.ugrid.update(