b873fb9 · helix · git.iko.soy

git.iko.soy/helix

on master help sign in

Fix Unicode (#135)

b873fb9Kirawi5 years ago

Cargo.lock+7

compare with master

unifiedsplit

@@ -265,6 +265,7 @@ dependencies = [

265 "tendril",

266 "toml",

267 "tree-sitter",

268+ "unicode-general-category",

268 "unicode-segmentation",

269 "unicode-segmentation",

269 "unicode-width",

270 "unicode-width",

270]

271]

@@ -969,6 +970,12 @@ dependencies = [

969 "matches",

970 "matches",

970]

971]

971

972

973+[[package]]

974+name = "unicode-general-category"

975+version = "0.4.0"

976+source = "registry+https://github.com/rust-lang/crates.io-index"

977+checksum = "07547e3ee45e28326cc23faac56d44f58f16ab23e413db526debce3b0bfd2742"

978+

972[[package]]

979[[package]]

973name = "unicode-normalization"

980name = "unicode-normalization"

974version = "0.1.19"

981version = "0.1.19"

helix-core/Cargo.toml+1

@@ -19,6 +19,7 @@ smallvec = "1.4"

19tendril = "0.4.2"

20unicode-segmentation = "1.6"

21unicode-width = "0.1"

22+unicode-general-category = "0.4.0"

22# slab = "0.4.2"

23# slab = "0.4.2"

23tree-sitter = "0.19"

24tree-sitter = "0.19"

24once_cell = "1.4"

25once_cell = "1.4"

helix-core/src/movement.rs+72−13

@@ -88,11 +88,11 @@ pub fn move_next_word_start(slice: RopeSlice, mut begin: usize, count: usize) ->

89 if is_word(ch) {

90 skip_over_next(slice, &mut end, is_word);

91− } else if ch.is_ascii_punctuation() {

91+ } else if is_punctuation(ch) {

92− skip_over_next(slice, &mut end, |ch| ch.is_ascii_punctuation());

92+ skip_over_next(slice, &mut end, is_punctuation);

93 }

95− skip_over_next(slice, &mut end, is_horiz_blank);

95+ skip_over_next(slice, &mut end, char::is_whitespace);

96 }

98 Some(Range::new(begin, end - 1))

@@ -119,15 +119,15 @@ pub fn move_prev_word_start(slice: RopeSlice, mut begin: usize, count: usize) ->

119

120 end = begin;

121

122− with_end = skip_over_prev(slice, &mut end, is_horiz_blank);

122+ with_end = skip_over_prev(slice, &mut end, char::is_whitespace);

123

124 // refetch

125 let ch = slice.char(end);

126

127 if is_word(ch) {

128 with_end = skip_over_prev(slice, &mut end, is_word);

129− } else if ch.is_ascii_punctuation() {

129+ } else if is_punctuation(ch) {

130− with_end = skip_over_prev(slice, &mut end, |ch| ch.is_ascii_punctuation());

130+ with_end = skip_over_prev(slice, &mut end, is_punctuation);

131 }

132 }

133

@@ -155,15 +155,15 @@ pub fn move_next_word_end(slice: RopeSlice, mut begin: usize, count: usize) -> O

155

156 end = begin;

157

158− skip_over_next(slice, &mut end, is_horiz_blank);

158+ skip_over_next(slice, &mut end, char::is_whitespace);

159

160 // refetch

161 let ch = slice.char(end);

162

163 if is_word(ch) {

164 skip_over_next(slice, &mut end, is_word);

165− } else if ch.is_ascii_punctuation() {

165+ } else if is_punctuation(ch) {

166− skip_over_next(slice, &mut end, |ch| ch.is_ascii_punctuation());

166+ skip_over_next(slice, &mut end, is_punctuation);

167 }

168 }

169

@@ -174,12 +174,28 @@ pub fn move_next_word_end(slice: RopeSlice, mut begin: usize, count: usize) -> O

174

175// used for by-word movement

176

177+#[inline]

177pub(crate) fn is_word(ch: char) -> bool {

178pub(crate) fn is_word(ch: char) -> bool {

178 ch.is_alphanumeric() || ch == '_'

179 ch.is_alphanumeric() || ch == '_'

179}

180}

180

181

182+#[inline]

181−pub(crate) fn is_horiz_blank(ch: char) -> bool {

183+pub(crate) fn is_punctuation(ch: char) -> bool {

184+ use unicode_general_category::{get_general_category, GeneralCategory};

185+

182− matches!(ch, ' ' | '\t')

186+ matches!(

187+ get_general_category(ch),

188+ GeneralCategory::OtherPunctuation

189+ | GeneralCategory::OpenPunctuation

190+ | GeneralCategory::ClosePunctuation

191+ | GeneralCategory::InitialPunctuation

192+ | GeneralCategory::FinalPunctuation

193+ | GeneralCategory::ConnectorPunctuation

194+ | GeneralCategory::DashPunctuation

195+ | GeneralCategory::MathSymbol

196+ | GeneralCategory::CurrencySymbol

197+ | GeneralCategory::ModifierSymbol

198+ )

183}

199}

184

200

185#[derive(Debug, Eq, PartialEq)]

201#[derive(Debug, Eq, PartialEq)]

@@ -191,14 +207,15 @@ pub(crate) enum Category {

191 Unknown,

207 Unknown,

192}

208}

193

209

210+#[inline]

194pub(crate) fn categorize(ch: char) -> Category {

211pub(crate) fn categorize(ch: char) -> Category {

195 if ch == '\n' {

212 if ch == '\n' {

196 Category::Eol

213 Category::Eol

197− } else if ch.is_ascii_whitespace() {

214+ } else if ch.is_whitespace() {

198 Category::Whitespace

215 Category::Whitespace

199 } else if is_word(ch) {

216 } else if is_word(ch) {

200 Category::Word

217 Category::Word

201− } else if ch.is_ascii_punctuation() {

218+ } else if is_punctuation(ch) {

202 Category::Punctuation

219 Category::Punctuation

203 } else {

220 } else {

204 Category::Unknown

221 Category::Unknown

@@ -213,6 +230,7 @@ where

213{

230{

214 let mut chars = slice.chars_at(*pos);

231 let mut chars = slice.chars_at(*pos);

215

232

233+ #[allow(clippy::while_let_on_iterator)]

216 while let Some(ch) = chars.next() {

234 while let Some(ch) = chars.next() {

217 if !fun(ch) {

235 if !fun(ch) {

218 break;

236 break;

@@ -231,6 +249,7 @@ where

231 // need to +1 so that prev() includes current char

249 // need to +1 so that prev() includes current char

232 let mut chars = slice.chars_at(*pos + 1);

250 let mut chars = slice.chars_at(*pos + 1);

233

251

252+ #[allow(clippy::while_let_on_iterator)]

234 while let Some(ch) = chars.prev() {

253 while let Some(ch) = chars.prev() {

235 if !fun(ch) {

254 if !fun(ch) {

236 break;

255 break;

@@ -259,4 +278,44 @@ mod test {

259 (1, 2).into()

278 (1, 2).into()

260 );

279 );

261 }

280 }

281+

282+ #[test]

283+ fn test_categorize() {

284+ const WORD_TEST_CASE: &'static str =

285+ "_hello_world_あいうえおー1234567890１２３４５６７８９０";

286+ const PUNCTUATION_TEST_CASE: &'static str = "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";

287+ const WHITESPACE_TEST_CASE: &'static str = "  　  ";

288+

289+ assert_eq!(Category::Eol, categorize('\n'));

290+

291+ for ch in WHITESPACE_TEST_CASE.chars() {

292+ assert_eq!(

293+ Category::Whitespace,

294+ categorize(ch),

295+ "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",

296+ ch,

297+ categorize(ch)

298+ );

299+ }

300+

301+ for ch in WORD_TEST_CASE.chars() {

302+ assert_eq!(

303+ Category::Word,

304+ categorize(ch),

305+ "Testing '{}', but got `{:?}` instead of `Category::Word`",

306+ ch,

307+ categorize(ch)

308+ );

309+ }

310+

311+ for ch in PUNCTUATION_TEST_CASE.chars() {

312+ assert_eq!(

313+ Category::Punctuation,

314+ categorize(ch),

315+ "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",

316+ ch,

317+ categorize(ch)

318+ );

319+ }

320+ }

262}

321}

helix-core/src/transaction.rs+1−1

@@ -758,7 +758,7 @@ mod test {

758

759 #[test]

760 fn combine_with_utf8() {

761− const TEST_CASE: &'static str = "Hello, これはヒレクスエディターです！";

761+ const TEST_CASE: &'static str = "Hello, これはヘリックスエディターです！";

762

763 let empty = Rope::from("");

764 let mut a = ChangeSet::new(&empty);

helix-core/src/words.rs+7−7

@@ -1,4 +1,4 @@

1−use crate::movement::{categorize, is_horiz_blank, is_word, skip_over_prev};

1+use crate::movement::{categorize, is_punctuation, is_word, skip_over_prev};

2use ropey::RopeSlice;

4#[must_use]

@@ -13,15 +13,15 @@ pub fn nth_prev_word_boundary(slice: RopeSlice, mut char_idx: usize, count: usiz

13 // return if not skip while?

14 skip_over_prev(slice, &mut char_idx, |ch| ch == '\n');

16− with_end = skip_over_prev(slice, &mut char_idx, is_horiz_blank);

16+ with_end = skip_over_prev(slice, &mut char_idx, char::is_whitespace);

18 // refetch

19 let ch = slice.char(char_idx);

21 if is_word(ch) {

22 with_end = skip_over_prev(slice, &mut char_idx, is_word);

23− } else if ch.is_ascii_punctuation() {

23+ } else if is_punctuation(ch) {

24− with_end = skip_over_prev(slice, &mut char_idx, |ch| ch.is_ascii_punctuation());

24+ with_end = skip_over_prev(slice, &mut char_idx, is_punctuation);

25 }

26 }

@@ -47,11 +47,11 @@ fn different_prev_word_boundary() {

47 t("hello, world", "hello, ");

48 t("hello, ", "hello");

49 t("hello", "");

50− t("こんにちは、世界！", "こんにちは、世界！"); // TODO: punctuation

50+ t("こんにちは、世界！", "こんにちは、世界");

51 t("こんにちは、世界", "こんにちは、");

52− t("こんにちは、", "こんにちは、"); // what?

52+ t("こんにちは、", "こんにちは");

53 t("こんにちは", "");

54− t("この世界。", "この世界。"); // what?

54+ t("この世界。", "この世界");

55 t("この世界", "");

56 t("お前はもう死んでいる", "");

57 t("その300円です", ""); // TODO: should stop at 300

helix-term/src/commands.rs+5−4

@@ -654,9 +654,10 @@ pub fn split_selection_on_newline(cx: &mut Context) {

654fn _search(doc: &mut Document, view: &mut View, contents: &str, regex: &Regex, extend: bool) {

655 let text = doc.text();

656 let selection = doc.selection(view.id);

657− let start = selection.cursor();

657+ let start = text.char_to_byte(selection.cursor());

658

659 // use find_at to find the next match after the cursor, loop around the end

660+ // Careful, `Regex` uses `bytes` as offsets, not character indices!

660 let mat = regex

661 let mat = regex

661 .find_at(contents, start)

662 .find_at(contents, start)

662 .or_else(|| regex.find(contents));

663 .or_else(|| regex.find(contents));

@@ -670,7 +671,7 @@ fn _search(doc: &mut Document, view: &mut View, contents: &str, regex: &Regex, e

670 return;

671 return;

671 }

672 }

672

673

673− let head = end;

674+ let head = end - 1;

674

675

675 let selection = if extend {

676 let selection = if extend {

676 selection.clone().push(Range::new(start, head))

677 selection.clone().push(Range::new(start, head))

@@ -1027,7 +1028,7 @@ pub fn command_mode(cx: &mut Context) {

1027 let mut prompt = Prompt::new(

1028 let mut prompt = Prompt::new(

1028 ":".to_owned(),

1029 ":".to_owned(),

1029 |input: &str| {

1030 |input: &str| {

1030− // we use .this over split_ascii_whitespace() because we care about empty segments

1031+ // we use .this over split_whitespace() because we care about empty segments

1031 let parts = input.split(' ').collect::<Vec<&str>>();

1032 let parts = input.split(' ').collect::<Vec<&str>>();

1032

1033

1033 // simple heuristic: if there's no just one part, complete command name.

1034 // simple heuristic: if there's no just one part, complete command name.

@@ -1069,7 +1070,7 @@ pub fn command_mode(cx: &mut Context) {

1069 return;

1070 return;

1070 }

1071 }

1071

1072

1072− let parts = input.split_ascii_whitespace().collect::<Vec<&str>>();

1073+ let parts = input.split_whitespace().collect::<Vec<&str>>();

1073 if parts.is_empty() {

1074 if parts.is_empty() {

1074 return;

1075 return;

1075 }

1076 }

helix-view/src/view.rs+1−1

@@ -106,7 +106,7 @@ impl View {

106 /// Calculates the last visible line on screen

107 #[inline]

108 pub fn last_line(&self, doc: &Document) -> usize {

109− let height = self.area.height.saturating_sub(2); // - 2 for statusline

109+ let height = self.area.height.saturating_sub(1); // - 1 for statusline

110 std::cmp::min(

111 self.first_line + height as usize,

112 doc.text().len_lines() - 1,