@@ -88,11 +88,11 @@ pub fn move_next_word_start(slice: RopeSlice, mut begin: usize, count: usize) ->
8888
8989 if is_word(ch) {
9090 skip_over_next(slice, &mut end, is_word);
91− } else if ch.is_ascii_punctuation() {
92− skip_over_next(slice, &mut end, |ch| ch.is_ascii_punctuation());
91+ } else if is_punctuation(ch) {
92+ skip_over_next(slice, &mut end, is_punctuation);
9393 }
9494
95− skip_over_next(slice, &mut end, is_horiz_blank);
95+ skip_over_next(slice, &mut end, char::is_whitespace);
9696 }
9797
9898 Some(Range::new(begin, end - 1))
@@ -119,15 +119,15 @@ pub fn move_prev_word_start(slice: RopeSlice, mut begin: usize, count: usize) ->
119119
120120 end = begin;
121121
122− with_end = skip_over_prev(slice, &mut end, is_horiz_blank);
122+ with_end = skip_over_prev(slice, &mut end, char::is_whitespace);
123123
124124 // refetch
125125 let ch = slice.char(end);
126126
127127 if is_word(ch) {
128128 with_end = skip_over_prev(slice, &mut end, is_word);
129− } else if ch.is_ascii_punctuation() {
130− with_end = skip_over_prev(slice, &mut end, |ch| ch.is_ascii_punctuation());
129+ } else if is_punctuation(ch) {
130+ with_end = skip_over_prev(slice, &mut end, is_punctuation);
131131 }
132132 }
133133
@@ -155,15 +155,15 @@ pub fn move_next_word_end(slice: RopeSlice, mut begin: usize, count: usize) -> O
155155
156156 end = begin;
157157
158− skip_over_next(slice, &mut end, is_horiz_blank);
158+ skip_over_next(slice, &mut end, char::is_whitespace);
159159
160160 // refetch
161161 let ch = slice.char(end);
162162
163163 if is_word(ch) {
164164 skip_over_next(slice, &mut end, is_word);
165− } else if ch.is_ascii_punctuation() {
166− skip_over_next(slice, &mut end, |ch| ch.is_ascii_punctuation());
165+ } else if is_punctuation(ch) {
166+ skip_over_next(slice, &mut end, is_punctuation);
167167 }
168168 }
169169
@@ -174,12 +174,28 @@ pub fn move_next_word_end(slice: RopeSlice, mut begin: usize, count: usize) -> O
174174
175175// used for by-word movement
176176
177+#[inline]
177178pub(crate) fn is_word(ch: char) -> bool {
178179 ch.is_alphanumeric() || ch == '_'
179180}
180181
181−pub(crate) fn is_horiz_blank(ch: char) -> bool {
182− matches!(ch, ' ' | '\t')
182+#[inline]
183+pub(crate) fn is_punctuation(ch: char) -> bool {
184+ use unicode_general_category::{get_general_category, GeneralCategory};
185+
186+ matches!(
187+ get_general_category(ch),
188+ GeneralCategory::OtherPunctuation
189+ | GeneralCategory::OpenPunctuation
190+ | GeneralCategory::ClosePunctuation
191+ | GeneralCategory::InitialPunctuation
192+ | GeneralCategory::FinalPunctuation
193+ | GeneralCategory::ConnectorPunctuation
194+ | GeneralCategory::DashPunctuation
195+ | GeneralCategory::MathSymbol
196+ | GeneralCategory::CurrencySymbol
197+ | GeneralCategory::ModifierSymbol
198+ )
183199}
184200
185201#[derive(Debug, Eq, PartialEq)]
@@ -191,14 +207,15 @@ pub(crate) enum Category {
191207 Unknown,
192208}
193209
210+#[inline]
194211pub(crate) fn categorize(ch: char) -> Category {
195212 if ch == '\n' {
196213 Category::Eol
197− } else if ch.is_ascii_whitespace() {
214+ } else if ch.is_whitespace() {
198215 Category::Whitespace
199216 } else if is_word(ch) {
200217 Category::Word
201− } else if ch.is_ascii_punctuation() {
218+ } else if is_punctuation(ch) {
202219 Category::Punctuation
203220 } else {
204221 Category::Unknown
@@ -213,6 +230,7 @@ where
213230{
214231 let mut chars = slice.chars_at(*pos);
215232
233+ #[allow(clippy::while_let_on_iterator)]
216234 while let Some(ch) = chars.next() {
217235 if !fun(ch) {
218236 break;
@@ -231,6 +249,7 @@ where
231249 // need to +1 so that prev() includes current char
232250 let mut chars = slice.chars_at(*pos + 1);
233251
252+ #[allow(clippy::while_let_on_iterator)]
234253 while let Some(ch) = chars.prev() {
235254 if !fun(ch) {
236255 break;
@@ -259,4 +278,44 @@ mod test {
259278 (1, 2).into()
260279 );
261280 }
281+
282+ #[test]
283+ fn test_categorize() {
284+ const WORD_TEST_CASE: &'static str =
285+ "_hello_world_あいうえおー1234567890１２３４５６７８９０";
286+ const PUNCTUATION_TEST_CASE: &'static str = "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
287+ const WHITESPACE_TEST_CASE: &'static str = " ";
288+
289+ assert_eq!(Category::Eol, categorize('\n'));
290+
291+ for ch in WHITESPACE_TEST_CASE.chars() {
292+ assert_eq!(
293+ Category::Whitespace,
294+ categorize(ch),
295+ "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
296+ ch,
297+ categorize(ch)
298+ );
299+ }
300+
301+ for ch in WORD_TEST_CASE.chars() {
302+ assert_eq!(
303+ Category::Word,
304+ categorize(ch),
305+ "Testing '{}', but got `{:?}` instead of `Category::Word`",
306+ ch,
307+ categorize(ch)
308+ );
309+ }
310+
311+ for ch in PUNCTUATION_TEST_CASE.chars() {
312+ assert_eq!(
313+ Category::Punctuation,
314+ categorize(ch),
315+ "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
316+ ch,
317+ categorize(ch)
318+ );
319+ }
320+ }
262321}