Get rid of the option to use the system wcwidth

The system wcwidth() is often wrong. Not to mention that if you SSH into
a different machine, then you have a potentially different wcwidth. The
only sane way to deal with this is to use the unicode standard.

README.asciidoc

@@ -407,25 +407,21 @@ brew or MacPorts as well.
 
 === Some special symbols are rendered small/truncated in kitty?
 
-The number of cells a unicode character takes up are controlled by the
-`wcwidth()` system function. If wcwidth() returns 2 then kitty will render the
-character in two cells, otherwise it will render it in one cell. Often the
-system `wcwidth()` is old/outdated. You can use the `use_system_wcwidth=no`
-setting in your kitty.conf to workaround this. But note that it might cause
-other issues, since now kitty and the programs running inside it may not agree
-on how wide characters should be. When a symbol does not fit, it will either be
-rescaled to be smaller or truncated (depending on how much extra space it
-needs). This is often different from other terminals which just let the
-character overflow into neighboring cells, leading to ugly artifacts.
+The number of cells a unicode character takes up are controlled by the unicode
+standard.  All characters are rendered in a single cell unless the unicode
+standard says they should be rendered in two cells. When a symbol does not fit,
+it will either be rescaled to be smaller or truncated (depending on how much
+extra space it needs). This is often different from other terminals which just
+let the character overflow into neighboring cells, which is fine if the
+neighboring cell is empty, but looks terrible if it is not.
 
-In addition to the problem with `wcwidth()` above, some programs, like
-powerline, vim with fancy gutter symbols/status-bar, etc. use unicode
-characters from the private use area to represent symbols. Often these symbols
-are square and should be rendered in two cells.  However, since private use
-area symbols all have `wcwdith() == 1` kitty renders them either smaller or
-truncated. The correct solution for this is to use either use different symbols
-that are not square, or to use a font that defines ligatures with the space
-character for these symbols. See
+Some programs, like powerline, vim with fancy gutter symbols/status-bar, etc.
+use unicode characters from the private use area to represent symbols. Often
+these symbols are square and should be rendered in two cells.  However, since
+private use area symbols all have their width set to one in the unicode
+standard, kitty renders them either smaller or truncated. The correct solution
+for this is to use either use different symbols that are not square, or to use
+a font that defines ligatures with the space character for these symbols. See
 link:https://github.com/kovidgoyal/kitty/issues/182[#182] for a discussion of
 the approach using ligatures.
 

gen-wcwidth.py

@@ -221,6 +221,7 @@ def add(p, comment, chars_, ret):
         p('\tswitch(code) {')
 
         non_printing = class_maps['Cc'] | class_maps['Cf'] | class_maps['Cs']
+        add(p, 'Null', {0}, 0)
         add(p, 'Non-printing characters', non_printing, -1)
         add(p, 'Marks', marks, -1)
         add(p, 'Private use', class_maps['Co'], -3)

kitty/config.py

@@ -273,7 +273,6 @@ def url_style(x):
     'remember_window_size': to_bool,
     'initial_window_width': positive_int,
     'initial_window_height': positive_int,
-    'use_system_wcwidth': to_bool,
     'macos_hide_titlebar': to_bool,
     'macos_option_as_alt': to_bool,
     'box_drawing_scale': box_drawing_scale,

kitty/data-types.c

@@ -75,12 +75,6 @@ wcwidth_wrap(PyObject UNUSED *self, PyObject *chr) {
     return PyLong_FromUnsignedLong(safe_wcwidth(PyLong_AsLong(chr)));
 }
 
-static PyObject*
-change_wcwidth_wrap(PyObject UNUSED *self, PyObject *use9) {
-    change_wcwidth(PyObject_IsTrue(use9));
-    Py_RETURN_NONE;
-}
-
 static PyObject*
 redirect_std_streams(PyObject UNUSED *self, PyObject *args) {
     char *devnull = NULL;
@@ -147,7 +141,6 @@ static PyMethodDef module_methods[] = {
     {"parse_bytes_dump", (PyCFunction)parse_bytes_dump, METH_VARARGS, ""},
     {"redirect_std_streams", (PyCFunction)redirect_std_streams, METH_VARARGS, ""},
     {"wcwidth", (PyCFunction)wcwidth_wrap, METH_O, ""},
-    {"change_wcwidth", (PyCFunction)change_wcwidth_wrap, METH_O, ""},
     {"install_sigchld_handler", (PyCFunction)install_sigchld_handler, METH_NOARGS, ""},
 #ifdef __APPLE__
     METHODB(user_cache_dir, METH_NOARGS),

kitty/data-types.h

@@ -256,7 +256,6 @@ color_type colorprofile_to_color(ColorProfile *self, color_type entry, color_typ
 void copy_color_table_to_buffer(ColorProfile *self, color_type *address, int offset, size_t stride);
 
 unsigned int safe_wcwidth(uint32_t ch);
-void change_wcwidth(bool use9);
 void set_mouse_cursor(MouseShape);
 void mouse_event(int, int);
 void focus_in_event();

kitty/emoji.h

@@ -1,4 +1,4 @@
-// unicode data, built from the unicode standard on: 2018-01-18
+// unicode data, built from the unicode standard on: 2018-02-04
 // see gen-wcwidth.py
 #pragma once
 #include "data-types.h"

kitty/fonts/render.py

@@ -10,7 +10,7 @@
 from kitty.config import defaults
 from kitty.constants import is_macos
 from kitty.fast_data_types import (
-    Screen, change_wcwidth, get_fallback_font, send_prerendered_sprites,
+    Screen, get_fallback_font, send_prerendered_sprites,
     set_font, set_font_size, set_logical_dpi, set_options,
     set_send_sprite_to_gpu, sprite_map_set_limits, test_render_line,
     test_shape
@@ -260,7 +260,6 @@ def test_fallback_font(qtext=None, bold=False, italic=False):
 
 
 def showcase():
-    change_wcwidth(True)
     f = 'monospace' if is_macos else 'Liberation Mono'
     test_render_string('He\u0347\u0305llo\u0337, w\u0302or\u0306l\u0354d!', family=f)
     test_render_string('你好,世界', family=f)

kitty/kitty.conf

@@ -162,15 +162,6 @@ rectangle_select_modifiers ctrl+alt
 # Note that this even works over ssh connections.
 allow_remote_control no
 
-# Choose whether to use the system implementation of wcwidth() (used to
-# control how many cells a character is rendered in).  If you use the system
-# implementation, then kitty and any programs running in it will agree. The
-# problem is that system implementations often are based on outdated unicode
-# standards and get the width of many characters, such as emoji, wrong. So if
-# you are using kitty with programs that have their own up-to-date wcwidth()
-# implementation, set this option to no, otherwise set it to yes.
-use_system_wcwidth no
-
 # The value of the TERM environment variable to set
 term xterm-kitty
 

kitty/main.py

@@ -14,7 +14,7 @@
 from .config import initial_window_size, load_cached_values, save_cached_values
 from .constants import appname, glfw_path, is_macos, is_wayland, logo_data_file
 from .fast_data_types import (
-    change_wcwidth, create_os_window, glfw_init, glfw_terminate,
+    create_os_window, glfw_init, glfw_terminate,
     install_sigchld_handler, set_default_window_icon, set_options, show_window
 )
 from .fonts.box_drawing import set_scale
@@ -146,7 +146,6 @@ def main():
             single_instance.socket.sendall(data)
             return
     opts = create_opts(args)
-    change_wcwidth(not opts.use_system_wcwidth)
     init_graphics()
     try:
         with setup_profiling(args):

kitty/screen.c

@@ -271,20 +271,13 @@ screen_designate_charset(Screen *self, uint32_t which, uint32_t as) {
     }
 }
 
-static int (*wcwidth_impl)(wchar_t) = wcwidth;
-
 unsigned int
 safe_wcwidth(uint32_t ch) {
-    int ans = wcwidth_impl(ch);
+    int ans = wcwidth_std(ch);
     if (ans < 0) ans = 1;
     return MIN(2, ans);
 }
 
-void
-change_wcwidth(bool use_std) {
-    wcwidth_impl = use_std ? wcwidth_std : wcwidth;
-}
-
 
 void
 screen_draw(Screen *self, uint32_t och) {
@@ -1729,6 +1722,12 @@ COUNT_WRAP(cursor_down)
 COUNT_WRAP(cursor_down1)
 COUNT_WRAP(cursor_forward)
 
+static PyObject*
+wcwidth_wrap(PyObject UNUSED *self, PyObject *chr) {
+    return PyLong_FromUnsignedLong(wcwidth_std(PyLong_AsLong(chr)));
+}
+
+
 #define MND(name, args) {#name, (PyCFunction)name, args, #name},
 #define MODEFUNC(name) MND(name, METH_NOARGS) MND(set_##name, METH_O)
 
@@ -1757,6 +1756,7 @@ static PyMethodDef methods[] = {
     MND(cursor_down, METH_VARARGS)
     MND(cursor_down1, METH_VARARGS)
     MND(cursor_forward, METH_VARARGS)
+    {"wcwidth", (PyCFunction)wcwidth_wrap, METH_O, ""},
     {"wcswidth", (PyCFunction)screen_wcswidth, METH_O, ""},
     {"index", (PyCFunction)xxx_index, METH_VARARGS, ""},
     MND(refresh_sprite_positions, METH_NOARGS)

kitty/unicode-data.c

@@ -1,4 +1,4 @@
-// unicode data, built from the unicode standard on: 2018-01-18
+// unicode data, built from the unicode standard on: 2018-02-04
 // see gen-wcwidth.py
 #include "data-types.h"
 

kitty/wcwidth-std.h

@@ -1,4 +1,4 @@
-// unicode data, built from the unicode standard on: 2018-01-18
+// unicode data, built from the unicode standard on: 2018-02-04
 // see gen-wcwidth.py
 #pragma once
 #include "data-types.h"
@@ -8,8 +8,13 @@ START_ALLOW_CASE_RANGE
 static int
 wcwidth_std(int32_t code) {
 	switch(code) {
-		// Non-printing characters (2264 codepoints) {{{
-		case 0x0 ... 0x1f:
+		// Null (1 codepoints) {{{
+		case 0x0:
+			return 0;
+		// }}}
+
+		// Non-printing characters (2263 codepoints) {{{
+		case 0x1 ... 0x1f:
 			return -1;
 		case 0x7f ... 0x9f:
 			return -1;

kitty_tests/datatypes.py

@@ -2,9 +2,6 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
-from unittest import skipIf
-
 from kitty.config import build_ansi_color_table, defaults
 from kitty.fast_data_types import (
     REVERSE, ColorProfile, Cursor as C, HistoryBuf, LineBuf
@@ -334,7 +331,6 @@ def test_rewrap_narrower(self):
         lb2 = self.line_comparison_rewrap(lb, '123', '  a', 'bcd', 'e')
         self.assertContinued(lb2, False, True, True, True)
 
-    @skipIf('ANCIENT_WCWIDTH' in os.environ, 'wcwidth() is too old')
     def test_utils(self):
         self.ae(tuple(map(wcwidth, 'a1\0コニチ ')), (1, 1, 0, 2, 2, 2, 1))
         self.assertEqual(sanitize_title('a\0\01 \t\n\f\rb'), 'a b')

kitty_tests/fonts.py

@@ -6,7 +6,7 @@
 
 from kitty.constants import is_macos
 from kitty.fast_data_types import (
-    change_wcwidth, set_logical_dpi, set_send_sprite_to_gpu,
+    set_logical_dpi, set_send_sprite_to_gpu,
     sprite_map_set_layout, sprite_map_set_limits, test_render_line,
     test_sprite_position_for, wcwidth
 )
@@ -72,21 +72,17 @@ def test_font_rendering(self):
         self.ae(len(cells), sz)
 
     def test_shaping(self):
-        change_wcwidth(True)
-        try:
 
-            def groups(text, path=None):
-                return [x[:2] for x in shape_string(text, path=path)]
+        def groups(text, path=None):
+            return [x[:2] for x in shape_string(text, path=path)]
 
-            self.ae(groups('abcd'), [(1, 1) for i in range(4)])
-            self.ae(groups('A=>>B!=C', path='kitty_tests/FiraCode-Medium.otf'), [(1, 1), (3, 3), (1, 1), (2, 2), (1, 1)])
-            self.ae(groups('==!=<>==<><><>', path='kitty_tests/FiraCode-Medium.otf'), [(2, 2), (2, 2), (2, 2), (2, 2), (2, 2), (2, 2), (2, 2)])
-            colon_glyph = shape_string('9:30', path='kitty_tests/FiraCode-Medium.otf')[1][2]
-            self.assertNotEqual(colon_glyph, shape_string(':', path='kitty_tests/FiraCode-Medium.otf')[0][2])
-            self.ae(colon_glyph, 998)
-            self.ae(groups('9:30', path='kitty_tests/FiraCode-Medium.otf'), [(1, 1), (1, 1), (1, 1), (1, 1)])
-            self.ae(groups('|\U0001F601|\U0001F64f|\U0001F63a|'), [(1, 1), (2, 1), (1, 1), (2, 1), (1, 1), (2, 1), (1, 1)])
-            self.ae(groups('He\u0347\u0305llo\u0337,', path='kitty_tests/LiberationMono-Regular.ttf'),
-                    [(1, 1), (1, 3), (1, 1), (1, 1), (1, 2), (1, 1)])
-        finally:
-            change_wcwidth(False)
+        self.ae(groups('abcd'), [(1, 1) for i in range(4)])
+        self.ae(groups('A=>>B!=C', path='kitty_tests/FiraCode-Medium.otf'), [(1, 1), (3, 3), (1, 1), (2, 2), (1, 1)])
+        self.ae(groups('==!=<>==<><><>', path='kitty_tests/FiraCode-Medium.otf'), [(2, 2), (2, 2), (2, 2), (2, 2), (2, 2), (2, 2), (2, 2)])
+        colon_glyph = shape_string('9:30', path='kitty_tests/FiraCode-Medium.otf')[1][2]
+        self.assertNotEqual(colon_glyph, shape_string(':', path='kitty_tests/FiraCode-Medium.otf')[0][2])
+        self.ae(colon_glyph, 998)
+        self.ae(groups('9:30', path='kitty_tests/FiraCode-Medium.otf'), [(1, 1), (1, 1), (1, 1), (1, 1)])
+        self.ae(groups('|\U0001F601|\U0001F64f|\U0001F63a|'), [(1, 1), (2, 1), (1, 1), (2, 1), (1, 1), (2, 1), (1, 1)])
+        self.ae(groups('He\u0347\u0305llo\u0337,', path='kitty_tests/LiberationMono-Regular.ttf'),
+                [(1, 1), (1, 3), (1, 1), (1, 1), (1, 2), (1, 1)])

kitty_tests/parser.py

@@ -2,10 +2,8 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
 from binascii import hexlify
 from functools import partial
-from unittest import skipIf
 
 from kitty.fast_data_types import CURSOR_BLOCK, parse_bytes, parse_bytes_dump
 
@@ -41,7 +39,6 @@ def parse_bytes_dump(self, s, x, *cmds):
             q.append(('draw', current))
         self.ae(tuple(q), cmds)
 
-    @skipIf('ANCIENT_WCWIDTH' in os.environ, 'wcwidth() is too old')
     def test_simple_parsing(self):
         s = self.create_screen()
         pb = partial(self.parse_bytes_dump, s)

kitty_tests/screen.py

@@ -2,9 +2,6 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
-from unittest import skipIf
-
 from . import BaseTest
 from kitty.fast_data_types import DECAWM, IRM, Cursor, DECCOLM, DECOM
 
@@ -50,7 +47,6 @@ def test_draw_fast(self):
         self.ae(str(s.line(4)), 'ab123')
         self.ae((s.cursor.x, s.cursor.y), (2, 4))
 
-    @skipIf('ANCIENT_WCWIDTH' in os.environ, 'wcwidth() is too old')
     def test_draw_char(self):
         # Test in line-wrap, non-insert mode
         s = self.create_screen()
@@ -93,7 +89,6 @@ def test_draw_char(self):
         self.ae(str(s.line(4)), 'a\u0306b1\u030623')
         self.ae((s.cursor.x, s.cursor.y), (2, 4))
 
-    @skipIf('ANCIENT_WCWIDTH' in os.environ, 'wcwidth() is too old')
     def test_char_manipulation(self):
         s = self.create_screen()