Refactor VT parser for more speed

No longer copy bytes into a separate buffer, instead parse them in place
in the read buffer
This commit is contained in:
Kovid Goyal 2023-11-05 10:24:00 +05:30
parent 23bb2e1b67
commit 6205fb32fd
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
19 changed files with 1037 additions and 1015 deletions

View File

@ -242,6 +242,8 @@ def __call__(self, window_id: int, what: str, *a: Any) -> None:
elif what == 'bytes':
self.dump_bytes_to.write(a[0])
self.dump_bytes_to.flush()
elif what == 'error':
log_error(*a)
else:
if self.draw_dump_buf:
safe_print('draw', ''.join(self.draw_dump_buf))

View File

@ -39,8 +39,6 @@ extern PyTypeObject Screen_Type;
#endif
#define USE_RENDER_FRAMES (global_state.has_render_frames && OPT(sync_to_monitor))
static void (*parse_func)(Screen*, PyObject*, monotonic_t);
typedef struct {
char *data;
size_t sz;
@ -60,6 +58,7 @@ typedef struct {
Message *messages;
size_t messages_capacity, messages_count;
LoopData io_loop_data;
void (*parse_func)(void*, ParseData*, bool);
} ChildMonitor;
@ -178,8 +177,8 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
self->death_notify = death_notify; Py_INCREF(death_notify);
if (dump_callback != Py_None) {
self->dump_callback = dump_callback; Py_INCREF(dump_callback);
parse_func = parse_worker_dump;
} else parse_func = parse_worker;
self->parse_func = parse_worker_dump;
} else self->parse_func = parse_worker;
self->count = 0;
children_fds[0].fd = self->io_loop_data.wakeup_read_fd; children_fds[1].fd = self->io_loop_data.signal_read_fd;
children_fds[0].events = POLLIN; children_fds[1].events = POLLIN; children_fds[2].events = POLLIN;
@ -437,25 +436,16 @@ shutdown_monitor(ChildMonitor *self, PyObject *a UNUSED) {
static bool
do_parse(ChildMonitor *self, Screen *screen, monotonic_t now, bool flush) {
bool input_read = false;
screen_mutex(lock, read);
if (screen->read_buf_sz || vt_parser_has_pending_data(screen->vt_parser)) {
monotonic_t time_since_new_input = now - screen->new_input_at;
if (flush || time_since_new_input >= OPT(input_delay)) {
bool read_buf_full = screen->read_buf_sz >= READ_BUF_SZ;
input_read = true;
parse_func(screen, self->dump_callback, now);
if (read_buf_full) wakeup_io_loop(self, false); // Ensure the read fd has POLLIN set
screen->new_input_at = 0;
monotonic_t activated_at = vt_parser_pending_activated_at(screen->vt_parser);
if (activated_at) {
monotonic_t time_since_pending = MAX(0, now - activated_at);
set_maximum_wait(vt_parser_pending_wait_time(screen->vt_parser) - time_since_pending);
}
} else set_maximum_wait(OPT(input_delay) - time_since_new_input);
ParseData pd = {.dump_callback = self->dump_callback, .now = now};
self->parse_func(screen, &pd, flush);
if (pd.input_read) {
if (pd.write_space_created) wakeup_io_loop(self, false);
}
screen_mutex(unlock, read);
return input_read;
if (pd.input_read && pd.pending_activated_at) {
monotonic_t time_since_pending = MAX(0, now - pd.pending_activated_at);
set_maximum_wait(pd.pending_wait_time - time_since_pending);
} else set_maximum_wait(OPT(input_delay) - pd.time_since_new_input);
return pd.input_read;
}
static bool
@ -1343,16 +1333,13 @@ remove_children(ChildMonitor *self) {
static bool
read_bytes(int fd, Screen *screen) {
ssize_t len;
size_t available_buffer_space, orig_sz;
size_t available_buffer_space;
screen_mutex(lock, read);
orig_sz = screen->read_buf_sz;
if (orig_sz >= READ_BUF_SZ) { screen_mutex(unlock, read); return true; } // screen read buffer is full
available_buffer_space = READ_BUF_SZ - orig_sz;
screen_mutex(unlock, read);
uint8_t *buf = vt_parser_create_write_buffer(screen->vt_parser, &available_buffer_space);
if (!available_buffer_space) return true;
while(true) {
len = read(fd, screen->read_buf + orig_sz, available_buffer_space);
len = read(fd, buf, available_buffer_space);
if (len < 0) {
if (errno == EINTR || errno == EAGAIN) continue;
if (errno != EIO) perror("Call to read() from child fd failed");
@ -1360,16 +1347,8 @@ read_bytes(int fd, Screen *screen) {
}
break;
}
vt_parser_commit_write(screen->vt_parser, len);
if (UNLIKELY(len == 0)) return false;
screen_mutex(lock, read);
if (screen->new_input_at == 0) screen->new_input_at = monotonic();
if (orig_sz != screen->read_buf_sz) {
// The other thread consumed some of the screen read buffer
memmove(screen->read_buf + screen->read_buf_sz, screen->read_buf + orig_sz, len);
}
screen->read_buf_sz += len;
screen_mutex(unlock, read);
return true;
}
@ -1516,9 +1495,10 @@ io_loop(void *data) {
for (i = 0; i < self->count; i++) {
screen = children[i].screen;
/* printf("i:%lu id:%lu fd: %d read_buf_sz: %lu write_buf_used: %lu\n", i, children[i].id, children[i].fd, screen->read_buf_sz, screen->write_buf_used); */
screen_mutex(lock, read); screen_mutex(lock, write);
children_fds[EXTRA_FDS + i].events = (screen->read_buf_sz < READ_BUF_SZ ? POLLIN : 0) | (screen->write_buf_used ? POLLOUT : 0);
screen_mutex(unlock, read); screen_mutex(unlock, write);
children_fds[EXTRA_FDS + i].events = vt_parser_has_space_for_input(screen->vt_parser) ? POLLIN : 0;
screen_mutex(lock, write);
children_fds[EXTRA_FDS + i].events |= (screen->write_buf_used ? POLLOUT : 0);
screen_mutex(unlock, write);
}
if (has_pending_wakeups) {
now = monotonic();

View File

@ -78,6 +78,10 @@ def select_graphic_rendition(*a: int) -> None:
write(f'{CSI}{";".join(map(str, a))}m')
def deccara(*a: int) -> None:
write(f'{CSI}{";".join(map(str, a))}$r')
def screen_cursor_to_column(c: int) -> None:
write(f'{CSI}{c}G')

View File

@ -229,4 +229,5 @@
#define DECSCUSR 'q'
// File transfer OSC number
# define FILE_TRANSFER_CODE 5113
#define FILE_TRANSFER_CODE 5113
#define PENDING_MODE 2026

View File

@ -57,10 +57,10 @@ parse_color(int *params, unsigned int *i, unsigned int count, uint32_t *result)
if (*i < count) *result = (params[(*i)++] & 0xFF) << 8 | 1;
break;
case 2: \
if (*i < count - 2) {
if (*i + 2 < count) {
/* Ignore the first parameter in a four parameter RGB */
/* sequence (unused color space id), see https://github.com/kovidgoyal/kitty/issues/227 */
if (*i < count - 3) (*i)++;
if (*i +3 < count) (*i)++;
r = params[(*i)++] & 0xFF;
g = params[(*i)++] & 0xFF;
b = params[(*i)++] & 0xFF;
@ -73,7 +73,7 @@ parse_color(int *params, unsigned int *i, unsigned int count, uint32_t *result)
void
cursor_from_sgr(Cursor *self, int *params, unsigned int count) {
cursor_from_sgr(Cursor *self, int *params, unsigned int count, bool is_group) {
#define SET_COLOR(which) { parse_color(params, &i, count, &self->which); } break;
START_ALLOW_CASE_RANGE
unsigned int i = 0, attr;
@ -90,7 +90,7 @@ START_ALLOW_CASE_RANGE
case 3:
self->italic = true; break;
case 4:
if (i < count) { self->decoration = MIN(5, params[i]); i++; }
if (is_group && i < count) { self->decoration = MIN(5, params[i]); i++; }
else self->decoration = 1;
break;
case 7:
@ -134,13 +134,14 @@ START_ALLOW_CASE_RANGE
case DECORATION_FG_CODE + 1:
self->decoration_fg = 0; break;
}
if (is_group) break;
}
#undef SET_COLOR
END_ALLOW_CASE_RANGE
}
void
apply_sgr_to_cells(GPUCell *first_cell, unsigned int cell_count, int *params, unsigned int count) {
apply_sgr_to_cells(GPUCell *first_cell, unsigned int cell_count, int *params, unsigned int count, bool is_group) {
#define RANGE for(unsigned c = 0; c < cell_count; c++, cell++)
#define SET_COLOR(which) { color_type color = 0; parse_color(params, &i, count, &color); if (color) { RANGE { cell->which = color; }} } break;
#define SIMPLE(which, val) RANGE { cell->which = (val); } break;
@ -165,7 +166,7 @@ apply_sgr_to_cells(GPUCell *first_cell, unsigned int cell_count, int *params, un
S(italic, true);
case 4: {
uint8_t val = 1;
if (i < count) { val = MIN(5, params[i]); i++; }
if (is_group && i < count) { val = MIN(5, params[i]); i++; }
S(decoration, val);
}
case 7:
@ -211,6 +212,7 @@ END_ALLOW_CASE_RANGE
case DECORATION_FG_CODE + 1:
SIMPLE(decoration_fg, 0);
}
if (is_group) break;
}
#undef SET_COLOR
#undef RANGE

View File

@ -322,7 +322,6 @@ expand_ansi_c_escapes(PyObject *self UNUSED, PyObject *src) {
}
START_ALLOW_CASE_RANGE
#define C0_EXCEPT_NL_AND_SPACE 0x0 ... 0x9: case 0xb ... 0x1f: case 0x7f
static PyObject*
c0_replace_bytes(const char *input_data, Py_ssize_t input_sz) {
RAII_PyObject(ans, PyBytes_FromStringAndSize(NULL, input_sz * 3));
@ -333,7 +332,7 @@ c0_replace_bytes(const char *input_data, Py_ssize_t input_sz) {
for (Py_ssize_t i = 0; i < input_sz; i++) {
const char x = input_data[i];
switch (x) {
case C0_EXCEPT_NL_AND_SPACE: {
case C0_EXCEPT_NL_SPACE_TAB: {
const uint32_t ch = 0x2400 + x;
const unsigned sz = encode_utf8(ch, buf);
for (unsigned c = 0; c < sz; c++, j++) output[j] = buf[c];
@ -359,7 +358,7 @@ c0_replace_unicode(PyObject *input) {
bool changed = false;
for (Py_ssize_t i = 0; i < PyUnicode_GET_LENGTH(input); i++) {
Py_UCS4 ch = PyUnicode_READ(input_kind, input_data, i);
switch(ch) { case C0_EXCEPT_NL_AND_SPACE: ch += 0x2400; changed = true; }
switch(ch) { case C0_EXCEPT_NL_SPACE_TAB: ch += 0x2400; changed = true; }
if (ch > maxchar) maxchar = ch;
PyUnicode_WRITE(output_kind, output_data, i, ch);
}
@ -373,7 +372,7 @@ c0_replace_unicode(PyObject *input) {
END_ALLOW_CASE_RANGE
static PyObject*
replace_c0_codes_except_for_newline_and_space(PyObject *self UNUSED, PyObject *obj) {
replace_c0_codes_except_nl_space_tab(PyObject *self UNUSED, PyObject *obj) {
if (PyUnicode_Check(obj)) {
return c0_replace_unicode(obj);
} else if (PyBytes_Check(obj)) {
@ -402,7 +401,7 @@ find_in_memoryview(PyObject *self UNUSED, PyObject *args) {
}
static PyMethodDef module_methods[] = {
METHODB(replace_c0_codes_except_for_newline_and_space, METH_O),
METHODB(replace_c0_codes_except_nl_space_tab, METH_O),
{"wcwidth", (PyCFunction)wcwidth_wrap, METH_O, ""},
{"expand_ansi_c_escapes", (PyCFunction)expand_ansi_c_escapes, METH_O, ""},
{"get_docs_ref_map", (PyCFunction)get_docs_ref_map, METH_NOARGS, ""},

View File

@ -42,7 +42,9 @@
#define arraysz(x) (sizeof(x)/sizeof(x[0]))
#define zero_at_i(array, idx) memset((array) + (idx), 0, sizeof((array)[0]))
#define zero_at_ptr(p) memset((p), 0, sizeof((p)[0]))
#define literal_strlen(x) (sizeof(x)-1)
#define zero_at_ptr_count(p, count) memset((p), 0, (count) * sizeof((p)[0]))
#define C0_EXCEPT_NL_SPACE_TAB 0x0 ... 0x8: case 0xb ... 0x1f: case 0x7f
void log_error(const char *fmt, ...) __attribute__ ((format (printf, 1, 2)));
#define fatal(...) { log_error(__VA_ARGS__); exit(EXIT_FAILURE); }
static inline void cleanup_free(void *p) { free(*(void**)p); }
@ -344,8 +346,6 @@ typedef struct {int x;} *SPRITE_MAP_HANDLE;
#define FONTS_DATA_HEAD SPRITE_MAP_HANDLE sprite_map; double logical_dpi_x, logical_dpi_y, font_sz_in_pts; unsigned int cell_width, cell_height;
typedef struct {FONTS_DATA_HEAD} *FONTS_DATA_HANDLE;
#define READ_BUF_SZ (1024u*1024u)
#define clear_sprite_position(cell) (cell).sprite_x = 0; (cell).sprite_y = 0; (cell).sprite_z = 0;
#define ensure_space_for(base, array, type, num, capacity, initial_cap, zero_mem) \
@ -392,8 +392,8 @@ void cursor_reset(Cursor*);
Cursor* cursor_copy(Cursor*);
void cursor_copy_to(Cursor *src, Cursor *dest);
void cursor_reset_display_attrs(Cursor*);
void cursor_from_sgr(Cursor *self, int *params, unsigned int count);
void apply_sgr_to_cells(GPUCell *first_cell, unsigned int cell_count, int *params, unsigned int count);
void cursor_from_sgr(Cursor *self, int *params, unsigned int count, bool is_group);
void apply_sgr_to_cells(GPUCell *first_cell, unsigned int cell_count, int *params, unsigned int count, bool is_group);
const char* cell_as_sgr(const GPUCell *, const GPUCell *);
const char* cursor_as_sgr(const Cursor *);

View File

@ -1557,6 +1557,6 @@ def update_pointer_shape(os_window_id: int) -> None: ...
def os_window_focus_counters() -> Dict[int, int]: ...
def find_in_memoryview(buf: Union[bytes, memoryview, bytearray], chr: int) -> int: ...
@overload
def replace_c0_codes_except_for_newline_and_space(text: str) -> str:...
def replace_c0_codes_except_nl_space_tab(text: str) -> str:...
@overload
def replace_c0_codes_except_for_newline_and_space(text: Union[bytes, memoryview, bytearray]) -> bytes:...
def replace_c0_codes_except_nl_space_tab(text: Union[bytes, memoryview, bytearray]) -> bytes:...

View File

@ -718,6 +718,8 @@ decoration_as_sgr(uint8_t decoration) {
case 1: return "4;";
case 2: return "4:2;";
case 3: return "4:3;";
case 4: return "4:4";
case 5: return "4:5";
default: return "24;";
}
}

View File

@ -6,6 +6,7 @@
*/
#include "data-types.h"
#include "charsets.h"
#include <stdlib.h>
#include <stdarg.h>
#include <time.h>
@ -19,17 +20,37 @@ static bool use_os_log = false;
void
log_error(const char *fmt, ...) {
va_list ar;
struct timeval tv;
#ifdef __APPLE__
// Apple does not provide a varargs style os_logv
char logbuf[16 * 1024] = {0};
#else
char logbuf[4];
#endif
#define bufprint(fmt, ...) { \
if ((size_t)(p - logbuf) < sizeof(logbuf) - 2) { \
p += vsnprintf(p, sizeof(logbuf) - (p - logbuf), fmt, __VA_ARGS__); \
} }
char logbuf[16 * 1024];
char *p = logbuf;
#define bufprint(func, ...) { if ((size_t)(p - logbuf) < sizeof(logbuf) - 2) { p += func(p, sizeof(logbuf) - (p - logbuf), __VA_ARGS__); } }
va_list ar;
va_start(ar, fmt);
bufprint(fmt, ar);
va_end(ar);
RAII_ALLOC(char, sanbuf, calloc(1, 3*(p - logbuf) + 1));
char utf8buf[4];
START_ALLOW_CASE_RANGE
size_t j = 0;
for (char *x = logbuf; x < p; x++) {
switch(*x) {
case C0_EXCEPT_NL_SPACE_TAB: {
const uint32_t ch = 0x2400 + *x;
const unsigned sz = encode_utf8(ch, utf8buf);
for (unsigned c = 0; c < sz; c++, j++) sanbuf[j] = utf8buf[c];
} break;
default:
sanbuf[j++] = *x;
break;
}
}
sanbuf[j] = 0;
END_ALLOW_CASE_RANGE
if (!use_os_log) { // Apple's os_log already records timestamps
struct timeval tv;
gettimeofday(&tv, NULL);
struct tm stack_tmp;
struct tm *tmp = localtime_r(&tv.tv_sec, &stack_tmp);
@ -41,14 +62,11 @@ log_error(const char *fmt, ...) {
}
}
}
va_start(ar, fmt);
if (use_os_log) { bufprint(vsnprintf, fmt, ar); }
else vfprintf(stderr, fmt, ar);
va_end(ar);
#ifdef __APPLE__
if (use_os_log) os_log(OS_LOG_DEFAULT, "%{public}s", logbuf);
if (use_os_log) os_log(OS_LOG_DEFAULT, "%{public}s", sanbuf);
#endif
if (!use_os_log) fprintf(stderr, "\n");
if (!use_os_log) fprintf(stderr, "%s\n", sanbuf);
#undef bufprint
}
static PyObject*

View File

@ -99,10 +99,6 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
self = (Screen *)type->tp_alloc(type, 0);
if (self != NULL) {
if ((ret = pthread_mutex_init(&self->read_buf_lock, NULL)) != 0) {
Py_CLEAR(self); PyErr_Format(PyExc_RuntimeError, "Failed to create Screen read_buf_lock mutex: %s", strerror(ret));
return NULL;
}
if ((ret = pthread_mutex_init(&self->write_buf_lock, NULL)) != 0) {
Py_CLEAR(self); PyErr_Format(PyExc_RuntimeError, "Failed to create Screen write_buf_lock mutex: %s", strerror(ret));
return NULL;
@ -454,7 +450,6 @@ reset_callbacks(Screen *self, PyObject *a UNUSED) {
static void
dealloc(Screen* self) {
pthread_mutex_destroy(&self->read_buf_lock);
pthread_mutex_destroy(&self->write_buf_lock);
free_vt_parser(self->vt_parser); self->vt_parser = NULL;
Py_CLEAR(self->main_grman);
@ -724,7 +719,7 @@ screen_alignment_display(Screen *self) {
}
void
select_graphic_rendition(Screen *self, int *params, unsigned int count, Region *region_) {
select_graphic_rendition(Screen *self, int *params, unsigned int count, bool is_group, Region *region_) {
if (region_) {
Region region = *region_;
if (!region.top) region.top = 1;
@ -741,7 +736,7 @@ select_graphic_rendition(Screen *self, int *params, unsigned int count, Region *
num = MIN(num, self->columns - x);
for (index_type y = region.top; y < MIN(region.bottom + 1, self->lines); y++) {
linebuf_init_line(self->linebuf, y);
apply_sgr_to_cells(self->linebuf->line->gpu_cells + x, num, params, count);
apply_sgr_to_cells(self->linebuf->line->gpu_cells + x, num, params, count, is_group);
}
} else {
index_type x, num;
@ -749,18 +744,18 @@ select_graphic_rendition(Screen *self, int *params, unsigned int count, Region *
linebuf_init_line(self->linebuf, region.top);
x = MIN(region.left, self->columns-1);
num = MIN(self->columns - x, region.right - x + 1);
apply_sgr_to_cells(self->linebuf->line->gpu_cells + x, num, params, count);
apply_sgr_to_cells(self->linebuf->line->gpu_cells + x, num, params, count, is_group);
} else {
for (index_type y = region.top; y < MIN(region.bottom + 1, self->lines); y++) {
if (y == region.top) { x = MIN(region.left, self->columns - 1); num = self->columns - x; }
else if (y == region.bottom) { x = 0; num = MIN(region.right + 1, self->columns); }
else { x = 0; num = self->columns; }
linebuf_init_line(self->linebuf, y);
apply_sgr_to_cells(self->linebuf->line->gpu_cells + x, num, params, count);
apply_sgr_to_cells(self->linebuf->line->gpu_cells + x, num, params, count, is_group);
}
}
}
} else cursor_from_sgr(self->cursor, params, count);
} else cursor_from_sgr(self->cursor, params, count, is_group);
}
static void
@ -993,17 +988,6 @@ set_mode_from_const(Screen *self, unsigned int mode, bool val) {
if (val && self->linebuf == self->main_linebuf) screen_toggle_screen_buffer(self, mode == ALTERNATE_SCREEN, mode == ALTERNATE_SCREEN);
else if (!val && self->linebuf != self->main_linebuf) screen_toggle_screen_buffer(self, mode == ALTERNATE_SCREEN, mode == ALTERNATE_SCREEN);
break;
case PENDING_UPDATE:
if (val) {
vt_parser_set_pending_activated_at(self->vt_parser, monotonic());
} else {
if (!vt_parser_pending_activated_at(self->vt_parser)) log_error(
"Pending mode stop command issued while not in pending mode, this can"
" be either a bug in the terminal application or caused by a timeout with no data"
" received for too long or by too much data in pending mode");
else vt_parser_set_pending_activated_at(self->vt_parser, 0);
}
break;
case 7727 << 5:
log_error("Application escape mode is not supported, the extended keyboard protocol should be used instead");
break;
@ -3512,8 +3496,10 @@ static PyObject*
apply_sgr(Screen *self, PyObject *src) {
if (!PyUnicode_Check(src)) { PyErr_SetString(PyExc_TypeError, "A unicode string is required"); return NULL; }
if (PyUnicode_READY(src) != 0) { return PyErr_NoMemory(); }
int params[MAX_PARAMS] = {0};
parse_sgr(self, (const uint8_t*)PyUnicode_AsUTF8(src), PyUnicode_GET_LENGTH(src), params, "parse_sgr", NULL);
if (!parse_sgr(self, (const uint8_t*)PyUnicode_AsUTF8(src), PyUnicode_GET_LENGTH(src), "parse_sgr", false)) {
PyErr_Format(PyExc_ValueError, "Invalid SGR: %s", PyUnicode_AsUTF8(src));
return NULL;
}
Py_RETURN_NONE;
}
@ -3531,7 +3517,7 @@ static PyObject*
_select_graphic_rendition(Screen *self, PyObject *args) {
int params[256] = {0};
for (int i = 0; i < PyTuple_GET_SIZE(args); i++) { params[i] = PyLong_AsLong(PyTuple_GET_ITEM(args, i)); }
select_graphic_rendition(self, params, PyTuple_GET_SIZE(args), NULL);
select_graphic_rendition(self, params, PyTuple_GET_SIZE(args), false, NULL);
Py_RETURN_NONE;
}
@ -4462,12 +4448,15 @@ test_write_data(Screen *screen, PyObject *args) {
monotonic_t now = monotonic();
while (sz) {
size_t s = MIN(sz, READ_BUF_SZ);
memcpy(screen->read_buf, data, s);
screen->read_buf_sz = s;
size_t s;
uint8_t *buf = vt_parser_create_write_buffer(screen->vt_parser, &s);
s = MIN(s, (size_t)sz);
memcpy(buf, data, s);
vt_parser_commit_write(screen->vt_parser, s);
data += s; sz -= s;
if (dump_callback) parse_worker_dump(screen, dump_callback, now);
else parse_worker(screen, dump_callback, now);
ParseData pd = {.dump_callback=dump_callback,.now=now};
if (dump_callback) parse_worker_dump(screen, &pd, true);
else parse_worker(screen, &pd, true);
}
Py_RETURN_NONE;
}

View File

@ -9,7 +9,6 @@
#include "vt-parser.h"
#include "graphics.h"
#include "monotonic.h"
#define MAX_PARAMS 256
typedef enum ScrollTypes { SCROLL_LINE = -999999, SCROLL_PAGE, SCROLL_FULL } ScrollType;
@ -107,10 +106,9 @@ typedef struct {
ColorProfile *color_profile;
monotonic_t start_visual_bell_at;
uint8_t read_buf[READ_BUF_SZ], *write_buf;
monotonic_t new_input_at;
size_t read_buf_sz, write_buf_sz, write_buf_used;
pthread_mutex_t read_buf_lock, write_buf_lock;
uint8_t *write_buf;
size_t write_buf_sz, write_buf_used;
pthread_mutex_t write_buf_lock;
CursorRenderInfo cursor_render_info;
unsigned int render_unfocused_cursor;
@ -156,8 +154,6 @@ typedef struct {
} Screen;
void parse_worker(Screen *screen, PyObject *dump_callback, monotonic_t now);
void parse_worker_dump(Screen *screen, PyObject *dump_callback, monotonic_t now);
void screen_align(Screen*);
void screen_restore_cursor(Screen *);
void screen_save_cursor(Screen *);
@ -223,7 +219,7 @@ void set_color_table_color(Screen *self, unsigned int code, PyObject*);
void process_cwd_notification(Screen *self, unsigned int code, const char*, size_t);
void screen_request_capabilities(Screen *, char, const char *);
void report_device_attributes(Screen *self, unsigned int UNUSED mode, char start_modifier);
void select_graphic_rendition(Screen *self, int *params, unsigned int count, Region*);
void select_graphic_rendition(Screen *self, int *params, unsigned int count, bool is_group, Region *r);
void report_device_status(Screen *self, unsigned int which, bool UNUSED);
void report_mode_status(Screen *self, unsigned int which, bool);
void screen_apply_selection(Screen *self, void *address, size_t size);
@ -266,7 +262,7 @@ int screen_cursor_at_a_shell_prompt(const Screen *);
bool screen_fake_move_cursor_to_position(Screen *, index_type x, index_type y);
bool screen_send_signal_for_key(Screen *, char key);
bool get_line_edge_colors(Screen *self, color_type *left, color_type *right);
void parse_sgr(Screen *screen, const uint8_t *buf, unsigned int num, int *params, const char *report_name, Region *region);
bool parse_sgr(Screen *screen, const uint8_t *buf, unsigned int num, const char *report_name, bool is_deccara);
#define DECLARE_CH_SCREEN_HANDLER(name) void screen_##name(Screen *screen);
DECLARE_CH_SCREEN_HANDLER(bell)
DECLARE_CH_SCREEN_HANDLER(backspace)

View File

@ -131,7 +131,7 @@ def log_error(*a: Any, **k: str) -> None:
from .fast_data_types import log_error_string
output = getattr(log_error, 'redirect', log_error_string)
with suppress(Exception):
msg = k.get('sep', ' ').join(map(str, a)) + k.get('end', '').replace('\0', '')
msg = k.get('sep', ' ').join(map(str, a)) + k.get('end', '')
output(msg)

File diff suppressed because it is too large Load Diff

View File

@ -16,12 +16,27 @@ typedef struct Parser {
PARSER_STATE_HANDLE *state;
} Parser;
typedef struct ParseData {
PyObject *dump_callback;
monotonic_t now;
bool input_read, write_space_created;
monotonic_t pending_activated_at, pending_wait_time, time_since_new_input;
} ParseData;
// The must only be called on the main thread
Parser* alloc_vt_parser(id_type window_id);
void free_vt_parser(Parser*);
void reset_vt_parser(Parser*);
bool vt_parser_has_pending_data(Parser*);
monotonic_t vt_parser_pending_activated_at(Parser*);
monotonic_t vt_parser_pending_wait_time(Parser*);
void vt_parser_set_pending_activated_at(Parser*, monotonic_t);
void vt_parser_set_pending_wait_time(Parser*, monotonic_t);
// The following are thread safe, using an internal lock
uint8_t* vt_parser_create_write_buffer(Parser*, size_t*);
void vt_parser_commit_write(Parser*, size_t);
bool vt_parser_has_space_for_input(const Parser*);
void parse_worker(void *p, ParseData *data, bool flush);
void parse_worker_dump(void *p, ParseData *data, bool flush);

View File

@ -75,6 +75,7 @@
move_cursor_to_mouse_if_in_prompt,
pointer_name_to_css_name,
pt_to_px,
replace_c0_codes_except_nl_space_tab,
set_window_logo,
set_window_padding,
set_window_render_data,
@ -504,18 +505,6 @@ def set_extra(self, extra: str) -> None:
global_watchers = GlobalWatchers()
def replace_control_codes(text: str) -> str:
# Replace all control codes other than tab, newline and space with their graphical counterparts
def sub(m: 're.Match[str]') -> str:
c = ord(m.group())
if c < 0x20:
return chr(0x2400 + c)
if c == 0x7f:
return '\u2421'
return '\u2426'
return re.sub(r'[\0-\x08\x0b-\x19\x7f-\x9f]', sub, text)
class Window:
window_custom_type: str = ''
@ -1613,20 +1602,20 @@ def paste_with_actions(self, text: str) -> None:
import shlex
text = shlex.quote(text)
if 'replace-dangerous-control-codes' in opts.paste_actions:
text = replace_control_codes(text)
text = replace_c0_codes_except_nl_space_tab(text)
if 'replace-newline' in opts.paste_actions:
text = text.replace('\n', '\x1bE')
btext = text.encode('utf-8')
if 'confirm' in opts.paste_actions:
sanitized = replace_control_codes(text)
sanitized = replace_c0_codes_except_nl_space_tab(btext)
if not self.screen.in_bracketed_paste_mode:
# \n is converted to \r and \r is interpreted as the enter key
# by legacy programs that dont support the full kitty keyboard protocol,
# which in the case of shells can lead to command execution.
# \eE has the same visual effect as \r\n but without the
# command execution risk.
sanitized = sanitized.replace('\n', '\x1bE')
if sanitized != text:
# which in the case of shells can lead to command execution, so
# replace with <ESC>E (NEL) which has the newline visual effect \r\n but
# isnt interpreted as Enter.
sanitized = sanitized.replace(b'\n', b'\x1bE')
if sanitized != btext:
msg = _('The text to be pasted contains terminal control codes.\n\nIf the terminal program you are pasting into does not properly'
' sanitize pasted text, this can lead to \x1b[31mcode execution vulnerabilities\x1b[39m.\n\nHow would you like to proceed?')
get_boss().choose(
@ -1644,7 +1633,7 @@ def paste_with_actions(self, text: str) -> None:
return
self.paste_text(btext)
def handle_dangerous_paste_confirmation(self, unsanitized: bytes, sanitized: str, choice: str) -> None:
def handle_dangerous_paste_confirmation(self, unsanitized: bytes, sanitized: bytes, choice: str) -> None:
if choice == 's':
self.paste_text(sanitized)
elif choice == 'p':

View File

@ -13,7 +13,7 @@
LineBuf,
expand_ansi_c_escapes,
parse_input_from_terminal,
replace_c0_codes_except_for_newline_and_space,
replace_c0_codes_except_nl_space_tab,
strip_csi,
truncate_point_for_length,
wcswidth,
@ -41,13 +41,14 @@ class TestDataTypes(BaseTest):
def test_replace_c0_codes(self):
def t(x: str, expected: str):
q = replace_c0_codes_except_for_newline_and_space(x)
q = replace_c0_codes_except_nl_space_tab(x)
self.ae(expected, q)
q = replace_c0_codes_except_for_newline_and_space(x.encode('utf-8'))
q = replace_c0_codes_except_nl_space_tab(x.encode('utf-8'))
self.ae(expected.encode('utf-8'), q)
t('abc', 'abc')
t('a\0\x01b\x03\x04\t\rc', 'a\u2400\u2401b\u2403\u2404\u2409\u240dc')
t('a\0\x01😸\x03\x04\t\rc', 'a\u2400\u2401😸\u2403\u2404\u2409\u240dc')
t('a\0\x01b\x03\x04\t\rc', 'a\u2400\u2401b\u2403\u2404\t\u240dc')
t('a\0\x01😸\x03\x04\t\rc', 'a\u2400\u2401😸\u2403\u2404\t\u240dc')
t('a\nb\tc d', 'a\nb\tc d')
def test_to_color(self):
for x in 'xxx #12 #1234 rgb:a/b'.split():

View File

@ -25,6 +25,8 @@ def __call__(self, *a):
a = a[1:]
if a and a[0] == 'bytes':
return
if a and a[0] == 'error':
a = a[1:]
self.append(tuple(map(cnv, a)))
@ -75,7 +77,7 @@ def test_simple_parsing(self):
self.ae(str(s.line(1)), '6')
self.ae(str(s.line(2)), ' 123')
self.ae(str(s.line(3)), '45')
s.test_write_data(b'\rabcde')
pb(b'\rabcde', ('screen_carriage_return',), 'abcde')
self.ae(str(s.line(3)), 'abcde')
pb('\rßxyz1', ('screen_carriage_return',), 'ßxyz1')
self.ae(str(s.line(3)), 'ßxyz1')
@ -126,7 +128,7 @@ def test_csi_codes(self):
pb('x\033[2;7@y', 'x', ('CSI code @ has 2 > 1 parameters',), 'y')
pb('x\033[2;-7@y', 'x', ('CSI code @ has 2 > 1 parameters',), 'y')
pb('x\033[-2@y', 'x', ('CSI code @ is not allowed to have negative parameter (-2)',), 'y')
pb('x\033[2-3@y', 'x', ('CSI code can contain hyphens only at the start of numbers',), 'y')
pb('x\033[2-3@y', 'x', ('Invalid character in CSI: 3 (0x33), ignoring the sequence',), '@y')
pb('x\033[@y', 'x', ('screen_insert_characters', 1), 'y')
pb('x\033[345@y', 'x', ('screen_insert_characters', 345), 'y')
pb('x\033[345;@y', 'x', ('screen_insert_characters', 345), 'y')
@ -148,31 +150,36 @@ def test_csi_codes(self):
pb('\033[=c', ('report_device_attributes', 0, 61))
s.reset()
def sgr(params):
return (('select_graphic_rendition', f'{x} ') for x in params.split())
def sgr(*params):
return (('select_graphic_rendition', f'{x}') for x in params)
pb('\033[1;2;3;4;7;9;34;44m', *sgr('1 2 3 4 7 9 34 44'))
for attr in 'bold italic reverse strikethrough dim'.split():
self.assertTrue(getattr(s.cursor, attr))
self.assertTrue(getattr(s.cursor, attr), attr)
self.ae(s.cursor.decoration, 1)
self.ae(s.cursor.fg, 4 << 8 | 1)
self.ae(s.cursor.bg, 4 << 8 | 1)
pb('\033[38;5;1;48;5;7m', ('select_graphic_rendition', '38 5 1 '), ('select_graphic_rendition', '48 5 7 '))
pb('\033[38;5;1;48;5;7m', ('select_graphic_rendition', '38:5:1'), ('select_graphic_rendition', '48:5:7'))
self.ae(s.cursor.fg, 1 << 8 | 1)
self.ae(s.cursor.bg, 7 << 8 | 1)
pb('\033[38;2;1;2;3;48;2;7;8;9m', ('select_graphic_rendition', '38 2 1 2 3 '), ('select_graphic_rendition', '48 2 7 8 9 '))
pb('\033[38;2;1;2;3;48;2;7;8;9m', ('select_graphic_rendition', '38:2:1:2:3'), ('select_graphic_rendition', '48:2:7:8:9'))
self.ae(s.cursor.fg, 1 << 24 | 2 << 16 | 3 << 8 | 2)
self.ae(s.cursor.bg, 7 << 24 | 8 << 16 | 9 << 8 | 2)
pb('\033[0;2m', *sgr('0 2'))
pb('\033[;2m', *sgr('0 2'))
pb('\033[m', *sgr('0 '))
pb('\033[m', *sgr('0'))
pb('\033[1;;2m', *sgr('1 0 2'))
pb('\033[38;5;1m', ('select_graphic_rendition', '38 5 1 '))
pb('\033[58;2;1;2;3m', ('select_graphic_rendition', '58 2 1 2 3 '))
pb('\033[38;2;1;2;3m', ('select_graphic_rendition', '38 2 1 2 3 '))
pb('\033[1001:2:1:2:3m', ('select_graphic_rendition', '1001 2 1 2 3 '))
pb('\033[38;5;1m', ('select_graphic_rendition', '38:5:1'))
pb('\033[58;2;1;2;3m', ('select_graphic_rendition', '58:2:1:2:3'))
pb('\033[38;2;1;2;3m', ('select_graphic_rendition', '38:2:1:2:3'))
pb('\033[1001:2:1:2:3m', ('select_graphic_rendition', '1001:2:1:2:3'))
pb('\033[38:2:1:2:3;48:5:9;58;5;7m', (
'select_graphic_rendition', '38 2 1 2 3 '), ('select_graphic_rendition', '48 5 9 '), ('select_graphic_rendition', '58 5 7 '))
'select_graphic_rendition', '38:2:1:2:3'), ('select_graphic_rendition', '48:5:9'), ('select_graphic_rendition', '58:5:7'))
s.reset()
pb('\033[1;2;3;4:5;7;9;34;44m', *sgr('1 2 3', '4:5', '7 9 34 44'))
for attr in 'bold italic reverse strikethrough dim'.split():
self.assertTrue(getattr(s.cursor, attr), attr)
self.ae(s.cursor.decoration, 5)
c = s.callbacks
pb('\033[5n', ('report_device_status', 5, 0))
self.ae(c.wtcbuf, b'\033[0n')
@ -241,7 +248,7 @@ def test_osc_codes(self):
c.clear()
pb('\033]\x07', ('set_title', ''), ('set_icon', ''))
self.ae(c.titlebuf, ['']), self.ae(c.iconbuf, '')
pb('\033]ab\x07', ('set_title', 'ab'), ('set_icon', 'ab'))
pb('1\033]ab\x072', '1', ('set_title', 'ab'), ('set_icon', 'ab'), '2')
self.ae(c.titlebuf, ['', 'ab']), self.ae(c.iconbuf, 'ab')
c.clear()
pb('\033]2;;;;\x07', ('set_title', ';;;'))
@ -378,18 +385,19 @@ def test_pending(self):
pb('\033P'), pb('='), pb('2s')
pb('\033\\', ('draw', 'e'), ('screen_stop_pending_mode',))
pb('\033P=1sxyz;.;\033\\''\033P=2skjf".,><?_+)98\033\\', ('screen_start_pending_mode',), ('screen_stop_pending_mode',))
pb('\033P=1s\033\\f\033P=1s\033\\', ('screen_start_pending_mode',), ('screen_start_pending_mode',))
pb('\033P=1s\033\\f\033P=1s\033\\', ('screen_start_pending_mode',),)
pb('\033P=2s\033\\', ('draw', 'f'), ('screen_stop_pending_mode',))
pb('\033P=1s\033\\XXX\033P=2s\033\\', ('screen_start_pending_mode',), ('draw', 'XXX'), ('screen_stop_pending_mode',))
pb('\033[?2026hXXX\033[?2026l', ('screen_set_mode', 2026, 1), ('draw', 'XXX'), ('screen_reset_mode', 2026, 1))
pb('\033[?2026h\033[32ma\033[?2026l', ('screen_set_mode', 2026, 1), ('select_graphic_rendition', '32 '), ('draw', 'a'), ('screen_reset_mode', 2026, 1))
pb('\033[?2026hXXX\033[?2026l', ('screen_start_pending_mode',), ('draw', 'XXX'), ('screen_stop_pending_mode',))
pb('\033[?2026h\033[32ma\033[?2026l', ('screen_start_pending_mode',), ('select_graphic_rendition', '32'),
('draw', 'a'), ('screen_stop_pending_mode',))
pb('\033[?2026h\033P+q544e\033\\ama\033P=2s\033\\',
('screen_set_mode', 2026, 1), ('screen_request_capabilities', 43, '544e'), ('draw', 'ama'), ('screen_stop_pending_mode',))
('screen_start_pending_mode',), ('screen_request_capabilities', 43, '544e'), ('draw', 'ama'), ('screen_stop_pending_mode',))
s.reset()
s.set_pending_timeout(timeout)
pb('\033[?2026h', ('screen_set_mode', 2026, 1),)
pb('\033[?2026h', ('screen_start_pending_mode',),)
pb('\033P+q')
time.sleep(1.2 * timeout)
pb('544e\033\\', ('screen_request_capabilities', 43, '544e'))
@ -398,28 +406,27 @@ def test_pending(self):
('Pending mode stop command issued while not in pending mode, this can be '
'either a bug in the terminal application or caused by a timeout with no '
'data received for too long or by too much data in pending mode',),
('screen_stop_pending_mode',)
)
self.assertEqual(str(s.line(0)), '')
pb('\033[?2026h', ('screen_set_mode', 2026, 1),)
pb('\033[?2026h', ('screen_start_pending_mode',),)
pb('ab')
s.set_pending_activated_at(0.00001)
pb('cd', ('draw', 'abcd'))
pb('\033[?2026h', ('screen_set_mode', 2026, 1),)
pb('\033[?2026h', ('screen_start_pending_mode',),)
pb('\033')
s.set_pending_activated_at(0.00001)
pb('7', ('screen_save_cursor',))
pb('\033[?2026h\033]', ('screen_set_mode', 2026, 1),)
pb('\033[?2026h\033]', ('screen_start_pending_mode',),)
s.set_pending_activated_at(0.00001)
pb('8;;\x07', ('set_active_hyperlink', None, None))
pb('\033[?2026h\033', ('screen_set_mode', 2026, 1),)
pb('\033[?2026h\033', ('screen_start_pending_mode',),)
s.set_pending_activated_at(0.00001)
pb(']8;;\x07', ('set_active_hyperlink', None, None))
pb('😀'.encode()[:-1])
pb('\033[?2026h', ('screen_set_mode', 2026, 1),)
pb('\033[?2026h', ('screen_start_pending_mode',),)
pb('😀'.encode()[-1:])
pb('\033[?2026l', '\ufffd', ('screen_reset_mode', 2026, 1),)
pb('\033[?2026l', '\ufffd', ('screen_stop_pending_mode',),)
pb('a', ('draw', 'a'))
def test_oth_codes(self):
@ -477,9 +484,9 @@ def e(cmd, err):
def test_deccara(self):
s = self.create_screen()
pb = partial(self.parse_bytes_dump, s)
pb('\033[$r', ('deccara', '0 0 0 0 0 '))
pb('\033[$r', ('deccara', '0 0 0 0 0'))
pb('\033[;;;;4:3;38:5:10;48:2:1:2:3;1$r',
('deccara', '0 0 0 0 4 3 '), ('deccara', '0 0 0 0 38 5 10 '), ('deccara', '0 0 0 0 48 2 1 2 3 '), ('deccara', '0 0 0 0 1 '))
('deccara', '0 0 0 0 4:3'), ('deccara', '0 0 0 0 38:5:10'), ('deccara', '0 0 0 0 48:2:1:2:3'), ('deccara', '0 0 0 0 1'))
for y in range(s.lines):
line = s.line(y)
for x in range(s.columns):
@ -490,7 +497,7 @@ def test_deccara(self):
self.ae(c.fg, (10 << 8) | 1)
self.ae(c.bg, (1 << 24 | 2 << 16 | 3 << 8 | 2))
self.ae(s.line(0).cursor_from(0).bold, True)
pb('\033[1;2;2;3;22;39$r', ('deccara', '1 2 2 3 22 '), ('deccara', '1 2 2 3 39 '))
pb('\033[1;2;2;3;22;39$r', ('deccara', '1 2 2 3 22 39'))
self.ae(s.line(0).cursor_from(0).bold, True)
line = s.line(0)
for x in range(1, s.columns):
@ -502,7 +509,7 @@ def test_deccara(self):
c = line.cursor_from(x)
self.ae(c.bold, False)
self.ae(line.cursor_from(3).bold, True)
pb('\033[2*x\033[3;2;4;3;34$r\033[*x', ('screen_decsace', 2), ('deccara', '3 2 4 3 34 '), ('screen_decsace', 0))
pb('\033[2*x\033[3;2;4;3;34$r\033[*x', ('screen_decsace', 2), ('deccara', '3 2 4 3 34'), ('screen_decsace', 0))
for y in range(2, 4):
line = s.line(y)
for x in range(s.columns):

View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
from kitty.fast_data_types import DECAWM, DECCOLM, DECOM, IRM, Cursor
from kitty.fast_data_types import DECAWM, DECCOLM, DECOM, IRM, VT_PARSER_BUFFER_SIZE, Cursor
from kitty.marks import marker_from_function, marker_from_regex
from kitty.window import pagerhist
@ -912,23 +912,27 @@ def test_osc_52(self):
def send(what: str):
return parse_bytes(s, f'\033]52;p;{what}\a'.encode('ascii'))
def t(q, use_pending_mode, *expected):
def t(q, use_pending_mode, *expected, expect_pending=True):
c.clear()
if use_pending_mode:
parse_bytes(s, b'\033[?2026h')
send(q)
del q
if use_pending_mode:
t.ex = list(expected)
del expected
if use_pending_mode and expect_pending:
self.ae(c.cc_buf, [])
parse_bytes(s, b'\033[?2026l')
try:
self.ae(c.cc_buf, list(expected))
self.ae(tuple(map(len, c.cc_buf)), tuple(map(len, t.ex)))
self.ae(c.cc_buf, t.ex)
finally:
del expected
del t.ex
for use_pending_mode in (False, True):
t('XYZ', use_pending_mode, ('p;XYZ', False))
t('a' * 8192, use_pending_mode, ('p;' + 'a' * (8192 - 6), True), (';' + 'a' * 6, False))
t('a' * VT_PARSER_BUFFER_SIZE, use_pending_mode, ('p;' + 'a' * (VT_PARSER_BUFFER_SIZE - 8), True),
(';' + 'a' * 8, False), expect_pending=False)
t('', use_pending_mode, ('p;', False))
t('!', use_pending_mode, ('p;!', False))