From c869774c58f160927cfe8ceab7aa2a52ebfc9017 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 3 Jan 2021 05:07:01 +0530 Subject: [PATCH] More tests for the disk cache --- kitty/disk-cache.c | 62 +++++++++++++++++++++++++++++++++-------- kitty/disk-cache.h | 1 + kitty_tests/graphics.py | 14 +++++++++- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/kitty/disk-cache.c b/kitty/disk-cache.c index c95a355db..9e28a94a1 100644 --- a/kitty/disk-cache.c +++ b/kitty/disk-cache.c @@ -241,12 +241,12 @@ cmp_pos_in_cache_file(void *a_, void *b_) { static inline void find_hole(DiskCache *self) { - off_t required_size = self->currently_writing.data_sz, prev = -1; + off_t required_size = self->currently_writing.data_sz, prev = -100; HASH_SORT(self->entries, cmp_pos_in_cache_file); CacheEntry *s, *tmp; HASH_ITER(hh, self->entries, s, tmp) { - if (s->pos_in_cache_file >= 0) { - if (prev >= 0 && s->pos_in_cache_file - prev > required_size) { + if (s->pos_in_cache_file >= 0 && s->data_sz > 0) { + if (prev >= 0 && s->pos_in_cache_file - prev >= required_size) { self->currently_writing.pos_in_cache_file = prev; return; } @@ -522,15 +522,22 @@ remove_from_disk_cache(PyObject *self_, const void *key, size_t key_sz) { return removed; } -static void -read_from_cache_entry(const DiskCache *self, const CacheEntry *s, uint8_t *dest) { - uint8_t *p = dest; - size_t sz = s->data_sz; - off_t pos = s->pos_in_cache_file; - if (pos < 0) { - PyErr_SetString(PyExc_OSError, "Cache entry was not written, could not read from it"); - return; +void +clear_disk_cache(PyObject *self_) { + DiskCache *self = (DiskCache*)self_; + CacheEntry *s, *tmp; + mutex(lock); + HASH_ITER(hh, self->entries, s, tmp) { + HASH_DEL(self->entries, s); + free_cache_entry(s); } + mutex(unlock); + wakeup_write_loop(self); +} + +static void +read_from_cache_file(const DiskCache *self, off_t pos, size_t sz, void *dest) { + uint8_t *p = dest; while (sz) { ssize_t n = pread(self->cache_file_fd, p, sz, pos); if (n > 0) { @@ -551,6 +558,17 @@ read_from_cache_entry(const DiskCache *self, const CacheEntry *s, uint8_t *dest) } } +static void +read_from_cache_entry(const DiskCache *self, const CacheEntry *s, void *dest) { + size_t sz = s->data_sz; + off_t pos = s->pos_in_cache_file; + if (pos < 0) { + PyErr_SetString(PyExc_OSError, "Cache entry was not written, could not read from it"); + return; + } + read_from_cache_file(self, pos, sz, dest); +} + void* read_from_disk_cache(PyObject *self_, const void *key, size_t key_sz, void*(allocator)(void*, size_t), void* allocator_data) { DiskCache *self = (DiskCache*)self_; @@ -596,7 +614,7 @@ disk_cache_wait_for_write(PyObject *self_, monotonic_t timeout) { mutex(unlock); if (!pending) return true; wakeup_write_loop(self); - usleep(100 * 1000); + usleep(10 * 1000); } return false; } @@ -622,6 +640,17 @@ PYWRAP(xor_data) { return ans; } +PYWRAP(read_from_cache_file) { + Py_ssize_t pos = 0, sz = -1; + PA("|nn", &pos, &sz); + if (sz < 0) sz = size_of_cache_file(self); + PyObject *ans = PyBytes_FromStringAndSize(NULL, sz); + if (ans) { + read_from_cache_file(self, pos, sz, PyBytes_AS_STRING(ans)); + } + return ans; +} + static PyObject* wait_for_write(PyObject *self, PyObject *args) { double timeout = 0; @@ -636,6 +665,13 @@ size_on_disk(PyObject *self, PyObject *args UNUSED) { return PyLong_FromUnsignedLongLong(ans); } +static PyObject* +clear(PyObject *self, PyObject *args UNUSED) { + clear_disk_cache(self); + Py_RETURN_NONE; +} + + static PyObject* add(PyObject *self, PyObject *args) { const char *key, *data; @@ -683,11 +719,13 @@ get(PyObject *self, PyObject *args) { #define MW(name, arg_type) {#name, (PyCFunction)py##name, arg_type, NULL} static PyMethodDef methods[] = { MW(ensure_state, METH_NOARGS), + MW(read_from_cache_file, METH_VARARGS), {"add", add, METH_VARARGS, NULL}, {"remove", pyremove, METH_VARARGS, NULL}, {"get", get, METH_VARARGS, NULL}, {"wait_for_write", wait_for_write, METH_VARARGS, NULL}, {"size_on_disk", size_on_disk, METH_NOARGS, NULL}, + {"clear", clear, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; diff --git a/kitty/disk-cache.h b/kitty/disk-cache.h index a5e944a3d..1ad8003b8 100644 --- a/kitty/disk-cache.h +++ b/kitty/disk-cache.h @@ -14,6 +14,7 @@ bool remove_from_disk_cache(PyObject *self_, const void *key, size_t key_sz); void* read_from_disk_cache(PyObject *self_, const void *key, size_t key_sz, void*(allocator)(void*, size_t), void*); bool disk_cache_wait_for_write(PyObject *self, monotonic_t timeout); size_t disk_cache_size_on_disk(PyObject *self); +void clear_disk_cache(PyObject *self); static inline void* disk_cache_malloc_allocator(void *x, size_t sz) { *((size_t*)x) = sz; diff --git a/kitty_tests/graphics.py b/kitty_tests/graphics.py index f78e8fa11..6618f7199 100644 --- a/kitty_tests/graphics.py +++ b/kitty_tests/graphics.py @@ -4,11 +4,12 @@ import os import tempfile +import time import unittest import zlib -from itertools import cycle from base64 import standard_b64decode, standard_b64encode from io import BytesIO +from itertools import cycle from kitty.constants import cache_dir from kitty.fast_data_types import ( @@ -194,6 +195,17 @@ def check_data(): check_data() self.assertRaises(KeyError, dc.get, key_as_bytes(x)) self.assertEqual(sz, dc.size_on_disk()) + for x in ('xy', 'C'*4, 'B'*6, 'A'*8): + add(x, x) + self.assertTrue(dc.wait_for_write()) + self.assertEqual(sz, dc.size_on_disk()) + check_data() + check_data() + dc.clear() + st = time.monotonic() + while dc.size_on_disk() and time.monotonic() - st < 2: + time.sleep(0.001) + self.assertEqual(dc.size_on_disk(), 0) def test_load_images(self): s, g, l, sl = load_helpers(self)