on master help sign in
[enh] add searx.shared

shared dictionary between the workers (UWSGI or werkzeug) scheduler: run a task once every x seconds (UWSGI or werkzeug)

commit6e2872f43625aba71eba019e16f7fbd74743f590 authorAlexandre Flament <alex@al-f.net> dateTue 5 Jan 2021, 11:22 (6 years ago) parent9c58146
7 files changed+156−1
dockerfiles/uwsgi.ini+3 searx/shared/__init__.py+31 searx/shared/shared_abstract.py+15 searx/shared/shared_simple.py+38 searx/shared/shared_uwsgi.py+62 utils/templates/etc/uwsgi/apps-archlinux/searx.ini+4−1 utils/templates/etc/uwsgi/apps-available/searx.ini+3
dockerfiles/uwsgi.ini+3
@@ -42,3 +42,6 @@ static-map = /static=/usr/local/searx/searx/static
4242static-expires = /* 864000
4343static-gzip-all = True
4444offload-threads = %k
45+
46+# Cache
47+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1
searx/shared/__init__.pynew file+31
@@ -0,0 +1,31 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import logging
4+
5+logger = logging.getLogger('searx.shared')
6+
7+try:
8+ import uwsgi
9+except:
10+ # no uwsgi
11+ from .shared_simple import SimpleSharedDict as SharedDict, schedule
12+ logger.info('Use shared_simple implementation')
13+else:
14+ try:
15+ uwsgi.cache_update('dummy', b'dummy')
16+ if uwsgi.cache_get('dummy') != b'dummy':
17+ raise Exception()
18+ except:
19+ # uwsgi.ini configuration problem: disable all scheduling
20+ logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
21+ 'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1')
22+ from .shared_simple import SimpleSharedDict as SharedDict
23+
24+ def schedule(delay, func, *args):
25+ pass
26+ else:
27+ # uwsgi
28+ from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
29+ logger.info('Use shared_uwsgi implementation')
30+
31+storage = SharedDict()
searx/shared/shared_abstract.pynew file+15
@@ -0,0 +1,15 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+class SharedDict:
4+
5+ def get_int(self, key):
6+ pass
7+
8+ def set_int(self, key, value):
9+ pass
10+
11+ def get_str(self, key):
12+ pass
13+
14+ def set_str(self, key, value):
15+ pass
searx/shared/shared_simple.pynew file+38
@@ -0,0 +1,38 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import threading
4+
5+from . import shared_abstract
6+
7+
8+class SimpleSharedDict(shared_abstract.SharedDict):
9+
10+ __slots__ = 'd',
11+
12+ def __init__(self):
13+ self.d = {}
14+
15+ def get_int(self, key):
16+ return self.d.get(key, None)
17+
18+ def set_int(self, key, value):
19+ self.d[key] = value
20+
21+ def get_str(self, key):
22+ return self.d.get(key, None)
23+
24+ def set_str(self, key, value):
25+ self.d[key] = value
26+
27+
28+def schedule(delay, func, *args):
29+ def call_later():
30+ t = threading.Timer(delay, wrapper)
31+ t.daemon = True
32+ t.start()
33+
34+ def wrapper():
35+ call_later()
36+ func(*args)
37+
38+ call_later()
searx/shared/shared_uwsgi.pynew file+62
@@ -0,0 +1,62 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+import time
4+import uwsgi # pylint: disable=E0401
5+from . import shared_abstract
6+
7+
8+_last_signal = 10
9+
10+
11+class UwsgiCacheSharedDict(shared_abstract.SharedDict):
12+
13+ def get_int(self, key):
14+ value = uwsgi.cache_get(key)
15+ if value is None:
16+ return value
17+ else:
18+ return int.from_bytes(value, 'big')
19+
20+ def set_int(self, key, value):
21+ b = value.to_bytes(4, 'big')
22+ uwsgi.cache_update(key, b)
23+
24+ def get_str(self, key):
25+ value = uwsgi.cache_get(key)
26+ if value is None:
27+ return value
28+ else:
29+ return value.decode('utf-8')
30+
31+ def set_str(self, key, value):
32+ b = value.encode('utf-8')
33+ uwsgi.cache_update(key, b)
34+
35+
36+def schedule(delay, func, *args):
37+ """
38+ Can be implemented using a spooler.
39+ https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html
40+
41+ To make the uwsgi configuration simple, use the alternative implementation.
42+ """
43+ global _last_signal
44+
45+ def sighandler(signum):
46+ now = int(time.time())
47+ uwsgi.lock()
48+ try:
49+ updating = uwsgi.cache_get('updating')
50+ if updating is not None:
51+ updating = int.from_bytes(updating, 'big')
52+ if now - updating < delay:
53+ return
54+ uwsgi.cache_update('updating', now.to_bytes(4, 'big'))
55+ finally:
56+ uwsgi.unlock()
57+ func(*args)
58+
59+ signal_num = _last_signal
60+ _last_signal += 1
61+ uwsgi.register_signal(signal_num, 'worker', sighandler)
62+ uwsgi.add_timer(signal_num, delay)
utils/templates/etc/uwsgi/apps-archlinux/searx.ini+4−1
@@ -82,4 +82,7 @@ http = ${SEARX_INTERNAL_HTTP}
8282# mkdir -p /run/uwsgi/app/searx
8383# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
8484#
85−# socket = /run/uwsgi/app/searx/socket
85+# socket = /run/uwsgi/app/searx/socket
86+
87+# Cache
88+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1
utils/templates/etc/uwsgi/apps-available/searx.ini+3
@@ -82,3 +82,6 @@ http = ${SEARX_INTERNAL_HTTP}
8282# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
8383#
8484# socket = /run/uwsgi/app/searx/socket
85+
86+# Cache
87+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1