mirror of
https://github.com/ilyakooo0/searx.git
synced 2024-09-11 09:15:54 +03:00
[enh] add searx.shared
shared dictionary between the workers (UWSGI or werkzeug) scheduler: run a task once every x seconds (UWSGI or werkzeug)
This commit is contained in:
parent
9c581466e1
commit
6e2872f436
@ -42,3 +42,6 @@ static-map = /static=/usr/local/searx/searx/static
|
||||
static-expires = /* 864000
|
||||
static-gzip-all = True
|
||||
offload-threads = %k
|
||||
|
||||
# Cache
|
||||
cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1
|
||||
|
31
searx/shared/__init__.py
Normal file
31
searx/shared/__init__.py
Normal file
@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger('searx.shared')
|
||||
|
||||
try:
|
||||
import uwsgi
|
||||
except:
|
||||
# no uwsgi
|
||||
from .shared_simple import SimpleSharedDict as SharedDict, schedule
|
||||
logger.info('Use shared_simple implementation')
|
||||
else:
|
||||
try:
|
||||
uwsgi.cache_update('dummy', b'dummy')
|
||||
if uwsgi.cache_get('dummy') != b'dummy':
|
||||
raise Exception()
|
||||
except:
|
||||
# uwsgi.ini configuration problem: disable all scheduling
|
||||
logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
|
||||
'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1')
|
||||
from .shared_simple import SimpleSharedDict as SharedDict
|
||||
|
||||
def schedule(delay, func, *args):
|
||||
pass
|
||||
else:
|
||||
# uwsgi
|
||||
from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
|
||||
logger.info('Use shared_uwsgi implementation')
|
||||
|
||||
storage = SharedDict()
|
15
searx/shared/shared_abstract.py
Normal file
15
searx/shared/shared_abstract.py
Normal file
@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
class SharedDict:
|
||||
|
||||
def get_int(self, key):
|
||||
pass
|
||||
|
||||
def set_int(self, key, value):
|
||||
pass
|
||||
|
||||
def get_str(self, key):
|
||||
pass
|
||||
|
||||
def set_str(self, key, value):
|
||||
pass
|
38
searx/shared/shared_simple.py
Normal file
38
searx/shared/shared_simple.py
Normal file
@ -0,0 +1,38 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import threading
|
||||
|
||||
from . import shared_abstract
|
||||
|
||||
|
||||
class SimpleSharedDict(shared_abstract.SharedDict):
|
||||
|
||||
__slots__ = 'd',
|
||||
|
||||
def __init__(self):
|
||||
self.d = {}
|
||||
|
||||
def get_int(self, key):
|
||||
return self.d.get(key, None)
|
||||
|
||||
def set_int(self, key, value):
|
||||
self.d[key] = value
|
||||
|
||||
def get_str(self, key):
|
||||
return self.d.get(key, None)
|
||||
|
||||
def set_str(self, key, value):
|
||||
self.d[key] = value
|
||||
|
||||
|
||||
def schedule(delay, func, *args):
|
||||
def call_later():
|
||||
t = threading.Timer(delay, wrapper)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
def wrapper():
|
||||
call_later()
|
||||
func(*args)
|
||||
|
||||
call_later()
|
62
searx/shared/shared_uwsgi.py
Normal file
62
searx/shared/shared_uwsgi.py
Normal file
@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import time
|
||||
import uwsgi # pylint: disable=E0401
|
||||
from . import shared_abstract
|
||||
|
||||
|
||||
_last_signal = 10
|
||||
|
||||
|
||||
class UwsgiCacheSharedDict(shared_abstract.SharedDict):
|
||||
|
||||
def get_int(self, key):
|
||||
value = uwsgi.cache_get(key)
|
||||
if value is None:
|
||||
return value
|
||||
else:
|
||||
return int.from_bytes(value, 'big')
|
||||
|
||||
def set_int(self, key, value):
|
||||
b = value.to_bytes(4, 'big')
|
||||
uwsgi.cache_update(key, b)
|
||||
|
||||
def get_str(self, key):
|
||||
value = uwsgi.cache_get(key)
|
||||
if value is None:
|
||||
return value
|
||||
else:
|
||||
return value.decode('utf-8')
|
||||
|
||||
def set_str(self, key, value):
|
||||
b = value.encode('utf-8')
|
||||
uwsgi.cache_update(key, b)
|
||||
|
||||
|
||||
def schedule(delay, func, *args):
|
||||
"""
|
||||
Can be implemented using a spooler.
|
||||
https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html
|
||||
|
||||
To make the uwsgi configuration simple, use the alternative implementation.
|
||||
"""
|
||||
global _last_signal
|
||||
|
||||
def sighandler(signum):
|
||||
now = int(time.time())
|
||||
uwsgi.lock()
|
||||
try:
|
||||
updating = uwsgi.cache_get('updating')
|
||||
if updating is not None:
|
||||
updating = int.from_bytes(updating, 'big')
|
||||
if now - updating < delay:
|
||||
return
|
||||
uwsgi.cache_update('updating', now.to_bytes(4, 'big'))
|
||||
finally:
|
||||
uwsgi.unlock()
|
||||
func(*args)
|
||||
|
||||
signal_num = _last_signal
|
||||
_last_signal += 1
|
||||
uwsgi.register_signal(signal_num, 'worker', sighandler)
|
||||
uwsgi.add_timer(signal_num, delay)
|
@ -82,4 +82,7 @@ http = ${SEARX_INTERNAL_HTTP}
|
||||
# mkdir -p /run/uwsgi/app/searx
|
||||
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
|
||||
#
|
||||
# socket = /run/uwsgi/app/searx/socket
|
||||
# socket = /run/uwsgi/app/searx/socket
|
||||
|
||||
# Cache
|
||||
cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1
|
||||
|
@ -82,3 +82,6 @@ http = ${SEARX_INTERNAL_HTTP}
|
||||
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
|
||||
#
|
||||
# socket = /run/uwsgi/app/searx/socket
|
||||
|
||||
# Cache
|
||||
cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1
|
||||
|
Loading…
Reference in New Issue
Block a user