git.iko.soy/searx
on master help sign in
[enh] Improve ranking based on language (#3053)

Add configurable setting to rank search results higher when part of the domain (e.g. 'en' in 'en.wikipedia.org' or 'de' in 'beispiel.de') matches the selected search language. Does not apply to e.g. 'be' in 'youtube.com'. Closes #206

commit8c3454fd1be4b578db5693ff1e939af238da6ac8 authorFinn <34682885+0xhtml@users.noreply.github.com> dateMon 15 Nov 2021, 20:31 (5 years ago) parenta880920
4 files changed+18−9
compare with master
unifiedsplit
searx/results.py+12−4 searx/search/__init__.py+1−1 searx/settings.yml+1 tests/unit/test_results.py+4−4
searx/results.py+12−4
@@ -6,6 +6,7 @@ from urllib.parse import urlparse, unquote
66from searx import logger
77from searx.engines import engines
88from searx.metrology.error_recorder import record_error
9+from searx import settings
910
1011
1112CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
@@ -129,13 +130,18 @@ def merge_two_infoboxes(infobox1, infobox2):
129130 infobox1['content'] = content2
130131
131132
132−def result_score(result):
133+def result_score(result, language):
133134 weight = 1.0
134135
135136 for result_engine in result['engines']:
136137 if hasattr(engines[result_engine], 'weight'):
137138 weight *= float(engines[result_engine].weight)
138139
140+ if settings['search']['prefer_configured_language']:
141+ domain_parts = result['parsed_url'].netloc.split('.')
142+ if language in domain_parts:
143+ weight *= 1.1
144+
139145 occurences = len(result['positions'])
140146
141147 return sum((occurences * weight) / position for position in result['positions'])
@@ -145,9 +151,10 @@ class ResultContainer:
145151 """docstring for ResultContainer"""
146152
147153 __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\
148− '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url', 'engine_data'
154+ '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url', 'engine_data',\
155+ '_language'
149156
150− def __init__(self):
157+ def __init__(self, language):
151158 super().__init__()
152159 self._merged_results = []
153160 self.infoboxes = []
@@ -161,6 +168,7 @@ class ResultContainer:
161168 self.unresponsive_engines = set()
162169 self.timings = []
163170 self.redirect_url = None
171+ self._language = language.lower().split('-')[0]
164172
165173 def extend(self, engine_name, results):
166174 standard_result_count = 0
@@ -299,7 +307,7 @@ class ResultContainer:
299307
300308 def order_results(self):
301309 for result in self._merged_results:
302− score = result_score(result)
310+ score = result_score(result, self._language)
303311 result['score'] = score
304312 with RLock():
305313 for result_engine in result['engines']:
searx/search/__init__.py+1−1
@@ -66,7 +66,7 @@ class Search:
6666 # init vars
6767 super().__init__()
6868 self.search_query = search_query
69− self.result_container = ResultContainer()
69+ self.result_container = ResultContainer(search_query.lang)
7070 self.start_time = None
7171 self.actual_timeout = None
7272
searx/settings.yml+1
@@ -19,6 +19,7 @@ search:
1919 default_lang : "" # Default search language - leave blank to detect from browser information or use codes from 'languages.py'
2020 ban_time_on_fail : 5 # ban time in seconds after engine errors
2121 max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
22+ prefer_configured_language: False # increase weight of results in confiugred language in ranking
2223
2324server:
2425 port : 8888
tests/unit/test_results.py+4−4
@@ -20,22 +20,22 @@ def fake_result(url='https://aa.bb/cc?dd=ee#ff',
2020class ResultContainerTestCase(SearxTestCase):
2121
2222 def test_empty(self):
23− c = ResultContainer()
23+ c = ResultContainer("en-US")
2424 self.assertEqual(c.get_ordered_results(), [])
2525
2626 def test_one_result(self):
27− c = ResultContainer()
27+ c = ResultContainer("en-US")
2828 c.extend('wikipedia', [fake_result()])
2929 self.assertEqual(c.results_length(), 1)
3030
3131 def test_one_suggestion(self):
32− c = ResultContainer()
32+ c = ResultContainer("en-US")
3333 c.extend('wikipedia', [fake_result(suggestion=True)])
3434 self.assertEqual(len(c.suggestions), 1)
3535 self.assertEqual(c.results_length(), 0)
3636
3737 def test_result_merge(self):
38− c = ResultContainer()
38+ c = ResultContainer("en-US")
3939 c.extend('wikipedia', [fake_result()])
4040 c.extend('wikidata', [fake_result(), fake_result(url='https://example.com/')])
4141 self.assertEqual(c.results_length(), 2)