git.iko.soy/searx
on master help sign in
Merge branch 'master' into dsgvo
78be030cy8aer
searx/engines/bing.py+4−1
compare with master
unifiedsplit
@@ -16,7 +16,7 @@
16from lxml import html
16from lxml import html
17from searx.engines.xpath import extract_text
17from searx.engines.xpath import extract_text
18from searx.url_utils import urlencode
18from searx.url_utils import urlencode
19−from searx.utils import match_language
19+from searx.utils import match_language, gen_useragent
20
20
21# engine dependent config
21# engine dependent config
22categories = ['general']
22categories = ['general']
@@ -43,6 +43,9 @@ def request(query, params):
43 offset=offset)
43 offset=offset)
44
44
45 params['url'] = base_url + search_path
45 params['url'] = base_url + search_path
46+
47+ params['headers']['User-Agent'] = gen_useragent('Windows NT 6.3; WOW64')
48+
46 return params
49 return params
47
50
48
51
searx/engines/wikidata.py+2−1
@@ -54,6 +54,7 @@ value_xpath = './/div[contains(@class,"wikibase-statementview-mainsnak")]'\
54 + '/*/div[contains(@class,"wikibase-snakview-value")]'
54 + '/*/div[contains(@class,"wikibase-snakview-value")]'
55language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]'
55language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]'
56calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
56calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
57+media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
57
58
58
59
59def request(query, params):
60def request(query, params):
@@ -313,7 +314,7 @@ def add_image(result):
313 for property_id in property_ids:
314 for property_id in property_ids:
314 image = result.xpath(property_xpath.replace('{propertyid}', property_id))
315 image = result.xpath(property_xpath.replace('{propertyid}', property_id))
315 if image:
316 if image:
316− image_name = image[0].xpath(value_xpath)
317+ image_name = image[0].xpath(media_xpath)
317 image_src = url_image.replace('{filename}', extract_text(image_name[0]))
318 image_src = url_image.replace('{filename}', extract_text(image_name[0]))
318 return image_src
319 return image_src
319
320
searx/utils.py+11−10
@@ -39,14 +39,15 @@ else:
39
39
40logger = logger.getChild('utils')
40logger = logger.getChild('utils')
41
41
42−ua_versions = ('40.0',
42+ua_versions = ('52.8.1',
43− '41.0',
43+ '53.0',
44− '42.0',
44+ '54.0',
45− '43.0',
45+ '55.0',
46− '44.0',
46+ '56.0',
47− '45.0',
47+ '57.0',
48− '46.0',
48+ '58.0',
49− '47.0')
49+ '59.0',
50+ '60.0.2')
50
51
51ua_os = ('Windows NT 6.3; WOW64',
52ua_os = ('Windows NT 6.3; WOW64',
52 'X11; Linux x86_64',
53 'X11; Linux x86_64',
@@ -58,9 +59,9 @@ blocked_tags = ('script',
58 'style')
59 'style')
59
60
60
61
61−def gen_useragent():
62+def gen_useragent(os=None):
62 # TODO
63 # TODO
63− return ua.format(os=choice(ua_os), version=choice(ua_versions))
64+ return ua.format(os=os or choice(ua_os), version=choice(ua_versions))
64
65
65
66
66def searx_useragent():
67def searx_useragent():
tests/unit/engines/test_wikidata.py+12−9
@@ -123,9 +123,10 @@ class TestWikidataEngine(SearxTestCase):
123 <div class="wikibase-statementview-mainsnak">
123 <div class="wikibase-statementview-mainsnak">
124 <div>
124 <div>
125 <div class="wikibase-snakview-value">
125 <div class="wikibase-snakview-value">
126+ <div class="commons-media-caption">
126− <a href="https://commons.wikimedia.org/wiki/File:image.png">
127+ <a href="https://commons.wikimedia.org/wiki/File:image.png">image.png</a>
127− image.png
128+ <br/>2,687 &#215; 3,356; 1.22 MB
128− </a>
129+ </div>
129 </div>
130 </div>
130 </div>
131 </div>
131 </div>
132 </div>
@@ -156,9 +157,10 @@ class TestWikidataEngine(SearxTestCase):
156 <div class="wikibase-statementview-mainsnak">
157 <div class="wikibase-statementview-mainsnak">
157 <div>
158 <div>
158 <div class="wikibase-snakview-value">
159 <div class="wikibase-snakview-value">
160+ <div class="commons-media-caption">
159− <a href="https://commons.wikimedia.org/wiki/File:icon.png">
161+ <a href="https://commons.wikimedia.org/wiki/File:icon.png">icon.png</a>
160− icon.png
162+ <br/>671 &#215; 671; 18 KB</div>
161− </a>
163+ </div>
162 </div>
164 </div>
163 </div>
165 </div>
164 </div>
166 </div>
@@ -179,9 +181,10 @@ class TestWikidataEngine(SearxTestCase):
179 <div class="wikibase-statementview-mainsnak">
181 <div class="wikibase-statementview-mainsnak">
180 <div>
182 <div>
181 <div class="wikibase-snakview-value">
183 <div class="wikibase-snakview-value">
184+ <div class="commons-media-caption">
182− <a href="https://commons.wikimedia.org/wiki/File:logo.png">
185+ <a href="https://commons.wikimedia.org/wiki/File:logo.png">logo.png</a>
183− logo.png
186+ <br/>170 &#215; 170; 1 KB
184− </a>
187+ </div>
185 </div>
188 </div>
186 </div>
189 </div>
187 </div>
190 </div>