git.iko.soy/searx
on master help sign in
Fix bing engine results count (#1387)

This PR fixes the result count from bing which was throwing an (hidden) error and add a validation to avoid reading more results than avalaible. For example : If there is 100 results from some search and we try to get results from 120 to 130, Bing will send back the results from 0 to 10 and no error. If we compare results count with the first parameter of the request we can avoid this "invalid" results.

commit88261e111ca2186f080c4048ab41b4c54cd5cf87 authorLéo Bourrel <leo.bourrel@epitech.eu> dateMon 5 Aug 2019, 16:15 (7 years ago) parent12f891d
2 files changed+119−43
compare with master
unifiedsplit
searx/engines/bing.py+28−9 tests/unit/engines/test_bing.py+91−34
searx/engines/bing.py+28−9
@@ -13,11 +13,15 @@
1313 @todo publishedDate
1414"""
1515
16+import re
1617from lxml import html
18+from searx import logger, utils
1719from searx.engines.xpath import extract_text
1820from searx.url_utils import urlencode
1921from searx.utils import match_language, gen_useragent
2022
23+logger = logger.getChild('bing engine')
24+
2125# engine dependent config
2226categories = ['general']
2327paging = True
@@ -30,9 +34,13 @@ base_url = 'https://www.bing.com/'
3034search_string = 'search?{query}&first={offset}'
3135
3236
37+def _get_offset_from_pageno(pageno):
38+ return (pageno - 1) * 10 + 1
39+
40+
3341# do search-request
3442def request(query, params):
35− offset = (params['pageno'] - 1) * 10 + 1
43+ offset = _get_offset_from_pageno(params.get('pageno', 0))
3644
3745 if params['language'] == 'all':
3846 lang = 'EN'
@@ -53,15 +61,9 @@ def request(query, params):
5361# get response from search-request
5462def response(resp):
5563 results = []
64+ result_len = 0
5665
5766 dom = html.fromstring(resp.text)
58−
59− try:
60− results.append({'number_of_results': int(dom.xpath('//span[@class="sb_count"]/text()')[0]
61− .split()[0].replace(',', ''))})
62− except:
63− pass
64−
6567 # parse results
6668 for result in dom.xpath('//div[@class="sa_cc"]'):
6769 link = result.xpath('.//h3/a')[0]
@@ -86,7 +88,24 @@ def response(resp):
8688 'title': title,
8789 'content': content})
8890
89− # return results
91+ try:
92+ result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))
93+ result_len_container = utils.to_string(result_len_container)
94+ if "-" in result_len_container:
95+ # Remove the part "from-to" for paginated request ...
96+ result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
97+
98+ result_len_container = re.sub('[^0-9]', '', result_len_container)
99+ if len(result_len_container) > 0:
100+ result_len = int(result_len_container)
101+ except Exception as e:
102+ logger.debug('result error :\n%s', e)
103+ pass
104+
105+ if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
106+ return []
107+
108+ results.append({'number_of_results': result_len})
90109 return results
91110
92111
tests/unit/engines/test_bing.py+91−34
@@ -1,3 +1,4 @@
1+# -*- coding: utf-8 -*-
12from collections import defaultdict
23import mock
34from searx.engines import bing
@@ -10,7 +11,7 @@ class TestBingEngine(SearxTestCase):
1011 bing.supported_languages = ['en', 'fr', 'zh-CHS', 'zh-CHT', 'pt-PT', 'pt-BR']
1112 query = u'test_query'
1213 dicto = defaultdict(dict)
13− dicto['pageno'] = 0
14+ dicto['pageno'] = 1
1415 dicto['language'] = 'fr-FR'
1516 params = bing.request(query.encode('utf-8'), dicto)
1617 self.assertTrue('url' in params)
@@ -23,70 +24,126 @@ class TestBingEngine(SearxTestCase):
2324 self.assertTrue('language' in params['url'])
2425
2526 def test_response(self):
27+ dicto = defaultdict(dict)
28+ dicto['pageno'] = 1
29+ dicto['language'] = 'fr-FR'
2630 self.assertRaises(AttributeError, bing.response, None)
2731 self.assertRaises(AttributeError, bing.response, [])
2832 self.assertRaises(AttributeError, bing.response, '')
2933 self.assertRaises(AttributeError, bing.response, '[]')
3034
3135 response = mock.Mock(text='<html></html>')
36+ response.search_params = dicto
3237 self.assertEqual(bing.response(response), [])
3338
3439 response = mock.Mock(text='<html></html>')
40+ response.search_params = dicto
3541 self.assertEqual(bing.response(response), [])
3642
3743 html = """
38− <div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
39− <div Class="sa_mc">
40− <div class="sb_tlst">
41− <h3>
42− <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
43− <strong>This</strong> should be the title</a>
44− </h3>
45− </div>
46− <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
47− <span class="c_tlbxTrg">
48− <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
49− </span>
50− </span>
51− </div>
52− <p><strong>This</strong> should be the content.</p>
44+ <div>
45+ <div id="b_tween">
46+ <span class="sb_count" data-bm="4">23 900 000 résultats</span>
5347 </div>
48+ <ol id="b_results" role="main">
49+ <div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
50+ <div Class="sa_mc">
51+ <div class="sb_tlst">
52+ <h3>
53+ <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
54+ <strong>This</strong> should be the title</a>
55+ </h3>
56+ </div>
57+ <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
58+ <span class="c_tlbxTrg">
59+ <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
60+ </span>
61+ </span>
62+ </div>
63+ <p><strong>This</strong> should be the content.</p>
64+ </div>
65+ </div>
66+ </ol>
5467 </div>
5568 """
5669 response = mock.Mock(text=html)
70+ response.search_params = dicto
5771 results = bing.response(response)
5872 self.assertEqual(type(results), list)
59− self.assertEqual(len(results), 1)
73+ self.assertEqual(len(results), 2)
6074 self.assertEqual(results[0]['title'], 'This should be the title')
6175 self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
6276 self.assertEqual(results[0]['content'], 'This should be the content.')
77+ self.assertEqual(results[-1]['number_of_results'], 23900000)
6378
6479 html = """
65− <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
66− <div Class="sa_mc">
67− <div class="sb_tlst">
68− <h2>
69− <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
70− <strong>This</strong> should be the title</a>
71− </h2>
72− </div>
73− <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
74− <span class="c_tlbxTrg">
75− <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
76− </span>
77− </span>
78− </div>
79− <p><strong>This</strong> should be the content.</p>
80+ <div>
81+ <div id="b_tween">
82+ <span class="sb_count" data-bm="4">9-18 résultats sur 23 900 000</span>
8083 </div>
81− </li>
84+ <ol id="b_results" role="main">
85+ <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
86+ <div Class="sa_mc">
87+ <div class="sb_tlst">
88+ <h2>
89+ <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
90+ <strong>This</strong> should be the title</a>
91+ </h2>
92+ </div>
93+ <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
94+ <span class="c_tlbxTrg">
95+ <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
96+ </span>
97+ </span>
98+ </div>
99+ <p><strong>This</strong> should be the content.</p>
100+ </div>
101+ </li>
102+ </ol>
103+ </div>
82104 """
105+ dicto['pageno'] = 2
83106 response = mock.Mock(text=html)
107+ response.search_params = dicto
84108 results = bing.response(response)
85109 self.assertEqual(type(results), list)
86− self.assertEqual(len(results), 1)
110+ self.assertEqual(len(results), 2)
87111 self.assertEqual(results[0]['title'], 'This should be the title')
88112 self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
89113 self.assertEqual(results[0]['content'], 'This should be the content.')
114+ self.assertEqual(results[-1]['number_of_results'], 23900000)
115+
116+ html = """
117+ <div>
118+ <div id="b_tween">
119+ <span class="sb_count" data-bm="4">23 900 000 résultats</span>
120+ </div>
121+ <ol id="b_results" role="main">
122+ <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
123+ <div Class="sa_mc">
124+ <div class="sb_tlst">
125+ <h2>
126+ <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
127+ <strong>This</strong> should be the title</a>
128+ </h2>
129+ </div>
130+ <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
131+ <span class="c_tlbxTrg">
132+ <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
133+ </span>
134+ </span>
135+ </div>
136+ <p><strong>This</strong> should be the content.</p>
137+ </div>
138+ </li>
139+ </ol>
140+ </div>
141+ """
142+ dicto['pageno'] = 33900000
143+ response = mock.Mock(text=html)
144+ response.search_params = dicto
145+ results = bing.response(response)
146+ self.assertEqual(bing.response(response), [])
90147
91148 def test_fetch_supported_languages(self):
92149 html = """<html></html>"""