on master help sign in
[fix] bing images
commit9ee8e552da4654cd41228ce1a3addfa1fa1f377d authormarc <a01200356@itesm.mx> dateSat 13 May 2017, 02:22 (9 years ago) parentc0bb89f
2 files changed+48−86
searx/engines/bing_images.py+11−10 tests/unit/engines/test_bing_images.py+37−76
searx/engines/bing_images.py+11−10
@@ -31,7 +31,6 @@ time_range_support = True
3131base_url = 'https://www.bing.com/'
3232search_string = 'images/search?{query}&count=10&first={offset}'
3333time_range_string = '&qft=+filterui:age-lt{interval}'
34−thumb_url = "https://www.bing.com/th?id={ihk}"
3534time_range_dict = {'day': '1440',
3635 'week': '10080',
3736 'month': '43200',
@@ -78,30 +77,32 @@ def response(resp):
7877 dom = html.fromstring(resp.text)
7978
8079 # parse results
81− for result in dom.xpath('//div[@class="dg_u"]/div'):
80+ for result in dom.xpath('//div[@id="mmComponent_images_1"]/ul/li/div/div[@class="imgpt"]'):
8281 link = result.xpath('./a')[0]
8382
83+ # TODO find actual title
84+ title = link.xpath('.//img/@alt')[0]
85+
8486 # parse json-data (it is required to add a space, to make it parsable)
8587 json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('m')))
8688
87− title = link.attrib.get('t1')
88− ihk = link.attrib.get('ihk')
89+ url = json_data.get('purl')
90+ img_src = json_data.get('murl')
8991
90− # url = 'http://' + link.attrib.get('t3')
91− url = json_data.get('surl')
92− img_src = json_data.get('imgurl')
92+ thumb_json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('mad')))
93+ thumbnail = thumb_json_data.get('turl')
9394
9495 # append result
9596 results.append({'template': 'images.html',
9697 'url': url,
9798 'title': title,
9899 'content': '',
99− 'thumbnail_src': thumb_url.format(ihk=ihk),
100+ 'thumbnail_src': thumbnail,
100101 'img_src': img_src})
101102
102103 # TODO stop parsing if 10 images are found
103− if len(results) >= 10:
104− break
104+ # if len(results) >= 10:
105+ # break
105106
106107 # return results
107108 return results
tests/unit/engines/test_bing_images.py+37−76
@@ -39,85 +39,46 @@ class TestBingImagesEngine(SearxTestCase):
3939 self.assertEqual(bing_images.response(response), [])
4040
4141 html = """
42− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>
43− <a href="/images/search?q=south&amp;view=detailv2&amp;&amp;id=7E92863981CCFB89FBDD55205C742DFDA3290CF6&amp;selectedIndex=9&amp;ccid=vzvIfv5u&amp;simid=608055786735667000&amp;thid=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0" ihk="OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0" t1="South Carolina" t2="747 x 589 &#183; 29 kB &#183; gif" t3="www.digital-topo-maps.com/county-map/south-carolina.shtml" hh="236" hw="300" m='{ns:"images",k:"5117",mid:"7E92863981CCFB89FBDD55205C742DFDA3290CF6",md5:"bf3bc87efe6e0e476be8cc34bf6cd80e",surl:"http://www.digital-topo-maps.com/county-map/south-carolina.shtml",imgurl:"http://www.digital-topo-maps.com/county-map/south-carolina-county-map.gif",tid:"OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0",ow:"480",docid:"608055786735667000",oh:"378",tft:"45"}' mid="7E92863981CCFB89FBDD55205C742DFDA3290CF6" h="ID=images,5117.1">
44− <img class="img_hid" src2="https://tse4.mm.bing.net/th?id=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0&amp;w=210&amp;h=154&amp;c=7&amp;rs=1&amp;qlt=90&amp;o=4&amp;pid=1.1" style="width:210px;height:154px;" width="210" height="154">
45− </a>
46−
47− </div></div>
48− """ # noqa
49− html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
50− response = mock.Mock(text=html)
51− results = bing_images.response(response)
52− self.assertEqual(type(results), list)
53− self.assertEqual(len(results), 1)
54− self.assertEqual(results[0]['title'], 'South Carolina')
55− self.assertEqual(results[0]['url'],
56− 'http://www.digital-topo-maps.com/county-map/south-carolina.shtml')
57− self.assertEqual(results[0]['content'], '')
58− self.assertEqual(results[0]['thumbnail_src'],
59− 'https://www.bing.com/th?id=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0')
60− self.assertEqual(results[0]['img_src'],
61− 'http://www.digital-topo-maps.com/county-map/south-carolina-county-map.gif')
62−
63− html = """
64− <a href="#" ihk="HN.608003696942779811"
65− m="{ns:&quot;images&quot;,k:&quot;5045&quot;,
66− mid:&quot;59EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;,
67− surl:&quot;http://www.page.url/&quot;,
68− imgurl:&quot;http://test.url/Test%20Query.jpg&quot;,oh:&quot;238&quot;,
69− tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}"
70− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"
71− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">
72− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7"
73− style="height:144px;" width="178" height="144"/>
74− </a>
75− """
76− response = mock.Mock(text=html)
77− results = bing_images.response(response)
78− self.assertEqual(type(results), list)
79− self.assertEqual(len(results), 0)
80−
81− html = """
82− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>
83− <a href="#" ihk="HN.608003696942779811"
84− m="{ns:&quot;images&quot;,k:&quot;5045&quot;,
85−mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;,
86−surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;,
87−oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}"
88− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"
89− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">
90− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7"
91− style="height:144px;" width="178" height="144"/>
92− </a>
93− </div></div>
94− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>
95− <a href="#" ihk="HN.608003696942779811"
96− m="{ns:&quot;images&quot;,k:&quot;5045&quot;,
97−mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;,
98−surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;,
99−oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}"
100− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"
101− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">
102− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7"
103− style="height:144px;" width="178" height="144"/>
104− </a>
105− </div></div>
106− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>
107− <a href="#" ihk="HN.608003696942779811"
108− m="{ns:&quot;images&quot;,k:&quot;5045&quot;,
109−mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;,
110−surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;,
111−oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}"
112− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"
113− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">
114− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7"
115− style="height:144px;" width="178" height="144"/>
116− </a>
117− </div></div>
42+ <div id="mmComponent_images_1">
43+ <ul>
44+ <li>
45+ <div>
46+ <div class="imgpt">
47+ <a m='{"purl":"page_url","murl":"img_url"}' mad='{"turl":"thumb_url"}'>
48+ <img src="" alt="alt text" />
49+ </a>
50+ </div>
51+ <div></div>
52+ </div>
53+ <div>
54+ <div class="imgpt">
55+ <a m='{"purl":"page_url2","murl":"img_url2"}' mad='{"turl":"thumb_url2"}'>
56+ <img src="" alt="alt text 2" />
57+ </a>
58+ </div>
59+ </div>
60+ </li>
61+ </ul>
62+ <ul>
63+ <li>
64+ <div>
65+ <div class="imgpt">
66+ <a m='{"purl":"page_url3","murl":"img_url3"}' mad='{"turl":"thumb_url3"}'>
67+ <img src="" alt="alt text 3" />
68+ </a>
69+ </div>
70+ </div>
71+ </li>
72+ </ul>
73+ </div>
11874 """
11975 html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
12076 response = mock.Mock(text=html)
12177 results = bing_images.response(response)
12278 self.assertEqual(type(results), list)
12379 self.assertEqual(len(results), 3)
80+ self.assertEqual(results[0]['title'], 'alt text')
81+ self.assertEqual(results[0]['url'], 'page_url')
82+ self.assertEqual(results[0]['content'], '')
83+ self.assertEqual(results[0]['thumbnail_src'], 'thumb_url')
84+ self.assertEqual(results[0]['img_src'], 'img_url')