git.iko.soy/searx
on master help sign in
Merge pull request #1186 from kvch/fix-bing-videos

Fix Bing videos engine

commit360f8fab97a364e2f5e7fc8c5329cccb1ad5654a authorAdam Tauber <asciimoo@gmail.com> dateSun 11 Feb 2018, 01:01 (9 years ago) parentccc69553ef8533
2 files changed+38−83
compare with master
unifiedsplit
searx/engines/bing_videos.py+5−17 tests/unit/engines/test_bing_videos.py+33−66
searx/engines/bing_videos.py+5−17
@@ -69,22 +69,11 @@ def response(resp):
6969 dom = html.fromstring(resp.text)
7070
7171 for result in dom.xpath('//div[@class="dg_u"]'):
72−
73− # try to extract the url
74− url_container = result.xpath('.//div[@class="sa_wrapper"]/@data-eventpayload')
75− if len(url_container) > 0:
76− url = loads(url_container[0])['purl']
77− else:
78− url = result.xpath('./a/@href')[0]
79−
80− # discard results that do not return an external url
81− # very recent results sometimes don't return the video's url
82− if url.startswith('/videos/search?'):
83− continue
84−
85− title = extract_text(result.xpath('./a//div[@class="tl"]'))
86− content = extract_text(result.xpath('.//div[@class="pubInfo"]'))
87− thumbnail = result.xpath('.//div[@class="vthumb"]/img/@src')[0]
72+ url = result.xpath('./div[@class="mc_vtvc"]/a/@href')[0]
73+ url = 'https://bing.com' + url
74+ title = extract_text(result.xpath('./div/a/div/div[@class="mc_vtvc_title"]/@title'))
75+ content = extract_text(result.xpath('./div/a/div/div/div/div/text()'))
76+ thumbnail = result.xpath('./div/a/div/div/img/@src')[0]
8877
8978 results.append({'url': url,
9079 'title': title,
@@ -92,7 +81,6 @@ def response(resp):
9281 'thumbnail': thumbnail,
9382 'template': 'videos.html'})
9483
95− # first page ignores requested number of results
9684 if len(results) >= number_of_results:
9785 break
9886
tests/unit/engines/test_bing_videos.py+33−66
@@ -47,87 +47,54 @@ class TestBingVideosEngine(SearxTestCase):
4747 self.assertEqual(bing_videos.response(response), [])
4848
4949 html = """
50− <div>
51− <div class="dg_u">
52− <a class="dv_i" href="/videos/search?abcde">
53− <div class="vthblock">
54− <div class="vthumb">
50+ <div class="dg_u">
51+ <div id="mc_vtvc_1" class="mc_vtvc">
52+ <a class="mc_vtvc_link" href="/video">
53+ <div class="mc_vtvc_th">
54+ <div class="cico">
5555 <img src="thumb_1.jpg" />
5656 </div>
57− <div>
58− <div class="tl">
59− Title 1
57+ <div class="mc_vtvc_ban_lo">
58+ <div class="vtbc">
59+ <div class="mc_bc_w b_smText">
60+ <div class="mc_bc pivot bpi_2">
61+ <span title="">
62+ <span class="mv_vtvc_play cipg "></span>
63+ </span>
64+ </div>
65+ <div class="mc_bc items">10:06</div>
66+ </div>
6067 </div>
6168 </div>
62− </div>
63− <div class="videoInfoPanel">
64− <div class="pubInfo">
65− <div>Content 1</div>
6669 </div>
67− </div>
68− </a>
69− <div class="sa_wrapper"
70− data-eventpayload="{&quot;purl&quot;: &quot;https://url.com/1&quot;}">
71− </div>
72− </div>
73− </div>
74− """
75− response = mock.Mock(text=html)
76− results = bing_videos.response(response)
77− self.assertEqual(type(results), list)
78− self.assertEqual(len(results), 1)
79− self.assertEqual(results[0]['title'], 'Title 1')
80− self.assertEqual(results[0]['url'], 'https://url.com/1')
81− self.assertEqual(results[0]['content'], 'Content 1')
82− self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg')
83−
84− html = """
85− <div>
86− <div class="dg_u">
87− <a class="dv_i" href="https://url.com/1">
88− <div class="vthblock">
89− <div class="vthumb">
90− <img src="thumb_1.jpg" />
91− </div>
92− <div>
93− <div class="tl">
94− Title 1
95− </div>
96− </div>
97− </div>
98− <div class="videoInfoPanel">
99− <div class="pubInfo">
100− <div>Content 1</div>
101− </div>
102− </div>
103− </a>
104− </div>
105− <div class="dg_u">
106− <a class="dv_i" href="/videos/search?abcde">
107− <div class="vthblock">
108− <div class="vthumb">
109− <img src="thumb_2.jpg" />
110− </div>
111− <div>
112− <div class="tl">
113− Title 2
70+ <div class="mc_vtvc_meta">
71+ <div class="mc_vtvc_title" title="Title 1"></div>
72+ <div class="mc_vtvc_meta_block_area">
73+ <div class="mc_vtvc_meta_block">
74+ <div class="mc_vtvc_meta_row">
75+ <span>65,696,000+ views</span>
76+ <span>1 year ago</span>
11477 </div>
78+ <div class="mc_vtvc_meta_row mc_vtvc_meta_row_channel">Content 1</div>
79+ <div class="mc_vtvc_meta_row"><span>
80+ <div class="cico mc_vtvc_src_ico">
81+ <div></div>
82+ </div>
83+ <span>YouTube</span>
84+ </span></div>
11585 </div>
116− </div>
117− <div class="videoInfoPanel">
118− <div class="pubInfo">
119− <div>Content 2</div>
12086 </div>
12187 </div>
122− </a>
88+ <div class="vrhdata"></div>
89+ </a>
90+ </div>
12391 </div>
124− </div>
12592 """
12693 response = mock.Mock(text=html)
12794 results = bing_videos.response(response)
12895 self.assertEqual(type(results), list)
12996 self.assertEqual(len(results), 1)
13097 self.assertEqual(results[0]['title'], 'Title 1')
131− self.assertEqual(results[0]['url'], 'https://url.com/1')
98+ self.assertEqual(results[0]['url'], 'https://bing.com/video')
13299 self.assertEqual(results[0]['content'], 'Content 1')
133100 self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg')