git.iko.soy/searx
on master help sign in
Merge pull request #1186 from kvch/fix-bing-videos
360f8faAdam Tauber
searx/engines/bing_videos.py+5−17
compare with master
unifiedsplit
@@ -69,22 +69,11 @@ def response(resp):
69 dom = html.fromstring(resp.text)
69 dom = html.fromstring(resp.text)
70
70
71 for result in dom.xpath('//div[@class="dg_u"]'):
71 for result in dom.xpath('//div[@class="dg_u"]'):
72−
73− # try to extract the url
74− url_container = result.xpath('.//div[@class="sa_wrapper"]/@data-eventpayload')
72+ url = result.xpath('./div[@class="mc_vtvc"]/a/@href')[0]
75− if len(url_container) > 0:
73+ url = 'https://bing.com' + url
76− url = loads(url_container[0])['purl']
77− else:
78− url = result.xpath('./a/@href')[0]
79−
80− # discard results that do not return an external url
81− # very recent results sometimes don't return the video's url
82− if url.startswith('/videos/search?'):
83− continue
84−
85− title = extract_text(result.xpath('./a//div[@class="tl"]'))
74+ title = extract_text(result.xpath('./div/a/div/div[@class="mc_vtvc_title"]/@title'))
86− content = extract_text(result.xpath('.//div[@class="pubInfo"]'))
75+ content = extract_text(result.xpath('./div/a/div/div/div/div/text()'))
87− thumbnail = result.xpath('.//div[@class="vthumb"]/img/@src')[0]
76+ thumbnail = result.xpath('./div/a/div/div/img/@src')[0]
88
77
89 results.append({'url': url,
78 results.append({'url': url,
90 'title': title,
79 'title': title,
@@ -92,7 +81,6 @@ def response(resp):
92 'thumbnail': thumbnail,
81 'thumbnail': thumbnail,
93 'template': 'videos.html'})
82 'template': 'videos.html'})
94
83
95− # first page ignores requested number of results
96 if len(results) >= number_of_results:
84 if len(results) >= number_of_results:
97 break
85 break
98
86
tests/unit/engines/test_bing_videos.py+33−66
@@ -47,87 +47,54 @@ class TestBingVideosEngine(SearxTestCase):
47 self.assertEqual(bing_videos.response(response), [])
47 self.assertEqual(bing_videos.response(response), [])
48
48
49 html = """
49 html = """
50− <div>
50+ <div class="dg_u">
51− <div class="dg_u">
51+ <div id="mc_vtvc_1" class="mc_vtvc">
52− <a class="dv_i" href="/videos/search?abcde">
52+ <a class="mc_vtvc_link" href="/video">
53− <div class="vthblock">
53+ <div class="mc_vtvc_th">
54− <div class="vthumb">
54+ <div class="cico">
55 <img src="thumb_1.jpg" />
55 <img src="thumb_1.jpg" />
56 </div>
56 </div>
57− <div>
57+ <div class="mc_vtvc_ban_lo">
58− <div class="tl">
58+ <div class="vtbc">
59− Title 1
59+ <div class="mc_bc_w b_smText">
60+ <div class="mc_bc pivot bpi_2">
61+ <span title="">
62+ <span class="mv_vtvc_play cipg "></span>
63+ </span>
64+ </div>
65+ <div class="mc_bc items">10:06</div>
66+ </div>
60 </div>
67 </div>
61 </div>
68 </div>
62− </div>
63− <div class="videoInfoPanel">
64− <div class="pubInfo">
65− <div>Content 1</div>
66 </div>
69 </div>
67− </div>
68− </a>
69− <div class="sa_wrapper"
70+ <div class="mc_vtvc_meta">
70− data-eventpayload="{&quot;purl&quot;: &quot;https://url.com/1&quot;}">
71+ <div class="mc_vtvc_title" title="Title 1"></div>
71− </div>
72+ <div class="mc_vtvc_meta_block_area">
72− </div>
73− </div>
74− """
75− response = mock.Mock(text=html)
76− results = bing_videos.response(response)
77− self.assertEqual(type(results), list)
78− self.assertEqual(len(results), 1)
79− self.assertEqual(results[0]['title'], 'Title 1')
80− self.assertEqual(results[0]['url'], 'https://url.com/1')
81− self.assertEqual(results[0]['content'], 'Content 1')
82− self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg')
83−
84− html = """
85− <div>
86− <div class="dg_u">
87− <a class="dv_i" href="https://url.com/1">
88− <div class="vthblock">
89− <div class="vthumb">
90− <img src="thumb_1.jpg" />
91− </div>
92− <div>
93− <div class="tl">
73+ <div class="mc_vtvc_meta_block">
94− Title 1
74+ <div class="mc_vtvc_meta_row">
95− </div>
75+ <span>65,696,000+ views</span>
96− </div>
76+ <span>1 year ago</span>
97− </div>
98− <div class="videoInfoPanel">
99− <div class="pubInfo">
100− <div>Content 1</div>
101− </div>
102− </div>
103− </a>
104− </div>
105− <div class="dg_u">
106− <a class="dv_i" href="/videos/search?abcde">
107− <div class="vthblock">
108− <div class="vthumb">
109− <img src="thumb_2.jpg" />
110− </div>
111− <div>
112− <div class="tl">
113− Title 2
114 </div>
77 </div>
78+ <div class="mc_vtvc_meta_row mc_vtvc_meta_row_channel">Content 1</div>
79+ <div class="mc_vtvc_meta_row"><span>
80+ <div class="cico mc_vtvc_src_ico">
81+ <div></div>
82+ </div>
83+ <span>YouTube</span>
84+ </span></div>
115 </div>
85 </div>
116− </div>
117− <div class="videoInfoPanel">
118− <div class="pubInfo">
119− <div>Content 2</div>
120 </div>
86 </div>
121 </div>
87 </div>
88+ <div class="vrhdata"></div>
122− </a>
89+ </a>
90+ </div>
123 </div>
91 </div>
124− </div>
125 """
92 """
126 response = mock.Mock(text=html)
93 response = mock.Mock(text=html)
127 results = bing_videos.response(response)
94 results = bing_videos.response(response)
128 self.assertEqual(type(results), list)
95 self.assertEqual(type(results), list)
129 self.assertEqual(len(results), 1)
96 self.assertEqual(len(results), 1)
130 self.assertEqual(results[0]['title'], 'Title 1')
97 self.assertEqual(results[0]['title'], 'Title 1')
131− self.assertEqual(results[0]['url'], 'https://url.com/1')
98+ self.assertEqual(results[0]['url'], 'https://bing.com/video')
132 self.assertEqual(results[0]['content'], 'Content 1')
99 self.assertEqual(results[0]['content'], 'Content 1')
133 self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg')
100 self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg')