on master help sign in
Merge pull request #1444 from Venca24/devel_google_videos

[fix] google videos engine

commitabcbcec0b5ab9a3108ccc972876fdb60c7911e7a authorNoémi Ványi <kvch@users.noreply.github.com> dateSat 5 Jan 2019, 18:08 (8 years ago) parent899ba5d2456b8f
2 files changed+41−10
searx/engines/google_videos.py+18−6 tests/unit/engines/test_google_videos.py+23−4
searx/engines/google_videos.py+18−6
@@ -7,7 +7,7 @@
77 @using-api no
88 @results HTML
99 @stable no
10− @parse url, title, content
10+ @parse url, title, content, thumbnail
1111"""
1212
1313from datetime import date, timedelta
@@ -15,7 +15,7 @@ from json import loads
1515from lxml import html
1616from searx.engines.xpath import extract_text
1717from searx.url_utils import urlencode
18−
18+import re
1919
2020# engine dependent config
2121categories = ['videos']
@@ -25,7 +25,7 @@ time_range_support = True
2525number_of_results = 10
2626
2727search_url = 'https://www.google.com/search'\
28− '?{query}'\
28+ '?q={query}'\
2929 '&tbm=vid'\
3030 '&{search_options}'
3131time_range_attr = "qdr:{range}"
@@ -69,15 +69,27 @@ def response(resp):
6969 # parse results
7070 for result in dom.xpath('//div[@class="g"]'):
7171
72− title = extract_text(result.xpath('.//h3/a'))
73− url = result.xpath('.//h3/a/@href')[0]
72+ title = extract_text(result.xpath('.//h3'))
73+ url = result.xpath('.//div[@class="r"]/a/@href')[0]
7474 content = extract_text(result.xpath('.//span[@class="st"]'))
7575
76+ # get thumbnails
77+ script = str(dom.xpath('//script[contains(., "_setImagesSrc")]')[0].text)
78+ id = result.xpath('.//div[@class="s"]//img/@id')[0]
79+ thumbnails_data = re.findall('s=\'(.*?)(?:\\\\[a-z,1-9,\\\\]+\'|\')\;var ii=\[(?:|[\'vidthumb\d+\',]+)\'' + id,
80+ script)
81+ tmp = []
82+ if len(thumbnails_data) != 0:
83+ tmp = re.findall('(data:image/jpeg;base64,[a-z,A-Z,0-9,/,\+]+)', thumbnails_data[0])
84+ thumbnail = ''
85+ if len(tmp) != 0:
86+ thumbnail = tmp[-1]
87+
7688 # append result
7789 results.append({'url': url,
7890 'title': title,
7991 'content': content,
80− 'thumbnail': '',
92+ 'thumbnail': thumbnail,
8193 'template': 'videos.html'})
8294
8395 return results
tests/unit/engines/test_google_videos.py+23−4
@@ -30,16 +30,34 @@ class TestGoogleVideosEngine(SearxTestCase):
3030 <div>
3131 <div>
3232 <div class="g">
33− <div>
34− <h3><a href="url_1">Title 1</h3>
33+ <div class="r">
34+ <a href="url_1"><h3>Title 1</h3></a>
35+ </div>
36+ <div class="s">
37+ <div>
38+ <a>
39+ <g-img>
40+ <img id="vidthumb1">
41+ </g-img>
42+ </a>
43+ </div>
3544 </div>
3645 <div>
3746 <span class="st">Content 1</span>
3847 </div>
3948 </div>
4049 <div class="g">
41− <div>
42− <h3><a href="url_2">Title 2</h3>
50+ <div class="r">
51+ <a href="url_2"><h3>Title 2</h3></a>
52+ </div>
53+ <div class="s">
54+ <div>
55+ <a>
56+ <g-img>
57+ <img id="vidthumb2">
58+ </g-img>
59+ </a>
60+ </div>
4361 </div>
4462 <div>
4563 <span class="st">Content 2</span>
@@ -47,6 +65,7 @@ class TestGoogleVideosEngine(SearxTestCase):
4765 </div>
4866 </div>
4967 </div>
68+ <script>function _setImagesSrc(c,d,e){}</script>
5069 """
5170 response = mock.Mock(text=html)
5271 results = google_videos.response(response)