git.iko.soy/searx
on master help sign in
Merge pull request #1444 from Venca24/devel_google_videos
abcbcecNoémi Ványi
searx/engines/google_videos.py+18−6
compare with master
unifiedsplit
@@ -7,7 +7,7 @@
7 @using-api no
7 @using-api no
8 @results HTML
8 @results HTML
9 @stable no
9 @stable no
10− @parse url, title, content
10+ @parse url, title, content, thumbnail
11"""
11"""
12
12
13from datetime import date, timedelta
13from datetime import date, timedelta
@@ -15,7 +15,7 @@ from json import loads
15from lxml import html
15from lxml import html
16from searx.engines.xpath import extract_text
16from searx.engines.xpath import extract_text
17from searx.url_utils import urlencode
17from searx.url_utils import urlencode
18−
18+import re
19
19
20# engine dependent config
20# engine dependent config
21categories = ['videos']
21categories = ['videos']
@@ -25,7 +25,7 @@ time_range_support = True
25number_of_results = 10
25number_of_results = 10
26
26
27search_url = 'https://www.google.com/search'\
27search_url = 'https://www.google.com/search'\
28− '?{query}'\
28+ '?q={query}'\
29 '&tbm=vid'\
29 '&tbm=vid'\
30 '&{search_options}'
30 '&{search_options}'
31time_range_attr = "qdr:{range}"
31time_range_attr = "qdr:{range}"
@@ -69,15 +69,27 @@ def response(resp):
69 # parse results
69 # parse results
70 for result in dom.xpath('//div[@class="g"]'):
70 for result in dom.xpath('//div[@class="g"]'):
71
71
72− title = extract_text(result.xpath('.//h3/a'))
72+ title = extract_text(result.xpath('.//h3'))
73− url = result.xpath('.//h3/a/@href')[0]
73+ url = result.xpath('.//div[@class="r"]/a/@href')[0]
74 content = extract_text(result.xpath('.//span[@class="st"]'))
74 content = extract_text(result.xpath('.//span[@class="st"]'))
75
75
76+ # get thumbnails
77+ script = str(dom.xpath('//script[contains(., "_setImagesSrc")]')[0].text)
78+ id = result.xpath('.//div[@class="s"]//img/@id')[0]
79+ thumbnails_data = re.findall('s=\'(.*?)(?:\\\\[a-z,1-9,\\\\]+\'|\')\;var ii=\[(?:|[\'vidthumb\d+\',]+)\'' + id,
80+ script)
81+ tmp = []
82+ if len(thumbnails_data) != 0:
83+ tmp = re.findall('(data:image/jpeg;base64,[a-z,A-Z,0-9,/,\+]+)', thumbnails_data[0])
84+ thumbnail = ''
85+ if len(tmp) != 0:
86+ thumbnail = tmp[-1]
87+
76 # append result
88 # append result
77 results.append({'url': url,
89 results.append({'url': url,
78 'title': title,
90 'title': title,
79 'content': content,
91 'content': content,
80− 'thumbnail': '',
92+ 'thumbnail': thumbnail,
81 'template': 'videos.html'})
93 'template': 'videos.html'})
82
94
83 return results
95 return results
tests/unit/engines/test_google_videos.py+23−4
@@ -30,16 +30,34 @@ class TestGoogleVideosEngine(SearxTestCase):
30 <div>
30 <div>
31 <div>
31 <div>
32 <div class="g">
32 <div class="g">
33− <div>
33+ <div class="r">
34− <h3><a href="url_1">Title 1</h3>
34+ <a href="url_1"><h3>Title 1</h3></a>
35+ </div>
36+ <div class="s">
37+ <div>
38+ <a>
39+ <g-img>
40+ <img id="vidthumb1">
41+ </g-img>
42+ </a>
43+ </div>
35 </div>
44 </div>
36 <div>
45 <div>
37 <span class="st">Content 1</span>
46 <span class="st">Content 1</span>
38 </div>
47 </div>
39 </div>
48 </div>
40 <div class="g">
49 <div class="g">
41− <div>
50+ <div class="r">
42− <h3><a href="url_2">Title 2</h3>
51+ <a href="url_2"><h3>Title 2</h3></a>
52+ </div>
53+ <div class="s">
54+ <div>
55+ <a>
56+ <g-img>
57+ <img id="vidthumb2">
58+ </g-img>
59+ </a>
60+ </div>
43 </div>
61 </div>
44 <div>
62 <div>
45 <span class="st">Content 2</span>
63 <span class="st">Content 2</span>
@@ -47,6 +65,7 @@ class TestGoogleVideosEngine(SearxTestCase):
47 </div>
65 </div>
48 </div>
66 </div>
49 </div>
67 </div>
68+ <script>function _setImagesSrc(c,d,e){}</script>
50 """
69 """
51 response = mock.Mock(text=html)
70 response = mock.Mock(text=html)
52 results = google_videos.response(response)
71 results = google_videos.response(response)