on master help sign in
Merge branch 'master' into boilerplate
commit7beb49b1fb6f4bea5f4b99b853ab30ac47505790 authorMarkus Heiser <markus.heiser@darmarIT.de> dateTue 10 Dec 2019, 13:10 (7 years ago) parent1b90e14e9311ee
3 files changed+47−95
AUTHORS.rst+3−1 searx/engines/google.py+7−14 tests/unit/engines/test_google.py+37−80
AUTHORS.rst+3−1
@@ -1,4 +1,4 @@
1−Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament and Noémi Ványi.
1+Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament, Noémi Ványi, @pofilo and Markus Heiser.
22
33Major contributing authors:
44
@@ -9,6 +9,8 @@ Major contributing authors:
99- @Cqoicebordel
1010- Noémi Ványi
1111- Marc Abonce Seguin @a01200356
12+- @pofilo
13+- Markus Heiser @return42
1214
1315People who have submitted patches/translates, reported bugs, consulted features or
1416generally made searx better:
searx/engines/google.py+7−14
@@ -107,13 +107,12 @@ images_path = '/images'
107107supported_languages_url = 'https://www.google.com/preferences?#languages'
108108
109109# specific xpath variables
110−results_xpath = '//div[@class="g"]'
111−url_xpath = './/h3/a/@href'
112−title_xpath = './/h3'
113−content_xpath = './/span[@class="st"]'
114−content_misc_xpath = './/div[@class="f slp"]'
115−suggestion_xpath = '//p[@class="_Bmc"]'
116−spelling_suggestion_xpath = '//a[@class="spell"]'
110+results_xpath = '//div[contains(@class, "ZINbbc")]'
111+url_xpath = './/div[@class="kCrYT"][1]/a/@href'
112+title_xpath = './/div[@class="kCrYT"][1]/a/div[1]'
113+content_xpath = './/div[@class="kCrYT"][2]//div[contains(@class, "BNeawe")]//div[contains(@class, "BNeawe")]'
114+suggestion_xpath = '//div[contains(@class, "ZINbbc")][last()]//div[@class="rVLSBd"]/a//div[contains(@class, "BNeawe")]'
115+spelling_suggestion_xpath = '//div[@id="scc"]//a'
117116
118117# map : detail location
119118map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'
@@ -199,10 +198,6 @@ def request(query, params):
199198 params['headers']['Accept-Language'] = language + ',' + language + '-' + country
200199 params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
201200
202− # Force Safari 3.1 on Mac OS X (Leopard) user agent to avoid loading the new UI that Searx can't parse
203− params['headers']['User-Agent'] = ("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4)"
204− "AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1")
205−
206201 params['google_hostname'] = google_hostname
207202
208203 return params
@@ -274,9 +269,7 @@ def response(resp):
274269 content = extract_text_from_dom(result, content_xpath)
275270 if content is None:
276271 continue
277− content_misc = extract_text_from_dom(result, content_misc_xpath)
278− if content_misc is not None:
279− content = content_misc + "<br />" + content
272+
280273 # append result
281274 results.append({'url': url,
282275 'title': title,
tests/unit/engines/test_google.py+37−80
@@ -58,93 +58,50 @@ class TestGoogleEngine(SearxTestCase):
5858 self.assertEqual(google.response(response), [])
5959
6060 html = """
61− <div class="g">
62− <h3 class="r">
63− <a href="http://this.should.be.the.link/">
64− <b>This</b> is <b>the</b> title
65− </a>
66− </h3>
67− <div class="s">
68− <div class="kv" style="margin-bottom:2px">
69− <cite>
70− <b>test</b>.psychologies.com/
71− </cite>
72− <div class="_nBb"><U+200E>
73− <div style="display:inline" onclick="google.sham(this);" aria-expanded="false"
74− aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">
75− <span class="_O0">
76− </span>
61+ <div class="ZINbbc xpd O9g5cc uUPGi">
62+ <div>
63+ <div class="kCrYT">
64+ <a href="/url?q=http://this.should.be.the.link/">
65+ <div class="BNeawe">
66+ <b>This</b> is <b>the</b> title
7767 </div>
78− <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1">
79− <ul>
80− <li class="_Ykb">
81− <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent
82− .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/">
83− En cache
84− </a>
85− </li>
86− <li class="_Ykb">
87− <a class="_Zkb" href="/search?safe=off&amp;q=related:test.psy.com/">
88− Pages similaires
89− </a>
90− </li>
91− </ul>
68+ <div class="BNeawe">
69+ http://website
70+ </div>
71+ </a>
72+ </div>
73+ <div class="kCrYT">
74+ <div>
75+ <div class="BNeawe">
76+ <div>
77+ <div class="BNeawe">
78+ This should be the content.
79+ </div>
80+ </div>
9281 </div>
9382 </div>
9483 </div>
95− <span class="st">
96− This should be the content.
97− </span>
98− <br>
99− <div class="osl"><U+200E>
100− <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/">
101− Test Personnalité
102− </a> - <U+200E>
103− <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/">
104− Tests - Moi
105− </a> - <U+200E>
106− <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">
107− Test Couple
108− </a>
109− - <U+200E>
110− <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">
111− Test Amour
84+ </div>
85+ </p>
86+ <div class="ZINbbc xpd O9g5cc uUPGi">
87+ <div>
88+ <div class="kCrYT">
89+ <span>
90+ <div class="BNeawe">
91+ Related searches
92+ </div>
93+ </span>
94+ </div>
95+ <div class="rVLSBd">
96+ <a>
97+ <div>
98+ <div class="BNeawe">
99+ suggestion title
100+ </div>
101+ </div>
112102 </a>
113103 </div>
114104 </div>
115− </div>
116− <div class="g">
117− <h3 class="r">
118− <a href="http://www.google.com/images?q=toto">
119− <b>This</b>
120− </a>
121− </h3>
122− </div>
123− <div class="g">
124− <h3 class="r">
125− <a href="http://www.google.com/search?q=toto">
126− <b>This</b> is
127− </a>
128− </h3>
129− </div>
130− <div class="g">
131− <h3 class="r">
132− <a href="€">
133− <b>This</b> is <b>the</b>
134− </a>
135− </h3>
136− </div>
137− <div class="g">
138− <h3 class="r">
139− <a href="/url?q=url">
140− <b>This</b> is <b>the</b>
141− </a>
142− </h3>
143− </div>
144− <p class="_Bmc" style="margin:3px 8px">
145− <a href="/search?num=20&amp;safe=off&amp;q=t&amp;revid=1754833769&amp;sa=X&amp;ei=-&amp;ved=">
146− suggestion <b>title</b>
147− </a>
148105 </p>
149106 """
150107 response = self.mock_response(html)