@@ -1,64 +1,99 @@
1−#!/usr/bin/env python
1+## Yahoo (Web)
2+#
3+# @website https://search.yahoo.com/web
4+# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries
5+#
6+# @using-api no (because pricing)
7+# @results HTML (using search portal)
8+# @stable no (HTML can change)
9+# @parse url, title, content, suggestion
210
311from urllib import urlencode
412from urlparse import unquote
513from lxml import html
614from searx.engines.xpath import extract_text, extract_url
715
16+# engine dependent config
817categories = ['general']
9−search_url = 'http://search.yahoo.com/search?{query}&b={offset}'
18+paging = True
19+language_support = True
20+
21+# search-url
22+search_url = 'https://search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}'
23+
24+# specific xpath variables
1025results_xpath = '//div[@class="res"]'
1126url_xpath = './/h3/a/@href'
1227title_xpath = './/h3/a'
1328content_xpath = './/div[@class="abstr"]'
1429suggestion_xpath = '//div[@id="satat"]//a'
1530
16−paging = True
17−
1831
32+# remove yahoo-specific tracking-url
1933def parse_url(url_string):
2034 endings = ['/RS', '/RK']
2135 endpositions = []
2236 start = url_string.find('http', url_string.find('/RU=')+1)
37+
2338 for ending in endings:
2439 endpos = url_string.rfind(ending)
2540 if endpos > -1:
2641 endpositions.append(endpos)
2742
2843 end = min(endpositions)
44+
2945 return unquote(url_string[start:end])
3046
3147
48+# do search-request
3249def request(query, params):
3350 offset = (params['pageno'] - 1) * 10 + 1
51+
3452 if params['language'] == 'all':
3553 language = 'en'
3654 else:
3755 language = params['language'].split('_')[0]
56+
3857 params['url'] = search_url.format(offset=offset,
39− query=urlencode({'p': query}))
58+ query=urlencode({'p': query}),
59+ lang=language)
60+
61+ # TODO required?
4062 params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\
4163 .format(lang=language)
64+
4265 return params
4366
4467
68+# get response from search-request
4569def response(resp):
4670 results = []
71+
4772 dom = html.fromstring(resp.text)
4873
74+ # parse results
4975 for result in dom.xpath(results_xpath):
5076 try:
5177 url = parse_url(extract_url(result.xpath(url_xpath), search_url))
5278 title = extract_text(result.xpath(title_xpath)[0])
5379 except:
5480 continue
81+
5582 content = extract_text(result.xpath(content_xpath)[0])
56− results.append({'url': url, 'title': title, 'content': content})
5783
84+ # append result
85+ results.append({'url': url,
86+ 'title': title,
87+ 'content': content})
88+
89+ # if no suggestion found, return results
5890 if not suggestion_xpath:
5991 return results
6092
93+ # parse suggestion
6194 for suggestion in dom.xpath(suggestion_xpath):
95+ # append suggestion
6296 results.append({'suggestion': extract_text(suggestion)})
6397
98+ # return results
6499 return results