@@ -1,47 +1,79 @@
3+# @website https://startpage.com
4+# @provide-api no (nothing found)
8+# @stable no (HTML can change)
9+# @parse url, title, content
1from urllib import urlencode
13from urllib import urlencode
2from lxml import html
14from lxml import html
3from cgi import escape
15from cgi import escape
18+# engine dependent config
19+categories = ['general']
20+# there is a mechanism to block "bot" search (probably the parameter qid), require storing of qid's between mulitble search-calls
22+language_support = True
5−base_url = None
25+base_url = 'https://startpage.com/'
6−search_url = None
26+search_url = base_url + 'do/search'
8−# TODO paging
28+# specific xpath variables
9−paging = False
29+# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
10−# TODO complete list of country mapping
30+# not ads: div[@class="result"] are the direct childs of div[@id="results"]
11−country_map = {'en_US': 'eng',
31+results_xpath = '//div[@class="result"]'
12− 'en_UK': 'uk',
32+link_xpath = './/h3/a'
16def request(query, params):
36def request(query, params):
37+ offset = (params['pageno'] - 1) * 10
17 query = urlencode({'q': query})[2:]
38 query = urlencode({'q': query})[2:]
18 params['url'] = search_url
40 params['url'] = search_url
19 params['method'] = 'POST'
41 params['method'] = 'POST'
20 params['data'] = {'query': query,
42 params['data'] = {'query': query,
21− 'startat': (params['pageno'] - 1) * 10} # offset
43+ 'startat': offset}
22− country = country_map.get(params['language'], 'eng')
44+
23− params['cookies']['preferences'] = \
45+ # set language if specified
24− 'lang_homepageEEEs/air/{country}/N1NsslEEE1N1Nfont_sizeEEEmediumN1Nrecent_results_filterEEE1N1Nlanguage_uiEEEenglishN1Ndisable_open_in_new_windowEEE0N1Ncolor_schemeEEEnewN1Nnum_of_resultsEEE10N1N'.format(country=country) # noqa
46+ if params['language'] != 'all':
47+ params['data']['with_language'] = 'lang_' + params['language'].split('_')[0]
25 return params
49 return params
52+# get response from search-request
28def response(resp):
53def response(resp):
29 results = []
54 results = []
30 dom = html.fromstring(resp.content)
56 dom = html.fromstring(resp.content)
31− # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
57+
32− # not ads: div[@class="result"] are the direct childs of div[@id="results"]
58+ # parse results
33− for result in dom.xpath('//div[@class="result"]'):
59+ for result in dom.xpath(results_xpath):
34− link = result.xpath('.//h3/a')[0]
60+ link = result.xpath(link_xpath)[0]
35 url = link.attrib.get('href')
61 url = link.attrib.get('href')
36− if url.startswith('http://www.google.')\
37− or url.startswith('https://www.google.'):
39 title = escape(link.text_content())
62 title = escape(link.text_content())
41− content = ''
64+ # block google-ad url's
65+ if re.match("^http(s|)://www.google.[a-z]+/aclk.*$", url):
42 if result.xpath('./p[@class="desc"]'):
68 if result.xpath('./p[@class="desc"]'):
43 content = escape(result.xpath('./p[@class="desc"]')[0].text_content())
69 content = escape(result.xpath('./p[@class="desc"]')[0].text_content())
45− results.append({'url': url, 'title': title, 'content': content})
74+ results.append({'url': url,
47 return results
79 return results