@@ -1,30 +1,58 @@
1+## Stackoverflow (It)
2+#
3+# @website https://stackoverflow.com/
4+# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)
5+#
6+# @using-api no
7+# @results HTML
8+# @stable no (HTML can change)
9+# @parse url, title, content
10+
111from urlparse import urljoin
212from cgi import escape
313from urllib import urlencode
414from lxml import html
515
16+# engine dependent config
617categories = ['it']
18+paging = True
719
20+# search-url
821url = 'http://stackoverflow.com/'
922search_url = url+'search?{query}&page={pageno}'
10−result_xpath = './/div[@class="excerpt"]//text()'
1123
12−paging = True
24+# specific xpath variables
25+results_xpath = '//div[contains(@class,"question-summary")]'
26+link_xpath = './/div[@class="result-link"]//a|.//div[@class="summary"]//h3//a'
27+title_xpath = './/text()'
28+content_xpath = './/div[@class="excerpt"]//text()'
1329
1430
31+# do search-request
1532def request(query, params):
1633 params['url'] = search_url.format(query=urlencode({'q': query}),
1734 pageno=params['pageno'])
35+
1836 return params
1937
2038
39+# get response from search-request
2140def response(resp):
2241 results = []
42+
2343 dom = html.fromstring(resp.text)
24− for result in dom.xpath('//div[@class="question-summary search-result"]'):
25− link = result.xpath('.//div[@class="result-link"]//a')[0]
44+
45+ # parse results
46+ for result in dom.xpath(results_xpath):
47+ link = result.xpath(link_xpath)[0]
2648 href = urljoin(url, link.attrib.get('href'))
27− title = escape(' '.join(link.xpath('.//text()')))
28− content = escape(' '.join(result.xpath(result_xpath)))
29− results.append({'url': href, 'title': title, 'content': content})
49+ title = escape(' '.join(link.xpath(title_xpath)))
50+ content = escape(' '.join(result.xpath(content_xpath)))
51+
52+ # append result
53+ results.append({'url': href,
54+ 'title': title,
55+ 'content': content})
56+
57+ # return results
3058 return results