git.iko.soy/searx
on master help sign in
fix stackoverflow and add comments
a46bbb4Thomas Pointhuber
searx/engines/stackoverflow.py+35−7
compare with master
unifiedsplit
@@ -1,30 +1,58 @@
1+## Stackoverflow (It)
2+#
3+# @website https://stackoverflow.com/
4+# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)
5+#
6+# @using-api no
7+# @results HTML
8+# @stable no (HTML can change)
9+# @parse url, title, content
10+
1from urlparse import urljoin
11from urlparse import urljoin
2from cgi import escape
12from cgi import escape
3from urllib import urlencode
13from urllib import urlencode
4from lxml import html
14from lxml import html
5
15
16+# engine dependent config
6categories = ['it']
17categories = ['it']
18+paging = True
7
19
20+# search-url
8url = 'http://stackoverflow.com/'
21url = 'http://stackoverflow.com/'
9search_url = url+'search?{query}&page={pageno}'
22search_url = url+'search?{query}&page={pageno}'
10−result_xpath = './/div[@class="excerpt"]//text()'
11
23
12−paging = True
24+# specific xpath variables
25+results_xpath = '//div[contains(@class,"question-summary")]'
26+link_xpath = './/div[@class="result-link"]//a|.//div[@class="summary"]//h3//a'
27+title_xpath = './/text()'
28+content_xpath = './/div[@class="excerpt"]//text()'
13
29
14
30
31+# do search-request
15def request(query, params):
32def request(query, params):
16 params['url'] = search_url.format(query=urlencode({'q': query}),
33 params['url'] = search_url.format(query=urlencode({'q': query}),
17 pageno=params['pageno'])
34 pageno=params['pageno'])
35+
18 return params
36 return params
19
37
20
38
39+# get response from search-request
21def response(resp):
40def response(resp):
22 results = []
41 results = []
42+
23 dom = html.fromstring(resp.text)
43 dom = html.fromstring(resp.text)
44+
45+ # parse results
24− for result in dom.xpath('//div[@class="question-summary search-result"]'):
46+ for result in dom.xpath(results_xpath):
25− link = result.xpath('.//div[@class="result-link"]//a')[0]
47+ link = result.xpath(link_xpath)[0]
26 href = urljoin(url, link.attrib.get('href'))
48 href = urljoin(url, link.attrib.get('href'))
27− title = escape(' '.join(link.xpath('.//text()')))
49+ title = escape(' '.join(link.xpath(title_xpath)))
28− content = escape(' '.join(result.xpath(result_xpath)))
50+ content = escape(' '.join(result.xpath(content_xpath)))
51+
52+ # append result
29− results.append({'url': href, 'title': title, 'content': content})
53+ results.append({'url': href,
54+ 'title': title,
55+ 'content': content})
56+
57+ # return results
30 return results
58 return results
searx/settings.yml−1
@@ -90,7 +90,6 @@ engines:
90
90
91 - name : stackoverflow
91 - name : stackoverflow
92 engine : stackoverflow
92 engine : stackoverflow
93− categories : it
94 shortcut : st
93 shortcut : st
95
94
96 - name : startpage
95 - name : startpage